@akterjs/effect 0.0.0-stage → 0.1.1

package/src/host/fork/fork-computers.adapter.ts

@@ -0,0 +1,366 @@
+import {
+  Array as Arr,
+  Clock,
+  Context,
+  Effect,
+  FileSystem,
+  Layer,
+  Match,
+  Option,
+  Path,
+  Semaphore,
+  String as Str,
+} from "effect";
+
+import { ActorUnavailableError } from "#src/contract/client/actor-unavailable.errors";
+import { InvalidRequestError } from "#src/contract/client/invalid-request.errors";
+import { ActorComputerClients } from "#src/contract/computer/computer-client.adapter";
+import type { ComputerAddress, ComputerClient } from "#src/contract/computer/computer-client.contract";
+import { localLog } from "#src/host/computer/local-computer-log.layer";
+import { admitted, findRecord, listed, placeOf, saveRecord } from "#src/host/computer/local-computer-records.layer";
+import type { Failing, Place, Store } from "#src/host/computer/local-computer-records.layer";
+
+/** One actor fork's computers: the parent's, copied into the child, under the store's computers directory. */
+interface ComputersFork {
+  readonly child: ComputerAddress;
+  /** The fork's id: the hold it keeps on the parent's computers, and the name of the mark on the child's. */
+  readonly fork: string;
+  readonly parent: ComputerAddress;
+  readonly root: string;
+}
+
+/** One of the parent's computers the fork copies. */
+interface Held {
+  readonly name: string;
+}
+
+type ForkFailure = ActorUnavailableError | InvalidRequestError;
+
+type Needs = FileSystem.FileSystem | Path.Path;
+
+type Forked<Value> = Effect.Effect<Value, ForkFailure, Needs>;
+
+type Local<Value> = Effect.Effect<Value, Failing, Needs>;
+
+interface Found {
+  readonly place: Place;
+}
+
+const markPrefix = "@fork-";
+
+const oneAtATime = 1;
+
+// A computer's name starts with a letter and a key segment never holds "@", so a mark is never read as either.
+const markOf = (fork: string): string => `${markPrefix}${encodeURIComponent(fork)}`;
+
+const storeOf = Effect.fnUntraced(function* storeOf(root: string) {
+  const path = yield* Path.Path;
+  const admission = yield* Semaphore.make(oneAtATime);
+  const store: Store = { admission, path, root };
+
+  return store;
+});
+
+const failedWith =
+  (actor: string) =>
+  (failure: Failing): ForkFailure =>
+    Match.value(failure).pipe(
+      Match.tags({
+        ActorUnavailableError: (unavailable): ForkFailure => unavailable,
+        ComputerHeldError: ({ computer, hold }) =>
+          ActorUnavailableError.make({
+            actor,
+            message: `${actor} can't fork while its computer ${computer} is held by ${hold}, a fork that didn't finish. Retry that fork, or resume the computer with the hold ${hold}, to release it.`,
+          }),
+        ComputerLimitError: ({ maximum }) =>
+          InvalidRequestError.make({
+            actor,
+            message: `${actor} can't fork: copying its computers would pass the store's limit of ${String(maximum)} computers. Remove a computer, then fork again.`,
+          }),
+        InvalidRequestError: (invalid): ForkFailure => invalid,
+      }),
+      Match.orElse(() =>
+        ActorUnavailableError.make({
+          actor,
+          message: "the fork didn't copy its computers. Retry it with the same id.",
+        }),
+      ),
+    );
+
+const answered =
+  (actor: string) =>
+  <Value>(effect: Local<Value>): Forked<Value> =>
+    Effect.mapError(effect, failedWith(actor));
+
+const foundIn = (place: Place): Local<Option.Option<Found>> =>
+  Effect.map(
+    findRecord(place),
+    Option.map(() => ({ place })),
+  );
+
+/** The parent's computers as their directories record them, by name. */
+const parentComputers = Effect.fnUntraced(function* parentComputers(store: Store, parent: ComputerAddress) {
+  const names = yield* listed(placeOf(store)(parent, "").actor);
+  const places = Arr.map(Arr.sort(names, Str.Order), (name) => placeOf(store)(parent, name));
+  const found = yield* Effect.forEach(places, foundIn);
+
+  return Arr.getSomes(found);
+});
+
+/** The parent's computer client when the engine runs computers; none when nothing can change them meanwhile. */
+const clientOf = (parent: ComputerAddress): Effect.Effect<Option.Option<ComputerClient>> =>
+  Effect.map(
+    Effect.serviceOption(ActorComputerClients),
+    Option.map((clients) => clients.connect(parent)),
+  );
+
+/** A computer that is gone, or that no engine runs, needs no hold: nothing can change it while the fork copies it. */
+const unheld = <Value>(effect: Local<Value>): Local<void> =>
+  Effect.asVoid(effect).pipe(
+    Effect.catchTags({
+      ComputerLostError: () => Effect.void,
+      ComputerNotFoundError: () => Effect.void,
+      ComputersUnavailableError: () => Effect.void,
+    }),
+  );
+
+const heldOf = ({ place }: Found): Held => ({ name: place.name });
+
+/**
+ * Holds each of the parent's computers paused under the fork's id, so no step or ensure changes one until the fork
+ * lets it go, and answers which the fork copies. A hold survives a restart, and the same hold taken again is a resend.
+ */
+const holdParent = ({ fork, parent, root }: ComputersFork): Forked<readonly Held[]> =>
+  answered(parent.actor)(
+    Effect.gen(function* holdComputers() {
+      const store = yield* storeOf(root);
+      const computers = yield* parentComputers(store, parent);
+      const client = yield* clientOf(parent);
+
+      if (Option.isSome(client)) {
+        const hold = ({ place }: Found): Local<void> => unheld(client.value.pause(place.name, { hold: fork }));
+
+        yield* Effect.forEach(computers, hold, { discard: true });
+      }
+
+      return Arr.map(computers, heldOf);
+    }),
+  );
+
+/** Copies one computer's disk and spec into the child, stopped: a fork copies its files and none of its processes. */
+const copyOne = Effect.fnUntraced(function* copyOne(source: Place, target: Place) {
+  const fs = yield* FileSystem.FileSystem;
+  const record = yield* findRecord(source);
+
+  if (Option.isNone(record)) {
+    return;
+  }
+
+  yield* fs.makeDirectory(target.directory, { recursive: true });
+
+  if (yield* fs.exists(source.disk)) {
+    yield* fs.copy(source.disk, target.disk);
+  }
+
+  const createdAt = yield* Clock.currentTimeMillis;
+
+  yield* saveRecord(target, { createdAt, spec: record.value.spec, state: "stopped" });
+});
+
+const copied =
+  (store: Store, { child, fork, parent }: ComputersFork) =>
+  ({ name }: Held): Local<void> => {
+    const source = placeOf(store)(parent, name);
+    const target = placeOf(store)(child, name);
+    const note = `copied from ${parent.actor}/${parent.key} by the fork ${fork}, stopped`;
+
+    return Effect.andThen(
+      admitted({ computer: name, store })(copyOne(source, target)),
+      localLog(store.path).note(target, "info", note),
+    );
+  };
+
+const alreadyKept = ({ child, parent }: ComputersFork): ActorUnavailableError =>
+  ActorUnavailableError.make({
+    actor: parent.actor,
+    message: `computers are already kept for ${child.key}, which isn't stored, so the fork can't copy its own`,
+  });
+
+const copyInto = Effect.fnUntraced(function* copyInto(forking: ComputersFork, held: readonly Held[]) {
+  const fs = yield* FileSystem.FileSystem;
+  const store = yield* storeOf(forking.root);
+  const directory = placeOf(store)(forking.child, "").actor;
+
+  if (yield* fs.exists(directory)) {
+    return yield* alreadyKept(forking);
+  }
+
+  yield* fs.makeDirectory(store.path.join(directory, markOf(forking.fork)), { recursive: true });
+
+  return yield* Effect.forEach(held, copied(store, forking), { discard: true });
+});
+
+/**
+ * Copies the held computers into the child before it is published. The child's computers directory is marked with
+ * the fork's id first, so a fork that never publishes leaves computers its retry or the next host start removes.
+ */
+const copyHeld = ({ held, ...forking }: ComputersFork & { readonly held: readonly Held[] }): Forked<void> => {
+  if (!Arr.isReadonlyArrayNonEmpty(held)) {
+    return Effect.void;
+  }
+
+  return answered(forking.parent.actor)(copyInto(forking, held));
+};
+
+const released =
+  (client: ComputerClient, fork: string) =>
+  ({ name }: Held): Local<void> =>
+    unheld(client.resume(name, { hold: fork }));
+
+/** Lets go of the fork's holds: each computer returns where it was, a stopped one stopped. */
+const releaseParent = (client: ComputerClient, fork: string, held: readonly Held[]): Local<void> =>
+  Effect.forEach(held, released(client, fork), { discard: true });
+
+/** Every computer of the parent: a fork that crashed no longer knows which it held. */
+const everyComputer = (store: Store, parent: ComputerAddress): Local<readonly Held[]> =>
+  Effect.map(parentComputers(store, parent), Arr.map(heldOf));
+
+const releaseHeld = Effect.fnUntraced(function* releaseHeld(
+  store: Store,
+  forking: ComputersFork,
+  held: Option.Option<readonly Held[]>,
+) {
+  const client = yield* clientOf(forking.parent);
+
+  if (Option.isNone(client)) {
+    return;
+  }
+
+  const letGo = yield* Option.match(held, {
+    onNone: () => everyComputer(store, forking.parent),
+    onSome: Effect.succeed,
+  });
+
+  yield* releaseParent(client.value, forking.fork, letGo);
+});
+
+type Releasing = ComputersFork & { readonly held: Option.Option<readonly Held[]> };
+
+/** Lets go of the parent's computers after a fork failed, keeping the mark so the child's copies are discarded. */
+const releaseComputers = ({ held, ...forking }: Releasing): Forked<void> =>
+  answered(forking.parent.actor)(Effect.flatMap(storeOf(forking.root), (store) => releaseHeld(store, forking, held)));
+
+/**
+ * Finishes a published fork's computers: removes the mark from the child's and lets go of the parent's. A retry of a
+ * fork that crashed after publishing passes no `held`, so every computer is let go of.
+ */
+const settleComputers = ({ held, ...forking }: Releasing): Forked<void> =>
+  answered(forking.parent.actor)(
+    Effect.gen(function* settleFork() {
+      const fs = yield* FileSystem.FileSystem;
+      const store = yield* storeOf(forking.root);
+      const mark = store.path.join(placeOf(store)(forking.child, "").actor, markOf(forking.fork));
+
+      yield* fs.remove(mark, { force: true, recursive: true });
+      yield* releaseHeld(store, forking, held);
+    }),
+  );
+
+/** Removes the computers a fork that never published left for a child that isn't stored: only a marked directory. */
+const discardComputers = (forking: ComputersFork): Forked<void> =>
+  answered(forking.parent.actor)(
+    Effect.gen(function* discardChild() {
+      const fs = yield* FileSystem.FileSystem;
+      const store = yield* storeOf(forking.root);
+      const directory = placeOf(store)(forking.child, "").actor;
+      const names = yield* listed(directory);
+
+      if (Arr.some(names, Str.startsWith(markPrefix))) {
+        yield* fs.remove(directory, { force: true, recursive: true });
+      }
+    }),
+  );
+
+/** A stored child keeps its computers and loses the mark; a child that isn't stored loses its computers. */
+const sweptPath =
+  (path: Path.Path, root: string, stored: ReadonlySet<string>) =>
+  (entry: string): string => {
+    const mark = path.join(root, entry);
+    const owner = path.dirname(mark);
+
+    if (stored.has(Str.replaceAll(path.sep, "/")(path.relative(root, owner)))) {
+      return mark;
+    }
+
+    return owner;
+  };
+
+/**
+ * At a host's start, settles every mark crashed forks left on a child's computers. `storedActors` names each stored
+ * actor as `<Actor>/<key segment>`, and is read only when a mark is found.
+ */
+const sweepComputers = Effect.fnUntraced(function* sweepComputers<Unreadable>(
+  root: string,
+  storedActors: Effect.Effect<ReadonlySet<string>, Unreadable>,
+) {
+  const fs = yield* FileSystem.FileSystem;
+  const path = yield* Path.Path;
+
+  if (!(yield* fs.exists(root))) {
+    return;
+  }
+
+  const marks = yield* fs.glob(`*/*/${markPrefix}*/`, { root });
+
+  if (!Arr.isReadonlyArrayNonEmpty(marks)) {
+    return;
+  }
+
+  const stored = yield* storedActors;
+  const swept = Arr.map(marks, sweptPath(path, root, stored));
+
+  yield* Effect.forEach(swept, (removed) => fs.remove(removed, { force: true, recursive: true }), { discard: true });
+});
+
+type Swept = Effect.Error<ReturnType<FileSystem.FileSystem["remove"]>>;
+
+/** What `ForkComputers` offers, with the file system it runs on provided when its layer is built. */
+interface ForkComputersOperations {
+  readonly copy: (forking: ComputersFork & { readonly held: readonly Held[] }) => Effect.Effect<void, ForkFailure>;
+  readonly discard: (forking: ComputersFork) => Effect.Effect<void, ForkFailure>;
+  readonly hold: (forking: ComputersFork) => Effect.Effect<readonly Held[], ForkFailure>;
+  readonly release: (forking: Releasing) => Effect.Effect<void, ForkFailure>;
+  readonly settle: (forking: Releasing) => Effect.Effect<void, ForkFailure>;
+  readonly sweep: <Unreadable>(
+    root: string,
+    storedActors: Effect.Effect<ReadonlySet<string>, Unreadable>,
+  ) => Effect.Effect<void, Unreadable | Swept>;
+}
+
+const make = Effect.gen(function* makeForkComputers() {
+  const services = yield* Effect.context<Needs>();
+
+  const operations: ForkComputersOperations = {
+    copy: (forking) => Effect.provideContext(copyHeld(forking), services),
+    discard: (forking) => Effect.provideContext(discardComputers(forking), services),
+    hold: (forking) => Effect.provideContext(holdParent(forking), services),
+    release: (forking) => Effect.provideContext(releaseComputers(forking), services),
+    settle: (forking) => Effect.provideContext(settleComputers(forking), services),
+    sweep: (root, storedActors) => Effect.provideContext(sweepComputers(root, storedActors), services),
+  };
+
+  return operations;
+});
+
+/**
+ * The computers of an actor fork on the local engine: `hold` the parent's before the cut, `copy` them into the child
+ * before it is published, stopped, then `settle` once it is, or `release` after a failure, so `discard` finds the
+ * copies to remove. `sweep` settles what crashed forks left when a host starts.
+ */
+export class ForkComputers extends Context.Service<ForkComputers, ForkComputersOperations>()(
+  "@akterjs/effect/host/fork/fork-computers.adapter/ForkComputers",
+) {
+  public static readonly layer: Layer.Layer<ForkComputers, never, Needs> = Layer.effect(this, make);
+}
+
+export type { ComputersFork, Held };

package/src/host/hash/blob-hashing.service.ts

@@ -0,0 +1,14 @@
+import { Context } from "effect";
+import type { Effect } from "effect";
+
+interface BlobDigest {
+  readonly digest: Effect.Effect<string>;
+  readonly update: (bytes: Uint8Array) => Effect.Effect<void>;
+}
+
+export class BlobHashing extends Context.Service<
+  BlobHashing,
+  {
+    readonly sha256: Effect.Effect<BlobDigest>;
+  }
+>()("@akterjs/effect/host/hash/blob-hashing.service/BlobHashing") {}

package/src/host/job/job-durables.sql.ts

@@ -0,0 +1,214 @@
+import { Array as Arr, Clock, Deferred, Duration, Effect, Exit, FiberMap, MutableHashMap, Option } from "effect";
+import type { Scope } from "effect";
+import { WorkflowEngine } from "effect/workflow";
+import type { DurableClock, DurableDeferred } from "effect/workflow";
+
+import type { ActorUnavailableError } from "#src/contract/client/actor-unavailable.errors";
+import { jobAddress } from "#src/contract/job/job-id.pure";
+import type { JobAddress } from "#src/contract/job/job-id.pure";
+import type { StartedJob } from "#src/contract/job/started-job.schema";
+import { readStored, storeExit, storedExit, writeStored } from "./job-journal.sql.js";
+import { resume, trackedClock } from "./job-runs.pure.js";
+import type { Execution, JobStore, Runs } from "./job-runs.pure.js";
+
+/**
+ * The store of the session that has `address` open, opening the actor when no session has it. The actor stays
+ * resident until the scope closes, so a signal or poll from outside the job can't lose its session halfway. It fails
+ * with `ActorUnavailableError` when this host can't open the actor, such as when another host serves it.
+ */
+type Locate = (address: JobAddress) => Effect.Effect<JobStore, ActorUnavailableError, Scope.Scope>;
+
+type Encoded = WorkflowEngine.Encoded;
+
+interface Waiting {
+  readonly job: StartedJob;
+  readonly store: JobStore;
+}
+
+interface Engine extends Runs {
+  readonly locator: Deferred.Deferred<Locate>;
+  readonly waiting: MutableHashMap.MutableHashMap<string, Waiting>;
+}
+
+interface DoneDeferred {
+  readonly deferredName: string;
+  readonly executionId: string;
+  readonly exit: Exit.Exit<unknown, unknown>;
+}
+
+interface Clocked {
+  readonly clock: DurableClock.DurableClock;
+  readonly executionId: string;
+}
+
+interface ClockDue {
+  readonly at: number;
+  readonly now: number;
+}
+
+const noWait = 0;
+
+const located = (engine: Engine, executionId: string): Effect.Effect<JobStore, ActorUnavailableError, Scope.Scope> =>
+  Option.match(jobAddress(executionId), {
+    onNone: () => Effect.die(`${executionId} is not an Akter job id: start jobs with Akter.start in a command`),
+    onSome: (address) => Effect.flatMap(Deferred.await(engine.locator), (locate) => locate(address)),
+  });
+
+const storeOf = ({
+  engine,
+  executionId,
+}: {
+  readonly engine: Engine;
+  readonly executionId: string;
+}): Effect.Effect<JobStore, never, Scope.Scope> =>
+  Effect.suspend(() =>
+    Option.match(MutableHashMap.get(engine.executions, executionId), {
+      onNone: () => Effect.orDie(located(engine, executionId)),
+      onSome: (running) => Effect.succeed(running.store),
+    }),
+  );
+
+const executionOf = (engine: Engine): Effect.Effect<Execution, never, WorkflowEngine.WorkflowInstance> =>
+  WorkflowEngine.WorkflowInstance.use((instance) =>
+    Option.match(MutableHashMap.get(engine.executions, instance.executionId), {
+      onNone: () => Effect.die(`job ${instance.executionId} is not running here`),
+      onSome: Effect.succeed,
+    }),
+  );
+
+const isCurrent = (executionId: string): Effect.Effect<boolean> =>
+  Effect.map(Effect.serviceOption(WorkflowEngine.WorkflowInstance), (current) =>
+    Option.exists(current, (instance) => instance.executionId === executionId),
+  );
+
+const wakeDeferred = (engine: Engine, { deferredName, executionId }: DoneDeferred): Effect.Effect<void> =>
+  Effect.gen(function* waking() {
+    const wake = Effect.andThen(
+      engine.deferreds.deferredDone(executionId, deferredName),
+      resume({ executionId, mode: "automatic", runs: engine }),
+    );
+
+    if (yield* isCurrent(executionId)) {
+      return yield* Effect.asVoid(Effect.forkIn(wake, engine.scope));
+    }
+
+    return yield* wake;
+  });
+
+const completed = (engine: Engine, store: JobStore, done: DoneDeferred): Effect.Effect<void> =>
+  store.authority(
+    Effect.gen(function* completeWhileOwned() {
+      const { executionId, exit } = done;
+      const entry = `deferred/${done.deferredName}`;
+      const found = yield* readStored({ entry, executionId, store });
+
+      if (Option.isNone(found)) {
+        yield* storeExit({ entry, executionId, exit, store });
+        yield* wakeDeferred(engine, done);
+      }
+    }),
+  );
+
+/**
+ * A signal comes from outside the job, so it locates the actor's current session and keeps it resident until the
+ * result is stored, instead of trusting the store a running execution was started with, whose session can close
+ * between the lookup and the write. `deferredDone` has no error channel, so a signal this host can't deliver dies
+ * with the locator's `ActorUnavailableError`, which names the actor and what to do.
+ */
+const doneDeferred =
+  (engine: Engine): Encoded["deferredDone"] =>
+  (done) => {
+    const store = Effect.orDie(located(engine, done.executionId));
+
+    return Effect.scoped(Effect.flatMap(store, (found) => completed(engine, found, done)));
+  };
+
+/** A clock fires inside its own session, which interrupts it on close, so it completes on that session's store. */
+const fireClock = (engine: Engine, store: JobStore, { clock, executionId }: Clocked): Effect.Effect<void> =>
+  completed(engine, store, { deferredName: clock.deferred.name, executionId, exit: Exit.void });
+
+const clockDue = (clocked: Clocked, store: JobStore): Effect.Effect<ClockDue> =>
+  Effect.gen(function* dueAt() {
+    const { clock, executionId } = clocked;
+    const now = yield* Clock.currentTimeMillis;
+    const entry = `clock/${clock.name}`;
+    const planned = String(now + Duration.toMillis(clock.duration));
+
+    yield* writeStored({ entry, executionId, store, value: planned });
+
+    const stored = yield* readStored({ entry, executionId, store });
+
+    return { at: Number(Option.getOrElse(stored, () => planned)), now };
+  });
+
+const scheduleClock =
+  (engine: Engine): Encoded["scheduleClock"] =>
+  (_workflow, clocked) =>
+    Effect.gen(function* scheduling() {
+      const { clock, executionId } = clocked;
+      const store = yield* storeOf({ engine, executionId });
+      const fired = yield* readStored({ entry: `deferred/${clock.deferred.name}`, executionId, store });
+
+      if (Option.isSome(fired)) {
+        return;
+      }
+
+      const { at, now } = yield* clockDue(clocked, store);
+      const delay = Duration.millis(Math.max(at - now, noWait));
+
+      const firing = store.authority(Effect.delay(fireClock(engine, store, clocked), delay));
+      const key = yield* trackedClock({ executionId, name: clock.name, runs: engine });
+
+      yield* FiberMap.run(engine.clocks, key, firing, { onlyIfMissing: true });
+    }).pipe(Effect.scoped);
+
+const deferredResult =
+  (engine: Engine): Encoded["deferredResult"] =>
+  (deferred: DurableDeferred.Any) =>
+    Effect.flatMap(executionOf(engine), (execution) =>
+      execution.store
+        .authority(
+          storedExit({
+            entry: `deferred/${deferred.name}`,
+            executionId: execution.executionId,
+            store: execution.store,
+          }),
+        )
+        .pipe(
+          Effect.tap((found) =>
+            Effect.sync(() => {
+              if (Option.isNone(found)) {
+                execution.waiting.add(deferred.name);
+              }
+            }),
+          ),
+        ),
+    );
+
+/**
+ * Wakes this store's runs that wait on a deferred a committed command has since stored a result for (`Akter.signal`
+ * writes it with the command), once the command commits. Only the deferreds a run found empty are checked.
+ */
+const wakeSignalled = ({ engine, store }: { readonly engine: Engine; readonly store: JobStore }): Effect.Effect<void> =>
+  Effect.suspend(() => {
+    const waiting = Arr.flatMap(Arr.fromIterable(engine.owned.get(store) ?? []), ({ executionId, waiting: names }) =>
+      Arr.map(Arr.fromIterable(names), (deferredName) => ({ deferredName, executionId })),
+    );
+
+    return Effect.forEach(
+      waiting,
+      ({ deferredName, executionId }) =>
+        Effect.flatMap(readStored({ entry: `deferred/${deferredName}`, executionId, store }), (found) => {
+          if (Option.isNone(found)) {
+            return Effect.void;
+          }
+
+          return wakeDeferred(engine, { deferredName, executionId, exit: Exit.void });
+        }),
+      { discard: true },
+    );
+  });
+
+export { deferredResult, doneDeferred, executionOf, scheduleClock, storeOf, wakeSignalled };
+
+export type { Encoded, Engine, Locate, Waiting };