@akterjs/effect 0.0.0-stage → 0.1.1

package/src/host/job/job-results.service.ts

@@ -0,0 +1,111 @@
+import { Context, Effect, identity, Layer, Option, Schema, String as Str } from "effect";
+import { SqlError } from "effect/sql";
+
+import { Caller, actorKind } from "#src/contract/caller/caller.schema";
+import { jsonCodec } from "#src/contract/json-codec.pure";
+import { CommandRunner } from "#src/host/command/command-runner.service";
+import { resultCommandId } from "#src/host/command/command-id-reuse.pure";
+import { markUndelivered, readEntry, saveResult } from "./job-journal.sql.js";
+import { settling, stalling } from "./undelivered-result.pure.js";
+import type { Finishing, Guard, Stalling, Unstored } from "./undelivered-result.pure.js";
+
+/** A result command's journaled exit, when its handler failed or died. */
+const failedExit = Schema.decodeUnknownOption(
+  Schema.fromJsonString(Schema.TaggedStruct("Failure", { cause: Schema.Unknown })),
+);
+
+interface Delivering {
+  readonly actor: string;
+  readonly executionId: string;
+  readonly job: string;
+  readonly key: string;
+}
+
+/** The job's result is kept and the job closes, but a failed handler's state change is lost, so say why. */
+const handlerFailed =
+  (delivering: Delivering) =>
+  (failed: Option.Option<unknown>): Effect.Effect<void> =>
+    Effect.forEach(
+      Option.toArray(failed),
+      (cause) =>
+        Effect.annotateLogs(
+          Effect.logError("a job's result handler failed, so its state change was not applied", cause),
+          { ...delivering },
+        ),
+      { discard: true },
+    );
+
+const delivering = (runner: CommandRunner["Service"]): ((finishing: Finishing) => Effect.Effect<void, Unstored>) =>
+  Effect.fn("JobResults.deliver")(function* deliver({
+    executionId,
+    job,
+    key,
+    measured = identity,
+    opened,
+    result,
+  }: Finishing) {
+    const actor = opened.actor.name;
+    const prefix = `${actor}/`;
+
+    if (!Str.startsWith(prefix)(job)) {
+      return yield* Effect.die(`${job} is not a job of ${actor}`);
+    }
+
+    const principal = yield* Effect.orDie(
+      Schema.encodeEffect(jsonCodec(Caller))({ kind: actorKind, principal: { actor, id: `${actor}/${key}`, key } }),
+    );
+
+    const command = runner
+      .result({
+        opened,
+        request: {
+          commandId: resultCommandId(executionId),
+          job: Str.slice(prefix.length)(job),
+          key,
+          payload: result,
+          principal,
+        },
+      })
+      .pipe(
+        Effect.map((reply) => failedExit(reply).pipe(Option.map(({ cause }) => cause))),
+        Effect.catchIf(
+          (failure) => !SqlError.isSqlError(failure),
+          (failure) => Effect.succeedSome<unknown>(failure),
+        ),
+        Effect.flatMap(handlerFailed({ actor, executionId, job, key })),
+      );
+
+    return yield* opened.sql.withTransaction(
+      Effect.andThen(saveResult({ executionId, sql: opened.sql, value: result }), measured(command)),
+    );
+  });
+
+export class JobResults extends Context.Service<
+  JobResults,
+  {
+    /**
+     * Stores the result and runs its command in one transaction. Past the storage ceiling a result command that grows
+     * the database is refused, and the result rolls back with it, so a retry delivers it once the store has room.
+     */
+    readonly deliver: (finishing: Finishing) => Effect.Effect<void, Unstored>;
+    /**
+     * Delivers with retries: a failed attempt waits out a capped backoff, and the job's journal shows it as
+     * undelivered. After the last retry the job stays open and is due again later, on a longer capped backoff.
+     */
+    readonly settle: (finishing: Finishing, guard: Guard) => Effect.Effect<void>;
+    /** Records a run that ended before its result was stored as a spent round, so the job is due at its next. */
+    readonly stall: (stalled: Stalling, guard: Guard) => Effect.Effect<void, SqlError.SqlError>;
+  }
+>()("@akterjs/effect/host/job/job-results.service/JobResults", {
+  make: Effect.gen(function* makeJobResults() {
+    const runner = yield* CommandRunner;
+
+    const deliver = delivering(runner);
+
+    const delivery = { deliver, mark: markUndelivered, read: readEntry };
+
+    return { deliver, settle: settling(delivery), stall: stalling(delivery) } as const;
+  }),
+}) {
+  public static readonly layer = Layer.effect(this, this.make).pipe(Layer.provide(CommandRunner.layer));
+}

package/src/host/job/job-runs.pure.ts

@@ -0,0 +1,445 @@
+import {
+  Array as Arr,
+  Cause,
+  Context,
+  Deferred,
+  Effect,
+  Exit,
+  Fiber,
+  FiberMap,
+  MutableHashMap,
+  Option,
+  Predicate,
+  Scope,
+} from "effect";
+import { Workflow, WorkflowEngine } from "effect/workflow";
+import type { SqlClient } from "effect/sql";
+import type { ExecutionGuard } from "#src/host/upkeep/execution-authority.pure";
+
+type Execute = Parameters<WorkflowEngine.Encoded["register"]> extends readonly [Workflow.Any, infer Run] ? Run : never;
+
+type Payload = Parameters<Execute> extends readonly [infer Input, string] ? Input : never;
+
+type WakeMode = "explicit" | "automatic";
+
+type RunResult = Workflow.Result<unknown, unknown>;
+
+type RunExit = Exit.Exit<RunResult>;
+
+type Instance = WorkflowEngine.WorkflowInstance["Service"];
+
+interface FinishedJob {
+  readonly executionId: string;
+  readonly job: string;
+  readonly result: string;
+}
+
+/** A job's run that ended with no result stored while its store was still attached, and why. */
+interface StalledJob {
+  readonly cause: Cause.Cause<unknown>;
+  readonly executionId: string;
+  readonly job: string;
+}
+
+interface JobStore {
+  readonly id: string;
+  readonly authority: ExecutionGuard;
+  readonly exclusive: <Success, Failure, Needs>(
+    effect: Effect.Effect<Success, Failure, Needs>,
+  ) => Effect.Effect<Success, Failure, Needs>;
+  readonly finish: (finished: FinishedJob) => Effect.Effect<void>;
+  /** Records a stalled run and schedules its next round; the engine has already forgotten the run. */
+  readonly stalled: (stalled: StalledJob) => Effect.Effect<void>;
+  readonly withSql: <Success, Failure, Needs>(
+    use: (sql: SqlClient.SqlClient) => Effect.Effect<Success, Failure, Needs>,
+  ) => Effect.Effect<Success, Failure, Needs>;
+}
+
+/** A job whose run is suspended now, and the journal entries it waits on (`deferred/<name>`). */
+interface Parked {
+  readonly id: string;
+  readonly waits: readonly string[];
+}
+
+interface Registered {
+  readonly execute: Execute;
+  readonly scope: Scope.Scope;
+  readonly workflow: Workflow.Any;
+}
+
+interface Execution {
+  /** When each activity of this run executing now started: while any is, its actor shouldn't close. */
+  readonly activities: number[];
+  readonly clocks: Set<string>;
+  readonly done: Deferred.Deferred<void>;
+  readonly executionId: string;
+  fiber: Option.Option<Fiber.Fiber<RunResult>>;
+  instance: Instance;
+  interrupted: boolean;
+  interruptedUnsafe: boolean;
+  readonly payload: Payload;
+  rerun: Option.Option<WakeMode>;
+  readonly scope: Scope.Closeable;
+  readonly store: JobStore;
+  /** Deferreds the current run read and found no result for, so a signal stored later has to wake it. */
+  readonly waiting: Set<string>;
+  readonly workflow: Workflow.Any;
+}
+
+interface Runs {
+  readonly clocks: FiberMap.FiberMap<string>;
+  readonly deferreds: WorkflowEngine.DeferredState;
+  readonly detached: WeakSet<JobStore>;
+  readonly executions: MutableHashMap.MutableHashMap<string, Execution>;
+  readonly owned: Map<JobStore, Set<Execution>>;
+  readonly record: (execution: Execution, exit: Exit.Exit<unknown, unknown>) => Effect.Effect<void>;
+  readonly registered: MutableHashMap.MutableHashMap<string, Registered>;
+  readonly scope: Scope.Scope;
+  readonly workflowEngine: Deferred.Deferred<WorkflowEngine.WorkflowEngine["Service"]>;
+}
+
+interface Running {
+  readonly execution: Execution;
+  readonly runs: Runs;
+}
+
+interface Waking {
+  readonly executionId: string;
+  readonly mode: WakeMode;
+  readonly runs: Runs;
+}
+
+interface Starting {
+  readonly executionId: string;
+  readonly payload: Payload;
+  readonly runs: Runs;
+  readonly store: JobStore;
+  readonly workflow: Workflow.Any;
+}
+
+const noneLeft = 0;
+
+const nameOf = ({ _tag: name }: Workflow.Any): string => name;
+
+const isSuspended = (result: RunResult): boolean => Predicate.isTagged(result, "Suspended");
+
+const isUnwakeable = (result: RunResult, mode: WakeMode): boolean =>
+  Predicate.isTagged(result, "Complete") || (mode === "automatic" && result.failed === true);
+
+const resultOf = (exit: RunExit): Option.Option<RunResult> => Exit.getSuccess(exit);
+
+const lastExit = (execution: Execution): Option.Option<RunExit> =>
+  Option.flatMap(execution.fiber, (fiber) => Option.fromNullishOr(fiber.pollUnsafe()));
+
+const runningFiber = (execution: Execution): Option.Option<Fiber.Fiber<RunResult>> =>
+  Option.filter(execution.fiber, (fiber) => Predicate.isUndefined(fiber.pollUnsafe()));
+
+const suspendsOnFailure = (instance: Instance, exit: Exit.Exit<unknown, unknown>): boolean =>
+  Context.get(instance.workflow.annotations, Workflow.SuspendOnFailure) && Exit.isFailure(exit);
+
+const interruptsRun = (execution: Execution, instance: Instance, exit: Exit.Exit<unknown, unknown>): boolean =>
+  execution.interrupted &&
+  (execution.interruptedUnsafe || instance.suspended || instance.abandoned || suspendsOnFailure(instance, exit));
+
+const interruptOnExit =
+  (execution: Execution, instance: Instance) =>
+  (exit: Exit.Exit<unknown, unknown>): Effect.Effect<void> => {
+    if (!interruptsRun(execution, instance, exit)) {
+      return Effect.void;
+    }
+
+    instance.interrupted = true;
+    instance.suspended = false;
+
+    return Effect.withFiber((fiber) => Effect.interruptible(Fiber.interrupt(fiber)));
+  };
+
+const recordResult =
+  ({ execution, runs }: Running) =>
+  (result: RunResult): Effect.Effect<void> => {
+    if (Predicate.isTagged(result, "Suspended")) {
+      return Effect.void;
+    }
+
+    return runs.record(execution, result.exit);
+  };
+
+const track = (runs: Runs, execution: Execution): void => {
+  const mine = runs.owned.get(execution.store) ?? new Set<Execution>();
+
+  mine.add(execution);
+  runs.owned.set(execution.store, mine);
+  MutableHashMap.set(runs.executions, execution.executionId, execution);
+};
+
+const forget = ({ execution, runs }: Running): void => {
+  const mine = runs.owned.get(execution.store);
+
+  mine?.delete(execution);
+
+  if (mine?.size === noneLeft) {
+    runs.owned.delete(execution.store);
+  }
+
+  if (Option.contains(MutableHashMap.get(runs.executions, execution.executionId), execution)) {
+    MutableHashMap.remove(runs.executions, execution.executionId);
+  }
+};
+
+/**
+ * A run that died with no result while its store is still attached, such as a defect recording its result, is
+ * stalled: the engine forgets it so it can start again, and the store records it and schedules its next round. A job's
+ * own defects never get here; they come back to the actor as its failed result.
+ */
+const stalledRun = ({ execution, runs }: Running, exit: RunExit): Effect.Effect<void> =>
+  Effect.suspend(() => {
+    // An interrupt ends a run on a close, a shutdown or a refused batch, and each of those reopens the actor.
+    // The next session resumes such a run, so only a defect stalls it.
+    if (Exit.isSuccess(exit) || Cause.hasInterruptsOnly(exit.cause) || runs.detached.has(execution.store)) {
+      return Effect.void;
+    }
+
+    forget({ execution, runs });
+
+    return execution.store.stalled({
+      cause: exit.cause,
+      executionId: execution.executionId,
+      job: nameOf(execution.workflow),
+    });
+  });
+
+const finished =
+  ({ execution, runs }: Running) =>
+  (exit: RunExit): Effect.Effect<void> => {
+    const result = resultOf(exit);
+
+    if (Option.exists(result, isSuspended)) {
+      return Effect.void;
+    }
+
+    const closed = Scope.close(execution.scope, Exit.void);
+
+    if (Option.exists(result, (found) => Predicate.isTagged(found, "Complete"))) {
+      const done = Effect.andThen(closed, Deferred.done(execution.done, Exit.void));
+
+      return Effect.andThen(
+        done,
+        Effect.sync(() => {
+          forget({ execution, runs });
+        }),
+      );
+    }
+
+    return Effect.andThen(closed, stalledRun({ execution, runs }, exit));
+  };
+
+const runOf = (running: Running, { execute }: Registered, instance: Instance): Effect.Effect<RunResult> => {
+  const { execution, runs } = running;
+
+  return execute(execution.payload, execution.executionId).pipe(
+    execution.store.authority,
+    Effect.onExit(interruptOnExit(execution, instance)),
+    Workflow.intoResult,
+    Effect.provideServiceEffect(WorkflowEngine.WorkflowEngine, Deferred.await(runs.workflowEngine)),
+    (effect) => runs.deferreds.trackRun(instance, effect),
+    Effect.tap(recordResult(running)),
+    Effect.onExit(finished(running)),
+  );
+};
+
+const launch = (running: Running, entry: Registered): Effect.Effect<void> => {
+  const { execution } = running;
+  const scope = Scope.forkUnsafe(execution.scope);
+  const instance = WorkflowEngine.WorkflowInstance.initial(execution.workflow, execution.executionId, { scope });
+
+  execution.instance = instance;
+  execution.waiting.clear();
+
+  return Effect.map(Effect.forkIn(runOf(running, entry, instance), entry.scope), (fiber) => {
+    execution.fiber = Option.some(fiber);
+  });
+};
+
+const startRun = (running: Running): Effect.Effect<void> =>
+  running.execution.store.authority(
+    Effect.suspend(() =>
+      Option.match(MutableHashMap.get(running.runs.registered, nameOf(running.execution.workflow)), {
+        onNone: () => Effect.void,
+        onSome: (entry) => launch(running, entry),
+      }),
+    ),
+  );
+
+const wakeIdle = (running: Running, mode: WakeMode): Effect.Effect<void> => {
+  const done = Option.flatMap(lastExit(running.execution), resultOf);
+
+  if (Option.exists(done, (result) => isUnwakeable(result, mode))) {
+    return Effect.void;
+  }
+
+  return startRun(running);
+};
+
+const requestRerun = (execution: Execution, mode: WakeMode): boolean => {
+  const requested = execution.rerun;
+
+  if (!Option.contains(requested, "explicit")) {
+    execution.rerun = Option.some(mode);
+  }
+
+  return Option.isNone(requested);
+};
+
+const takeRerun = (execution: Execution): WakeMode => {
+  const mode = Option.getOrElse(execution.rerun, (): WakeMode => "automatic");
+
+  execution.rerun = Option.none();
+
+  return mode;
+};
+
+const resume = (waking: Waking): Effect.Effect<void> =>
+  Effect.suspend(() =>
+    Option.match(MutableHashMap.get(waking.runs.executions, waking.executionId), {
+      onNone: () => Effect.void,
+      onSome: (execution) =>
+        execution.store.authority(
+          Option.match(runningFiber(execution), {
+            onNone: () => wakeIdle({ execution, runs: waking.runs }, waking.mode),
+            onSome: (fiber) => {
+              if (!requestRerun(execution, waking.mode)) {
+                return Effect.void;
+              }
+
+              const again = Effect.suspend(() => resume({ ...waking, mode: takeRerun(execution) }));
+              const rerun = Effect.andThen(Fiber.await(fiber), again);
+
+              return Effect.asVoid(Effect.forkIn(rerun, waking.runs.scope));
+            },
+          }),
+        ),
+    }),
+  );
+
+const startExecution = ({ executionId, payload, runs, store, workflow }: Starting): Effect.Effect<void> =>
+  Effect.suspend(() => {
+    if (runs.detached.has(store) || MutableHashMap.has(runs.executions, executionId)) {
+      return Effect.void;
+    }
+
+    const execution: Execution = {
+      activities: [],
+      clocks: new Set(),
+      done: Deferred.makeUnsafe(),
+      executionId,
+      fiber: Option.none(),
+      instance: WorkflowEngine.WorkflowInstance.initial(workflow, executionId),
+      interrupted: false,
+      interruptedUnsafe: false,
+      payload,
+      rerun: Option.none(),
+      scope: Scope.makeUnsafe(),
+      store,
+      waiting: new Set(),
+      workflow,
+    };
+
+    track(runs, execution);
+
+    return resume({ executionId, mode: "explicit", runs });
+  });
+
+const detachRun = ({ execution, runs }: Running): Effect.Effect<void> =>
+  Effect.gen(function* detachingRun() {
+    forget({ execution, runs });
+
+    // The run stops first, so it can't schedule a clock after its clocks are removed.
+    yield* Option.match(execution.fiber, { onNone: () => Effect.void, onSome: Fiber.interrupt });
+    yield* Effect.forEach(execution.clocks, (clock) => FiberMap.remove(runs.clocks, clock), { discard: true });
+    yield* Scope.close(execution.scope, Exit.void);
+  });
+
+const detachStore = ({ runs, store }: { readonly runs: Runs; readonly store: JobStore }): Effect.Effect<void> =>
+  Effect.suspend(() => {
+    runs.detached.add(store);
+
+    const mine = Arr.fromIterable(runs.owned.get(store) ?? []);
+
+    return Effect.forEach(mine, (execution) => detachRun({ execution, runs }), { discard: true });
+  });
+
+interface Clocking {
+  readonly executionId: string;
+  readonly name: string;
+  readonly runs: Runs;
+}
+
+const trackedClock = ({ executionId, name, runs }: Clocking): Effect.Effect<string> =>
+  Effect.sync(() => {
+    const key = `${executionId}/${name}`;
+
+    Option.map(MutableHashMap.get(runs.executions, executionId), (execution) => execution.clocks.add(key));
+
+    return key;
+  });
+
+/**
+ * The journal entries a run waits on, when it is suspended now with no rerun asked for: a signal or a clock wakes it,
+ * so nothing is due for it until one of those entries is stored.
+ */
+const parkedOn = (execution: Execution): Option.Option<Parked["waits"]> =>
+  lastExit(execution).pipe(
+    Option.flatMap(resultOf),
+    Option.filter((result) => isSuspended(result) && Option.isNone(execution.rerun)),
+    Option.map(() => Arr.map(Arr.fromIterable(execution.waiting), (name) => `deferred/${name}`)),
+  );
+
+const awaitRun = (execution: Execution): Effect.Effect<RunResult> =>
+  Effect.gen(function* awaiting() {
+    const { fiber, instance } = execution;
+
+    if (Option.isNone(fiber)) {
+      return yield* Effect.die(`job ${execution.executionId} has no handler registered yet`);
+    }
+
+    const exit = yield* Fiber.await(fiber.value);
+
+    if (Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause) && instance.suspended) {
+      return Workflow.Suspended.make({});
+    }
+
+    return yield* exit;
+  });
+
+const settled = (execution: Execution): Effect.Effect<boolean> =>
+  Effect.suspend(() => {
+    const busy = Option.filter(
+      execution.fiber,
+      () => Option.isNone(lastExit(execution)) || Option.isSome(execution.rerun),
+    );
+
+    return Option.match(busy, {
+      onNone: () => Effect.succeed(false),
+      onSome: (fiber) => {
+        const again = Effect.andThen(Effect.yieldNow, settled(execution));
+
+        return Effect.as(Effect.andThen(Fiber.await(fiber), again), true);
+      },
+    });
+  });
+
+export { awaitRun, detachStore, lastExit, nameOf, parkedOn, resume, settled, startExecution, trackedClock };
+
+export type {
+  Execute,
+  Execution,
+  FinishedJob,
+  JobStore,
+  Parked,
+  Payload,
+  Registered,
+  RunResult,
+  Runs,
+  StalledJob,
+  WakeMode,
+};