@akterjs/effect 0.0.0-stage → 0.1.1

package/src/host/command/command-journal.sql.ts

@@ -0,0 +1,232 @@
+import { Array as Arr, Effect, Option, Stream } from "effect";
+import type { SqlClient, SqlError } from "effect/sql";
+
+import type { JournalEntry } from "#src/contract/history/actor-journal.service";
+import type { HostedOutcome } from "#src/contract/actor/actor.contract";
+
+import { saveMessages } from "#src/host/outbox/actor-outbox.sql";
+import { saveTimers } from "#src/host/timer/actor-timers.sql";
+import { saveJobs, saveSignals } from "#src/host/job/job-journal.sql";
+import { CommandRejectedError } from "./command-rejected.errors.js";
+import { stalledCursor } from "./replay/replay-cursor.pure.js";
+import { retainFileInputs } from "./files/command-file-inputs.sql.js";
+
+interface JournalRow {
+  readonly at: number;
+  readonly command: string;
+  readonly commandId: string;
+  readonly exit: string;
+  readonly files: string;
+  readonly ids: string;
+  readonly payload: string;
+  readonly principal: string;
+  readonly times: string;
+}
+
+interface Lookup {
+  readonly commandId: string;
+  readonly sql: SqlClient.SqlClient;
+}
+
+interface Recording {
+  readonly row: JournalRow;
+  /** The seq to write the row at, inside an engine batch whose journal assigns it; otherwise SQLite picks it. */
+  readonly seq: Option.Option<number>;
+  readonly sql: SqlClient.SqlClient;
+}
+
+interface Saving {
+  readonly sql: SqlClient.SqlClient;
+  readonly value: string;
+}
+
+interface Settling {
+  readonly at: number;
+  readonly outcome: HostedOutcome;
+  readonly principal: string;
+  readonly sql: SqlClient.SqlClient;
+}
+
+interface Window {
+  readonly after: number;
+  readonly limit: number;
+  readonly since: number;
+  readonly sql: SqlClient.SqlClient;
+}
+
+interface Pruning {
+  readonly before: number;
+  readonly sql: SqlClient.SqlClient;
+}
+
+interface Paging {
+  readonly after: number;
+  readonly limit: number;
+  readonly sql: SqlClient.SqlClient;
+}
+
+interface ReplayRow extends JournalRow {
+  readonly seq: number;
+}
+
+interface WindowRow {
+  readonly commands: number;
+  readonly firstAt: number | null;
+  readonly firstSeq: number | null;
+  readonly lastAt: number | null;
+  readonly lastSeq: number | null;
+}
+
+type Read<Value> = Effect.Effect<Option.Option<Value>, SqlError.SqlError>;
+
+interface RecordedCommand {
+  readonly command: string;
+  readonly exit: string;
+  readonly principal: string;
+}
+
+type Write = Effect.Effect<void, SqlError.SqlError>;
+
+const recordedExit = ({ commandId, sql }: Lookup): Read<string> =>
+  Effect.map(sql<{ readonly exit: string }>`SELECT exit FROM akter_commands WHERE command_id = ${commandId}`, (rows) =>
+    Option.map(Arr.head(rows), (row) => row.exit),
+  );
+
+const recordedCommand = ({ commandId, sql }: Lookup): Read<RecordedCommand> =>
+  Effect.map(
+    sql<RecordedCommand>`SELECT command, exit, principal FROM akter_commands WHERE command_id = ${commandId}`,
+    Arr.head,
+  );
+
+const insertCommand = ({ row, seq, sql }: Recording): Write =>
+  Option.match(seq, {
+    onNone: () =>
+      Effect.asVoid(
+        sql`INSERT INTO akter_commands (command_id, command, payload, principal, exit, ids, times, files, at) VALUES (${row.commandId}, ${row.command}, ${row.payload}, ${row.principal}, ${row.exit}, ${row.ids}, ${row.times}, ${row.files}, ${row.at})`,
+      ),
+    onSome: (at) =>
+      Effect.asVoid(
+        sql`INSERT INTO akter_commands (seq, command_id, command, payload, principal, exit, ids, times, files, at) VALUES (${at}, ${row.commandId}, ${row.command}, ${row.payload}, ${row.principal}, ${row.exit}, ${row.ids}, ${row.times}, ${row.files}, ${row.at})`,
+      ),
+  });
+
+const recordCommand = (recording: Recording): Write =>
+  Effect.andThen(
+    insertCommand(recording),
+    retainFileInputs({ commandId: recording.row.commandId, recorded: recording.row.files, sql: recording.sql }),
+  );
+
+const currentState = (sql: SqlClient.SqlClient): Read<string> =>
+  Effect.map(sql<{ readonly value: string }>`SELECT value FROM akter_state WHERE id = 1`, (rows) =>
+    Option.map(Arr.head(rows), (row) => row.value),
+  );
+
+const saveState = ({ sql, value }: Saving): Write =>
+  Effect.asVoid(
+    sql`INSERT INTO akter_state (id, value) VALUES (1, ${value}) ON CONFLICT (id) DO UPDATE SET value = excluded.value`,
+  );
+
+const settle = ({
+  at,
+  outcome,
+  principal,
+  sql,
+}: Settling): Effect.Effect<string, CommandRejectedError | SqlError.SqlError> =>
+  Option.match(outcome.state, {
+    onNone: () => Effect.fail(CommandRejectedError.make({ exit: outcome.exit })),
+    onSome: (value) =>
+      saveState({ sql, value }).pipe(
+        Effect.andThen(saveTimers({ at, principal, sql, timers: outcome.timers })),
+        Effect.andThen(saveMessages({ messages: outcome.messages, sql })),
+        Effect.andThen(saveJobs({ jobs: outcome.jobs, sql })),
+        // After the jobs, so a command can signal a job it starts.
+        Effect.andThen(saveSignals({ signals: outcome.signals, sql })),
+        Effect.as(outcome.exit),
+      ),
+  });
+
+const journalSince = ({
+  after,
+  limit,
+  since,
+  sql,
+}: Window): Effect.Effect<readonly JournalEntry[], SqlError.SqlError> =>
+  sql<JournalEntry>`SELECT seq, command_id AS commandId, command, payload, principal, exit, at FROM akter_commands WHERE at >= ${since} AND seq > ${after} ORDER BY seq LIMIT ${limit}`;
+
+const pruneJournal = ({ before, sql }: Pruning): Write =>
+  sql.withTransaction(
+    Effect.gen(function* pruneHistory() {
+      yield* sql`INSERT INTO akter_meta (key, value) SELECT 'pruned_before', ${before} WHERE EXISTS (SELECT 1 FROM akter_commands WHERE at < ${before}) ON CONFLICT (key) DO UPDATE SET value = MAX(value, excluded.value)`;
+      yield* sql`INSERT INTO akter_blob_releases (claim, actor, blob) SELECT claim, actor, blob FROM akter_blob_history AS history WHERE command_id IN (SELECT command_id FROM akter_commands WHERE at < ${before}) AND NOT EXISTS (SELECT 1 FROM akter_blob_links WHERE claim = history.claim) AND NOT EXISTS (SELECT 1 FROM akter_blob_receipts WHERE claim = history.claim AND state IN ('uploading', 'prepared')) ON CONFLICT (claim) DO NOTHING`;
+      yield* sql`DELETE FROM akter_blob_history WHERE command_id IN (SELECT command_id FROM akter_commands WHERE at < ${before})`;
+      yield* sql`DELETE FROM akter_commands WHERE at < ${before}`;
+    }),
+  );
+
+const prunedBefore = (sql: SqlClient.SqlClient): Read<number> =>
+  Effect.map(sql<{ readonly value: number }>`SELECT value FROM akter_meta WHERE key = 'pruned_before'`, (rows) =>
+    Option.map(Arr.head(rows), (row) => row.value),
+  );
+
+const journalWindow = (sql: SqlClient.SqlClient): Effect.Effect<readonly WindowRow[], SqlError.SqlError> =>
+  sql<WindowRow>`SELECT count(*) AS commands, min(seq) AS firstSeq, max(seq) AS lastSeq, (SELECT at FROM akter_commands ORDER BY seq LIMIT 1) AS firstAt, (SELECT at FROM akter_commands ORDER BY seq DESC LIMIT 1) AS lastAt FROM akter_commands`;
+
+const replayPage = ({ after, limit, sql }: Paging): Effect.Effect<readonly ReplayRow[], SqlError.SqlError> =>
+  sql<ReplayRow>`SELECT seq, command_id AS commandId, command, payload, principal, exit, ids, times, files, at FROM akter_commands WHERE seq > ${after} ORDER BY seq LIMIT ${limit}`;
+
+const replayPageSize = 500;
+
+const beforeFirst = 0;
+
+const nextAfter = (rows: readonly ReplayRow[]): Option.Option<number> =>
+  Option.map(
+    Option.filter(Arr.last(rows), () => rows.length === replayPageSize),
+    (row) => row.seq,
+  );
+
+const replayRows = (sql: SqlClient.SqlClient): Stream.Stream<ReplayRow, SqlError.SqlError> =>
+  Stream.paginate(beforeFirst, (after: number) =>
+    Effect.flatMap(replayPage({ after, limit: replayPageSize, sql }), (rows) =>
+      Option.match(nextAfter(rows), {
+        onNone: () => Effect.succeed([rows, Option.none<number>()] as const),
+        onSome: (cursor) =>
+          Option.match(stalledCursor(cursor, after), {
+            onNone: () => Effect.succeed([rows, Option.some(cursor)] as const),
+            onSome: Effect.die,
+          }),
+      }),
+    ),
+  );
+
+/**
+ * Starts a copied database's history at the copy, such as a fork's: the commands before it belong to the actor it was
+ * copied from, and their file inputs name that actor's files, so the copy reads as pruned before its first command.
+ */
+const startJournalAtCopy = (sql: SqlClient.SqlClient): Write =>
+  Effect.andThen(
+    sql`INSERT INTO akter_meta (key, value) SELECT 'pruned_before', MAX(at) + 1 FROM akter_commands HAVING COUNT(*) > 0 ON CONFLICT (key) DO UPDATE SET value = MAX(value, excluded.value)`,
+    sql`DELETE FROM akter_commands`,
+  );
+
+export {
+  currentState,
+  journalSince,
+  journalWindow,
+  prunedBefore,
+  pruneJournal,
+  recordCommand,
+  recordedCommand,
+  recordedExit,
+  replayPage,
+  replayRows,
+  settle,
+  startJournalAtCopy,
+};
+
+export type { JournalRow, RecordedCommand, ReplayRow, WindowRow };
+
+export { CommandBatchJournal } from "./batch/command-batch.sql.js";
+
+export { fileDigests } from "#src/host/file/actor-files.sql";
+
+export type { FileDigest } from "#src/host/file/actor-files.sql";

package/src/host/command/command-rejected.errors.ts

@@ -0,0 +1,8 @@
+import { Schema } from "effect";
+
+export class CommandRejectedError extends Schema.TaggedError<CommandRejectedError, { readonly brand: unique symbol }>()(
+  "CommandRejectedError",
+  {
+    exit: Schema.String,
+  },
+) {}

package/src/host/command/command-replay.service.ts

@@ -0,0 +1,223 @@
+import { Array as Arr, Context, Effect, FileSystem, Layer, Option, Ref, Stream } from "effect";
+import type { Equivalence } from "effect";
+import { SecureRandom } from "effect/crypto";
+
+import { ActorUnavailableError } from "#src/contract/client/actor-unavailable.errors";
+import type { HostedActor, HostedError } from "#src/contract/actor/actor.contract";
+import { HistoryPrunedError } from "./history-pruned.errors.js";
+
+import { ActorDatabase } from "#src/host/actor/actor-database.service";
+import { CommandContext } from "./command-context.adapter.js";
+import type { RecordedValues } from "./command-context.adapter.js";
+import { fileDigests, journalWindow, prunedBefore, recordedExit, replayRows } from "./command-journal.sql.js";
+import type { FileDigest, ReplayRow, WindowRow } from "./command-journal.sql.js";
+import { CommandRunner } from "./command-runner.service.js";
+import type { Running } from "./command-runner.service.js";
+import { SqliteDriver } from "#src/host/sqlite/sqlite-driver.service";
+
+interface Replaying {
+  readonly actor: HostedActor;
+  readonly database: string;
+  readonly key: string;
+}
+
+interface JournalPoint {
+  readonly at: number;
+  readonly seq: number;
+}
+
+interface ReplayWindow {
+  readonly commands: number;
+  readonly first: Option.Option<JournalPoint>;
+  readonly last: Option.Option<JournalPoint>;
+}
+
+type ReplayedOutcome =
+  | { readonly exit: string; readonly kind: "journaled" }
+  | { readonly error: HostedError; readonly kind: "refused" };
+
+interface ReplayDifference {
+  readonly actual: ReplayedOutcome;
+  readonly at: number;
+  readonly command: string;
+  readonly commandId: string;
+  readonly expected: string;
+  readonly seq: number;
+}
+
+interface FilesDifference {
+  readonly actual: readonly FileDigest[];
+  readonly expected: readonly FileDigest[];
+}
+
+interface ReplayReport {
+  readonly actor: string;
+  readonly difference: Option.Option<ReplayDifference>;
+  readonly files: Option.Option<FilesDifference>;
+  readonly key: string;
+  readonly window: ReplayWindow;
+}
+
+type ReplayFailure = ActorUnavailableError | HistoryPrunedError;
+
+interface Replayer {
+  readonly key: string;
+  readonly opened: Running["opened"];
+  readonly runner: CommandRunner["Service"];
+  readonly script: Ref.Ref<RecordedValues>;
+}
+
+type Sql = Running["opened"]["sql"];
+
+type Failure = Stream.Error<ReturnType<typeof replayRows>>;
+
+const noneRecorded: RecordedValues = { files: "[]", ids: "[]", times: "[]" };
+
+const emptyWindow: ReplayWindow = { commands: 0, first: Option.none(), last: Option.none() };
+
+const pointOf = (seq: number | null, at: number | null): Option.Option<JournalPoint> =>
+  Option.all({ at: Option.fromNullishOr(at), seq: Option.fromNullishOr(seq) });
+
+const windowOf = (rows: readonly WindowRow[]): ReplayWindow =>
+  Option.match(Arr.head(rows), {
+    onNone: () => emptyWindow,
+    onSome: (row) => ({
+      commands: row.commands,
+      first: pointOf(row.firstSeq, row.firstAt),
+      last: pointOf(row.lastSeq, row.lastAt),
+    }),
+  });
+
+const journaled = (exit: string): ReplayedOutcome => ({ exit, kind: "journaled" });
+
+const refused = (error: HostedError): ReplayedOutcome => ({ error, kind: "refused" });
+
+const runRow = ({ key, opened, runner }: Replayer, row: ReplayRow): ReturnType<CommandRunner["Service"]["replay"]> =>
+  runner.replay({
+    opened,
+    request: { commandId: row.commandId, key, member: row.command, payload: row.payload, principal: row.principal },
+  });
+
+const replayRow = (replayer: Replayer): ((row: ReplayRow) => Effect.Effect<Option.Option<ReplayDifference>, Failure>) =>
+  Effect.fnUntraced(function* replayOne(row: ReplayRow) {
+    yield* Ref.set(replayer.script, { files: row.files, ids: row.ids, times: row.times });
+
+    const actual = yield* runRow(replayer, row).pipe(
+      Effect.map(journaled),
+      Effect.catchTag("SqlError", Effect.fail, (error) =>
+        Effect.map(
+          recordedExit({ commandId: row.commandId, sql: replayer.opened.sql }),
+          Option.match({ onNone: () => refused(error), onSome: journaled }),
+        ),
+      ),
+    );
+
+    if (actual.kind === "journaled" && actual.exit === row.exit) {
+      return Option.none<ReplayDifference>();
+    }
+
+    return Option.some({
+      actual,
+      at: row.at,
+      command: row.command,
+      commandId: row.commandId,
+      expected: row.exit,
+      seq: row.seq,
+    });
+  });
+
+const sameDigest: Equivalence.Equivalence<FileDigest> = (left, right) =>
+  left.path === right.path && left.sha256 === right.sha256 && left.bytes === right.bytes;
+
+const sameDigests = Arr.makeEquivalence(sameDigest);
+
+const filesDifference = (source: Sql, replayed: Sql): Effect.Effect<Option.Option<FilesDifference>, Failure> =>
+  Effect.map(Effect.all({ actual: fileDigests(replayed), expected: fileDigests(source) }), (digests) =>
+    Option.liftPredicate(digests, ({ actual, expected }) => !sameDigests(actual, expected)),
+  );
+
+const unpruned = (actor: HostedActor, key: string, source: Sql): Effect.Effect<void, HistoryPrunedError | Failure> =>
+  Effect.flatMap(
+    prunedBefore(source),
+    Option.match({
+      onNone: () => Effect.void,
+      onSome: (coveredSince) => Effect.fail(HistoryPrunedError.make({ actor: actor.name, coveredSince, key })),
+    }),
+  );
+
+const firstDifference = (replayer: Replayer, source: Sql): Effect.Effect<Option.Option<ReplayDifference>, Failure> =>
+  replayRows(source).pipe(
+    Stream.mapEffect(replayRow(replayer)),
+    Stream.filter(Option.isSome),
+    Stream.runHead,
+    Effect.map(Option.flatten),
+  );
+
+export class CommandReplay extends Context.Service<
+  CommandReplay,
+  {
+    readonly replay: (replaying: Replaying) => Effect.Effect<ReplayReport, ReplayFailure>;
+  }
+>()("@akterjs/effect/host/command/command-replay.service/CommandReplay", {
+  make: Effect.gen(function* makeCommandReplay() {
+    const driver = yield* SqliteDriver;
+    const database = yield* ActorDatabase;
+    const fileSystem = yield* FileSystem.FileSystem;
+    const random = yield* SecureRandom.SecureRandom;
+
+    const replayFrom = Effect.fnUntraced(function* replayFrom({ actor, key }: Replaying, source: Sql) {
+      yield* unpruned(actor, key, source);
+
+      const window = yield* journalWindow(source);
+      const script = yield* Ref.make(noneRecorded);
+      const context = CommandContext.replaying(random, script);
+      const runner = yield* Effect.provideService(CommandRunner.make, CommandContext, context);
+      const opened = yield* database.replay({ actor, source });
+      const difference = yield* firstDifference({ key, opened, runner, script }, source);
+      const files = yield* filesDifference(source, opened.sql);
+
+      return { actor: actor.name, difference, files, key, window: windowOf(window) } satisfies ReplayReport;
+    });
+
+    const replayStored = Effect.fnUntraced(function* replayStored(replaying: Replaying) {
+      if (!(yield* fileSystem.exists(replaying.database))) {
+        return {
+          actor: replaying.actor.name,
+          difference: Option.none(),
+          files: Option.none(),
+          key: replaying.key,
+          window: emptyWindow,
+        };
+      }
+
+      const source = yield* driver.openQueryOnly(replaying.database);
+
+      return yield* source.withTransaction(replayFrom(replaying, source));
+    }, Effect.scoped);
+
+    const replay = Effect.fn("CommandReplay.replay")(function* replay(replaying: Replaying) {
+      return yield* replayStored(replaying).pipe(
+        Effect.catchTag("HistoryPrunedError", Effect.fail, (failure) =>
+          Effect.fail(ActorUnavailableError.make({ actor: replaying.actor.name, message: failure.message })),
+        ),
+      );
+    });
+
+    return { replay } as const;
+  }),
+}) {
+  public static readonly layer = Layer.effect(this, this.make).pipe(
+    Layer.provide([ActorDatabase.layer, SecureRandom.layerWebCrypto]),
+  );
+}
+
+export type {
+  FilesDifference,
+  JournalPoint,
+  ReplayDifference,
+  ReplayedOutcome,
+  ReplayFailure,
+  Replaying,
+  ReplayReport,
+  ReplayWindow,
+};

package/src/host/command/command-reply.pure.ts

@@ -0,0 +1,21 @@
+import { Effect, Option, Schema } from "effect";
+
+import { ActorAlreadyExistsError } from "#src/contract/actor/actor-already-exists.errors";
+import { HandlerFailedError } from "#src/contract/hosted/handler-failed.errors";
+
+const JournaledFailure = Schema.Union([ActorAlreadyExistsError, HandlerFailedError]);
+
+const JournaledFailureJson = Schema.fromJsonString(JournaledFailure);
+
+type JournaledFailure = typeof JournaledFailure.Type;
+
+const journalFailure = (failed: JournaledFailure): Effect.Effect<string> =>
+  Effect.orDie(Schema.encodeEffect(JournaledFailureJson)(failed));
+
+const replyOf = (journaled: string): Effect.Effect<string, JournaledFailure> =>
+  Option.match(Schema.decodeOption(JournaledFailureJson)(journaled), {
+    onNone: () => Effect.succeed(journaled),
+    onSome: (failed) => Effect.fail(failed),
+  });
+
+export { journalFailure, replyOf };