@akterjs/effect 0.0.0-stage → 0.1.0

package/src/host/file/blob-home.sql.ts

@@ -0,0 +1,55 @@
+import { Effect, FileSystem } from "effect";
+import type { Path, Scope } from "effect";
+import type { SecureRandom } from "effect/crypto";
+import type { SqlClient } from "effect/sql";
+
+import type { BlobSealing } from "#src/host/blob/blob-sealing.service";
+import type { BlobHashing } from "#src/host/hash/blob-hashing.service";
+import { blobCatalog, prepareCatalog } from "#src/host/blob/reference/blob-catalog.sql";
+import { blobTransfers } from "#src/host/blob/reference/blob-coordinator.pure";
+import type { BlobTransfers, TransferEffect, TransferHome } from "#src/host/blob/reference/blob-transfer.pure";
+import { blobDirectory } from "#src/host/blob/stream/blob-directory.pure";
+
+interface LocalBlobHome extends TransferHome {
+  readonly transfers: BlobTransfers;
+}
+
+const canonicalHome = (directory: string): Effect.Effect<string, never, FileSystem.FileSystem> =>
+  FileSystem.FileSystem.use((fileSystem) =>
+    Effect.andThen(fileSystem.makeDirectory(directory, { recursive: true }), fileSystem.realPath(directory)).pipe(
+      Effect.orDie,
+    ),
+  );
+
+interface HomeOpening {
+  readonly catalog: SqlClient.SqlClient;
+  readonly hashing: BlobHashing["Service"];
+  readonly root: string;
+  readonly sealing: BlobSealing["Service"];
+}
+
+/** A blob home on `root` whose catalog lives on the given connection. Its transfers close with the scope. */
+const blobHomeOn = ({
+  catalog,
+  hashing,
+  root,
+  sealing,
+}: HomeOpening): TransferEffect<
+  LocalBlobHome,
+  FileSystem.FileSystem | Path.Path | Scope.Scope | SecureRandom.SecureRandom
+> =>
+  Effect.gen(function* openBlobHome() {
+    yield* prepareCatalog(catalog);
+
+    const directory = yield* blobDirectory({ sealing, store: root });
+    const home = { catalog: blobCatalog(catalog), directory, root };
+    const transfers = yield* blobTransfers({ hashing, home, sealing });
+
+    yield* Effect.addFinalizer(() => transfers.close);
+
+    return { ...home, transfers };
+  });
+
+export { blobHomeOn, canonicalHome };
+
+export type { LocalBlobHome };

package/src/host/file/local-blob-home.service.ts

@@ -0,0 +1,71 @@
+import { Context, Effect, Layer, Option, Path, SynchronizedRef } from "effect";
+import type { Scope } from "effect";
+import type { SecureRandom } from "effect/crypto";
+
+import { BlobSealing } from "#src/host/blob/blob-sealing.service";
+import type { TransferEffect } from "#src/host/blob/reference/blob-transfer.pure";
+import { BlobHashing } from "#src/host/hash/blob-hashing.service";
+import { SqliteDriver } from "#src/host/sqlite/sqlite-driver.service";
+import { acquireBlobHome } from "./blob-home-lock.pure.js";
+import { blobHomeOn, canonicalHome } from "./blob-home.sql.js";
+import type { LocalBlobHome } from "./blob-home.sql.js";
+
+type LocalHomeServices =
+  | Effect.Services<typeof BlobSealing.makePlain>
+  | Path.Path
+  | Scope.Scope
+  | SecureRandom.SecureRandom
+  | SqliteDriver;
+
+const makeLocalBlobHome = Effect.fnUntraced(function* makeLocalBlobHome(root: string, sealing: BlobSealing["Service"]) {
+  const driver = yield* SqliteDriver;
+  const path = yield* Path.Path;
+  const hashing = yield* BlobHashing;
+
+  yield* acquireBlobHome({ directory: root, driver });
+
+  const catalog = yield* driver.open(path.join(root, "catalog.sqlite"));
+
+  return yield* blobHomeOn({ catalog, hashing, root, sealing });
+});
+
+const localBlobHome = (directory: string): TransferEffect<LocalBlobHome, LocalHomeServices> =>
+  Effect.gen(function* openPlaintextHome() {
+    const root = yield* canonicalHome(directory);
+    const sealing = yield* BlobSealing.makePlain;
+
+    return yield* makeLocalBlobHome(root, sealing);
+  });
+
+export class LocalBlobHomes extends Context.Service<
+  LocalBlobHomes,
+  {
+    readonly get: TransferEffect<LocalBlobHome>;
+  }
+>()("@akterjs/effect/host/file/local-blob-home.service/LocalBlobHomes") {
+  public static readonly open = localBlobHome;
+
+  public static layer(directory: string): Layer.Layer<LocalBlobHomes, never, Exclude<LocalHomeServices, Scope.Scope>> {
+    return Layer.effect(
+      this,
+      Effect.gen(function* makeLocalBlobHomes() {
+        const services = yield* Effect.context<LocalHomeServices>();
+        const current = yield* SynchronizedRef.make(Option.none<LocalBlobHome>());
+        const open = Effect.provide(LocalBlobHomes.open(directory), services);
+
+        const get = SynchronizedRef.modifyEffect(current, (found) =>
+          Option.match(found, {
+            onNone: () => Effect.map(open, (home) => [home, Option.some(home)] as const),
+            onSome: (home) => Effect.succeed([home, found] as const),
+          }),
+        );
+
+        return { get };
+      }),
+    );
+  }
+}
+
+export type { LocalBlobHome } from "./blob-home.sql.js";
+
+export type { LocalHomeServices };

package/src/host/file/local-blob-owner.sql.ts

@@ -0,0 +1,89 @@
+import { Effect, Option } from "effect";
+import type { Scope } from "effect";
+import type { SqlClient } from "effect/sql";
+
+import { actorExclusive } from "#src/host/actor/actor-exclusive.pure";
+import { blobForks } from "#src/host/blob/reference/blob-forks.sql";
+import { blobReceipts, prepareReceipts } from "#src/host/blob/reference/blob-receipts.sql";
+import type { BlobOwner, TransferEffect } from "#src/host/blob/reference/blob-transfer.pure";
+import type { LocalBlobHome, LocalBlobHomes } from "./local-blob-home.service.js";
+import { localFileUpkeep } from "./local-file-upkeep.pure.js";
+
+interface LocalOwner {
+  readonly actor: string;
+  readonly sql: SqlClient.SqlClient;
+}
+
+interface LocalActorFiles {
+  readonly home: LocalBlobHome;
+  readonly kick: Effect.Effect<void>;
+  readonly owner: BlobOwner;
+  readonly sql: SqlClient.SqlClient;
+}
+
+interface OwnerOpening extends LocalOwner {
+  readonly exclusive: BlobOwner["exclusive"];
+  readonly snapshot: BlobOwner["snapshot"];
+}
+
+/**
+ * The blob owner of one actor whose writes go through `exclusive`, the actor's own serializer, so a file write never
+ * interleaves with a command's transaction on the same connection.
+ */
+const blobOwnerOn = ({ actor, exclusive, snapshot, sql }: OwnerOpening): TransferEffect<BlobOwner> =>
+  Effect.gen(function* openBlobOwner() {
+    const receipts = blobReceipts({ actor, sql });
+
+    yield* exclusive(Effect.andThen(prepareReceipts(sql), receipts.resume));
+
+    return { actor, exclusive, forks: blobForks({ actor, sql }), receipts, snapshot, transaction: sql.withTransaction };
+  });
+
+const localBlobOwner = ({ actor, sql }: LocalOwner): TransferEffect<BlobOwner, Scope.Scope> =>
+  Effect.gen(function* openLocalBlobOwner() {
+    const exclusive = yield* actorExclusive;
+    const snapshot = yield* actorExclusive;
+
+    yield* Effect.addFinalizer(() => Effect.andThen(snapshot.close, exclusive.close));
+
+    return yield* blobOwnerOn({ actor, exclusive: exclusive.run, snapshot: snapshot.run, sql });
+  });
+
+/** Recovers and settles an owner's transfers on its home, then keeps its references settled until the scope closes. */
+const actorFilesOn = ({
+  home,
+  owner,
+  sql,
+}: Omit<LocalActorFiles, "kick">): TransferEffect<LocalActorFiles, Scope.Scope> =>
+  Effect.gen(function* openActorFiles() {
+    yield* home.transfers.recover(owner);
+    yield* home.transfers.settle(owner);
+    const { kick } = yield* localFileUpkeep({ home, owner });
+
+    return { home, kick, owner, sql };
+  });
+
+const openLocalActorFiles = (
+  homes: LocalBlobHomes["Service"],
+  opening: LocalOwner,
+): TransferEffect<LocalActorFiles, Scope.Scope> =>
+  Effect.gen(function* openActorFiles() {
+    const home = yield* homes.get;
+    const owner = yield* localBlobOwner(opening);
+
+    return yield* actorFilesOn({ home, owner, sql: opening.sql });
+  });
+
+type LocalActorFileOpener = (opening: LocalOwner) => TransferEffect<Option.Option<LocalActorFiles>, Scope.Scope>;
+
+const localActorFiles =
+  (homes: Option.Option<LocalBlobHomes["Service"]>): LocalActorFileOpener =>
+  (opening: LocalOwner): TransferEffect<Option.Option<LocalActorFiles>, Scope.Scope> =>
+    Option.match(homes, {
+      onNone: () => Effect.succeedNone,
+      onSome: (available) => Effect.asSome(openLocalActorFiles(available, opening)),
+    });
+
+export { actorFilesOn, blobOwnerOn, localActorFiles, localBlobOwner };
+
+export type { LocalActorFileOpener, LocalActorFiles, LocalOwner };

package/src/host/file/local-file-upkeep.pure.ts

@@ -0,0 +1,77 @@
+import { Array as Arr, Clock, Effect, Queue } from "effect";
+import type { Scope } from "effect";
+import type { SqlError } from "effect/sql";
+
+import type { BlobClaim } from "#src/host/blob/reference/blob-catalog.sql";
+import type { BlobOwner, TransferEffect } from "#src/host/blob/reference/blob-transfer.pure";
+import type { LocalBlobHome } from "./local-blob-home.service.js";
+
+interface FileUpkeep {
+  readonly home: LocalBlobHome;
+  readonly owner: BlobOwner;
+}
+
+const oneNotification = 1;
+
+const retryEvery = "1 minute";
+
+/** Whether the owner is erased, and which of its releases are ready at `now` (a hold keeps some waiting). */
+const pendingAt = (
+  owner: BlobOwner,
+  now: number,
+): Effect.Effect<{ readonly erased: boolean; readonly releases: readonly BlobClaim[] }, SqlError.SqlError> => {
+  const read = Effect.all({ erased: owner.receipts.erased, releases: owner.receipts.releases(now) });
+
+  return owner.exclusive(owner.transaction(read));
+};
+
+const localFileUpkeep = ({
+  home,
+  owner,
+}: FileUpkeep): Effect.Effect<{ readonly kick: Effect.Effect<void> }, never, Scope.Scope> =>
+  Effect.gen(function* keepLocalFiles() {
+    const events = yield* Queue.sliding<boolean>(oneNotification);
+    const kick = Effect.asVoid(Queue.offer(events, true));
+
+    const collect: TransferEffect<void> = Effect.gen(function* collectPages() {
+      for (;;) {
+        const removed = yield* home.transfers.collect;
+
+        if (!Arr.isReadonlyArrayNonEmpty(removed)) {
+          return;
+        }
+
+        yield* Effect.yieldNow;
+      }
+    });
+
+    const pass = Effect.gen(function* settleFileReferences() {
+      yield* home.transfers.settle(owner);
+      yield* collect;
+
+      const now = yield* Clock.currentTimeMillis;
+      const pending = yield* pendingAt(owner, now);
+
+      if (!pending.erased && Arr.isReadonlyArrayNonEmpty(pending.releases)) {
+        yield* kick;
+      }
+    }).pipe(
+      Effect.catchTags({
+        BlobTransferError: () => Effect.logWarning("local file cleanup will retry"),
+        SqlError: () => Effect.logWarning("local file cleanup will retry"),
+      }),
+    );
+
+    const waiting = Effect.timeoutOrElse(Queue.take(events), {
+      duration: retryEvery,
+      orElse: () => Effect.succeed(false),
+    });
+
+    const work = Effect.forever(Effect.andThen(waiting, pass));
+
+    yield* Effect.forkScoped(work);
+
+    return { kick };
+  });
+
+export { localFileUpkeep };

package/src/host/file/replay-file-storage.sql.ts

@@ -0,0 +1,43 @@
+import { Array as Arr, Effect, Option, Stream } from "effect";
+import type { SqlClient, SqlError } from "effect/sql";
+
+import { FilesUnavailableError } from "#src/contract/file/files-unavailable.errors";
+import type { OpenedActor } from "#src/host/actor/actor-database.service";
+import { actorFileStorage } from "./actor-file-storage.sql.js";
+import type { FileAccess } from "./actor-file-storage.sql.js";
+import type { LocalBlobHomes } from "./local-blob-home.service.js";
+
+type ReplayFileRestorer = (
+  source: SqlClient.SqlClient,
+  blank: OpenedActor,
+) => Effect.Effect<OpenedActor, SqlError.SqlError>;
+
+const replayFileStorage = (homes: Option.Option<LocalBlobHomes["Service"]>): ReplayFileRestorer => {
+  const open: FileAccess["open"] = Option.match(homes, {
+    onNone: () => () => Stream.fail(FilesUnavailableError.make({ message: "files aren't available on this host" })),
+    onSome: (available) => (opening) =>
+      Stream.unwrap(Effect.map(available.get, (home) => home.transfers.open(opening))),
+  });
+
+  return (source: SqlClient.SqlClient, blank: OpenedActor): Effect.Effect<OpenedActor, SqlError.SqlError> =>
+    Effect.gen(function* replayFiles() {
+      const owners = yield* source<{
+        readonly actor: string;
+        readonly state: string;
+      }>`SELECT actor, state FROM akter_blob_owners WHERE id = 1`;
+
+      const owner = Arr.head(owners);
+
+      if (Option.isNone(owner)) {
+        return blank;
+      }
+
+      const stored = owner.value;
+
+      yield* blank.sql`INSERT INTO akter_blob_owners (id, actor, state) VALUES (1, ${stored.actor}, ${stored.state})`;
+
+      return { ...blank, files: (sql) => actorFileStorage({ actor: stored.actor, open, sql }) };
+    });
+};
+
+export { replayFileStorage };

package/src/host/fork/actor-fork.sql.ts

@@ -0,0 +1,110 @@
+import { Array as Arr, Effect, Option } from "effect";
+import type { SqlClient, SqlError } from "effect/sql";
+
+import { clearCopiedFiles } from "#src/host/blob/reference/blob-receipts.sql";
+import { startJournalAtCopy } from "#src/host/command/command-journal.sql";
+import { cancelOpenJobs } from "#src/host/job/job-journal.sql";
+
+type Sql<Value> = Effect.Effect<Value, SqlError.SqlError>;
+
+/** A fork the parent recorded: `intent` until its child is published, then `published`. */
+interface ForkRecord {
+  readonly child: string;
+  readonly state: "intent" | "published";
+  readonly version: number;
+}
+
+interface ForkIntent {
+  readonly into: string;
+  readonly version: number;
+}
+
+/** Where a forked actor came from, kept in its `akter_meta` and exported as its lineage (ADR 0019). */
+interface Lineage {
+  readonly forkId: string;
+  /** The parent as `<Actor>/<key segment>`, the form both engines write. */
+  readonly origin: string;
+  readonly version: number;
+}
+
+/**
+ * The `akter_meta` keys a fork's copy keeps: its layout version, and where its history was last pruned. Every other
+ * key belongs to the parent alone, such as the key of the parent's last export (`export_record`) or the parent's own
+ * lineage, so a copy drops it. A key added to `akter_meta` stays with the parent until it is listed here.
+ */
+const copiedMeta: readonly string[] = ["layout", "pruned_before"];
+
+const prepareForks = (sql: SqlClient.SqlClient): Sql<void> =>
+  Effect.asVoid(
+    sql`CREATE TABLE IF NOT EXISTS akter_forks (command_id TEXT PRIMARY KEY, child TEXT NOT NULL, state TEXT NOT NULL CHECK (state IN ('intent', 'published')), version INTEGER NOT NULL) STRICT`,
+  );
+
+interface Named {
+  readonly commandId: string;
+  readonly sql: SqlClient.SqlClient;
+}
+
+const recordedFork = ({ commandId, sql }: Named): Sql<Option.Option<ForkRecord>> =>
+  Effect.map(sql<ForkRecord>`SELECT child, state, version FROM akter_forks WHERE command_id = ${commandId}`, Arr.head);
+
+const recordIntent = ({ commandId, into, sql, version }: ForkIntent & Named): Sql<void> =>
+  Effect.asVoid(
+    sql`INSERT INTO akter_forks (command_id, child, state, version) VALUES (${commandId}, ${into}, 'intent', ${version}) ON CONFLICT (command_id) DO UPDATE SET version = excluded.version`,
+  );
+
+const markPublished = ({ commandId, sql }: Named): Sql<void> =>
+  Effect.asVoid(sql`UPDATE akter_forks SET state = 'published' WHERE command_id = ${commandId}`);
+
+/**
+ * Makes a copy of an actor's database into a quiet fork (ADR 0017): its timers, undelivered messages and open jobs
+ * are dropped or cancelled, its history starts at the copy, its file rows are cleared for the fork to inherit the
+ * files under its own claims, its `akter_meta` keeps only `copiedMeta`, and its lineage is recorded. Runs on the copy
+ * before anything opens it as an actor.
+ */
+const quietCopy = ({ forkId, origin, sql, version }: Lineage & { readonly sql: SqlClient.SqlClient }): Sql<void> =>
+  sql.withTransaction(
+    Effect.gen(function* quietFork() {
+      yield* prepareForks(sql);
+      yield* sql`DELETE FROM akter_forks`;
+      yield* sql`DELETE FROM akter_timers`;
+      yield* sql`DELETE FROM akter_outbox`;
+      yield* cancelOpenJobs(sql);
+      yield* startJournalAtCopy(sql);
+      yield* clearCopiedFiles(sql);
+      yield* sql`DELETE FROM akter_meta WHERE key NOT IN ${sql.in(copiedMeta)}`;
+      yield* sql`INSERT INTO akter_meta (key, value) VALUES ('fork.origin', ${origin}), ('fork.id', ${forkId}), ('fork.version', ${version})`;
+    }),
+  );
+
+/** The file fork a staged copy began: the child's file identity, and the parent's it forks from. */
+interface ForkedFiles {
+  readonly actor: string;
+  readonly origin: string;
+}
+
+/**
+ * The file fork recorded in a quieted copy, or none before its files were forked. Read it only from a copy whose
+ * `fork.id` is this fork's: before quietCopy commits, a copy holds its parent's own file rows.
+ */
+const forkedFiles = (sql: SqlClient.SqlClient): Sql<Option.Option<ForkedFiles>> =>
+  Effect.map(
+    sql<ForkedFiles>`SELECT owner.actor AS actor, fork.origin AS origin FROM akter_blob_owners AS owner, akter_blob_fork AS fork`,
+    Arr.head,
+  );
+
+/** The fork id a published child carries, or none for an actor that was not forked. */
+const forkIdOf = (sql: SqlClient.SqlClient): Sql<Option.Option<string>> =>
+  Effect.map(sql<{ readonly value: string }>`SELECT value FROM akter_meta WHERE key = 'fork.id'`, (rows) =>
+    Option.map(Arr.head(rows), (row) => row.value),
+  );
+
+/** The lineage a forked actor's `akter_meta` records, or none for an actor that was not forked. */
+const lineageOf = (sql: SqlClient.SqlClient): Sql<Option.Option<Lineage>> =>
+  Effect.map(
+    sql<Lineage>`SELECT origin.value AS origin, id.value AS forkId, version.value AS version FROM akter_meta AS origin JOIN akter_meta AS id ON id.key = 'fork.id' JOIN akter_meta AS version ON version.key = 'fork.version' WHERE origin.key = 'fork.origin'`,
+    Arr.head,
+  );
+
+export { forkedFiles, forkIdOf, lineageOf, markPublished, prepareForks, quietCopy, recordedFork, recordIntent };
+
+export type { ForkedFiles, ForkRecord, Lineage };