@akterjs/effect 0.0.0-stage → 0.1.0

package/src/host/blob/blob-range.pure.ts

@@ -0,0 +1,33 @@
+import { Effect, Schema } from "effect";
+
+import type { BlobRange } from "./blob-sealing.contract.js";
+import { BlobTransferError } from "#src/contract/file/blob-transfer.errors";
+
+interface Ranging {
+  readonly blob: string;
+  readonly bytes: number;
+  readonly range?: BlobRange;
+}
+
+const zeroBytes = 0;
+
+const Range = Schema.Struct({
+  length: Schema.Int.check(Schema.isGreaterThanOrEqualTo(zeroBytes)),
+  offset: Schema.Int.check(Schema.isGreaterThanOrEqualTo(zeroBytes)),
+});
+
+const rangeOf = Effect.fnUntraced(function* rangeOf({ blob, bytes, range }: Ranging) {
+  const valid = yield* Schema.decodeEffect(Range)(range ?? { length: bytes, offset: zeroBytes }).pipe(
+    Effect.mapError(() =>
+      BlobTransferError.make({ blob, message: "the byte range must use non-negative safe integers" }),
+    ),
+  );
+
+  if (valid.length > bytes - valid.offset) {
+    return yield* BlobTransferError.make({ blob, message: "the byte range is outside the file" });
+  }
+
+  return valid;
+});
+
+export { rangeOf };

package/src/host/blob/blob-sealing.contract.ts

@@ -0,0 +1,46 @@
+import type { Effect, Stream } from "effect";
+
+import type { BlobTransferError } from "#src/contract/client/client-failure.contract";
+
+interface BlobRange {
+  readonly length: number;
+  readonly offset: number;
+}
+
+interface SealedBlob {
+  readonly bytes: number;
+  readonly grant: string;
+  readonly sha256: string;
+}
+
+interface Sealing<Failure, Services> {
+  readonly actor: string;
+  readonly blob: string;
+  readonly filename: string;
+  readonly source: Stream.Stream<Uint8Array, Failure, Services>;
+}
+
+interface Opening {
+  readonly actor: string;
+  readonly blob: string;
+  readonly filename: string;
+  readonly grant: string;
+  readonly range?: BlobRange;
+}
+
+interface Rewrapping {
+  readonly blob: string;
+  readonly from: string;
+  readonly grant: string;
+  readonly to: string;
+}
+
+interface BlobSealingOperations {
+  readonly open: (opening: Opening) => Stream.Stream<Uint8Array, BlobTransferError>;
+  readonly rewrap: (rewrapping: Rewrapping) => Effect.Effect<string, BlobTransferError>;
+  readonly seal: <Failure, Services>(
+    sealing: Sealing<Failure, Services>,
+  ) => Effect.Effect<SealedBlob, BlobTransferError | Failure, Services>;
+}
+
+export type { BlobRange, BlobSealingOperations, Opening, Rewrapping, SealedBlob, Sealing };

package/src/host/blob/blob-sealing.service.ts

@@ -0,0 +1,25 @@
+import { Context, Effect, Layer } from "effect";
+import { Aead, SecureRandom } from "effect/crypto";
+
+import { BlobGrants } from "./blob-grants.service.js";
+import { BlobHashing } from "#src/host/hash/blob-hashing.service";
+import type { BlobSealingOperations } from "./blob-sealing.contract.js";
+import { plainBlobSealing } from "./plain-blob-sealing.pure.js";
+import { sealedBlobSealing } from "./sealed-blob-sealing.pure.js";
+
+export class BlobSealing extends Context.Service<BlobSealing, BlobSealingOperations>()(
+  "@akterjs/effect/host/blob/blob-sealing.service/BlobSealing",
+) {
+  public static readonly makePlain = Effect.flatMap(BlobHashing, plainBlobSealing);
+
+  public static readonly plaintext = Layer.effect(this, this.makePlain);
+
+  public static readonly sealed = Layer.effect(
+    this,
+    Effect.flatMap(Effect.all({ grants: BlobGrants, hashing: BlobHashing }), sealedBlobSealing),
+  ).pipe(
+    Layer.provide(BlobGrants.layer),
+    Layer.provide(Aead.layerWebCrypto),
+    Layer.provide(SecureRandom.layerWebCrypto),
+  );
+}

package/src/host/blob/export/blob-archive.sql.ts

@@ -0,0 +1,153 @@
+import { Array as Arr, Effect, Option } from "effect";
+import type { Stream } from "effect";
+import type { SqlClient, SqlError } from "effect/sql";
+
+import type { BlobTransferError } from "#src/contract/file/blob-transfer.errors";
+import { prepareReceipts } from "#src/host/blob/reference/blob-receipts.sql";
+import type { BlobTransfers } from "#src/host/blob/reference/blob-transfer.pure";
+
+/** A file an archived actor keeps, under the claim and grant the engine it came from recorded. */
+interface ArchivedFile {
+  readonly blob: string;
+  readonly bytes: number;
+  readonly claim: string;
+  readonly grant: string;
+  readonly sha256: string;
+}
+
+/** An archived actor's database and the home its files move into. */
+interface ArchivedOwner<Failure, Services> {
+  /** The identity this home holds the actor's files by. */
+  readonly actor: string;
+  readonly adopt: BlobTransfers["adopt"];
+  readonly sql: SqlClient.SqlClient;
+  /** The archived bytes of the file with this SHA-256, read only when the home lacks its blob. */
+  readonly stored: (sha256: string) => Stream.Stream<Uint8Array, Failure, Services>;
+}
+
+interface AdoptedFile {
+  readonly claim: string;
+  readonly grant: string;
+}
+
+const keptPageSize = 100;
+
+const ownerTables = [
+  "akter_blob_owners",
+  "akter_blob_receipts",
+  "akter_blob_links",
+  "akter_blob_releases",
+  "akter_blob_history",
+  "akter_blob_ancestry",
+] as const;
+
+const grantTables = ["akter_blob_links", "akter_blob_ancestry", "akter_blob_receipts"] as const;
+
+const recordedOwner = (sql: SqlClient.SqlClient): Effect.Effect<Option.Option<string>, SqlError.SqlError> =>
+  Effect.map(sql<{ readonly actor: string }>`SELECT actor FROM akter_blob_owners WHERE id = 1`, (rows) =>
+    Option.map(Arr.head(rows), (row) => row.actor),
+  );
+
+/**
+ * A page of the files the actor keeps after the claim `after`, the same files an export snapshot holds: its linked
+ * files, its ancestry's, and the receipts its history keeps.
+ */
+const keptFiles = (
+  sql: SqlClient.SqlClient,
+  after: string,
+): Effect.Effect<readonly ArchivedFile[], SqlError.SqlError> =>
+  sql<ArchivedFile>`SELECT blob, bytes, claim, grant, sha256 FROM (SELECT blob, bytes, claim, grant, sha256 FROM akter_blob_links UNION SELECT blob, bytes, claim, grant, sha256 FROM akter_blob_ancestry UNION SELECT blob, bytes, claim, grant, sha256 FROM akter_blob_receipts AS receipt WHERE EXISTS (SELECT 1 FROM akter_blob_history WHERE claim = receipt.claim) AND NOT EXISTS (SELECT 1 FROM akter_blob_links WHERE claim = receipt.claim)) WHERE claim > ${after} ORDER BY claim LIMIT ${keptPageSize}`;
+
+const adoptedPage = <Failure, Services>(
+  owner: ArchivedOwner<Failure, Services>,
+  origin: string,
+  files: readonly ArchivedFile[],
+): Effect.Effect<readonly AdoptedFile[], BlobTransferError | SqlError.SqlError | Failure, Services> =>
+  Effect.forEach(files, ({ blob, bytes, claim, grant, sha256 }) =>
+    Effect.map(
+      owner.adopt({
+        claim: { actor: owner.actor, blob, claim },
+        file: { bytes, grant, sha256 },
+        origin,
+        stored: owner.stored(sha256),
+      }),
+      (adopted): AdoptedFile => ({ claim, grant: adopted }),
+    ),
+  );
+
+const adoptedFiles = Effect.fnUntraced(function* adoptedFiles<Failure, Services>(
+  owner: ArchivedOwner<Failure, Services>,
+  origin: string,
+) {
+  const adopted: AdoptedFile[] = [];
+  let after = "";
+
+  for (;;) {
+    const files = yield* keptFiles(owner.sql, after);
+
+    if (!Arr.isReadonlyArrayNonEmpty(files)) {
+      return adopted;
+    }
+
+    adopted.push(...(yield* adoptedPage(owner, origin, files)));
+    after = Arr.lastNonEmpty(files).claim;
+  }
+});
+
+/**
+ * Rewrites the file tables for the actor's new identity and grants. Uploads no command attached never left their
+ * engine, so their receipts go, and so do releases of claims the actor no longer keeps.
+ */
+const rewritten = (
+  sql: SqlClient.SqlClient,
+  actor: string,
+  adopted: readonly AdoptedFile[],
+): Effect.Effect<void, SqlError.SqlError> =>
+  sql.withTransaction(
+    Effect.gen(function* rewriteFiles() {
+      yield* sql`DELETE FROM akter_blob_receipts AS receipt WHERE NOT EXISTS (SELECT 1 FROM akter_blob_links WHERE claim = receipt.claim) AND NOT EXISTS (SELECT 1 FROM akter_blob_history WHERE claim = receipt.claim)`;
+      yield* sql`DELETE FROM akter_blob_releases WHERE claim NOT IN (SELECT claim FROM akter_blob_links UNION SELECT claim FROM akter_blob_ancestry UNION SELECT claim FROM akter_blob_receipts)`;
+      yield* sql`DELETE FROM akter_blob_export_holds`;
+      yield* Effect.forEach(ownerTables, (table) => sql.unsafe(`UPDATE ${table} SET actor = ?`, [actor]), {
+        discard: true,
+      });
+      yield* Effect.forEach(
+        adopted,
+        ({ claim, grant }) =>
+          Effect.forEach(
+            grantTables,
+            (table) => sql.unsafe(`UPDATE ${table} SET grant = ? WHERE claim = ?`, [grant, claim]),
+            {
+              discard: true,
+            },
+          ),
+        { discard: true },
+      );
+    }),
+  );
+
+/**
+ * Takes an archived actor's files into this engine's home in place: each file it keeps is adopted under its own
+ * claim for `owner.actor`, then its file tables are rewritten for that identity and the grants this home answered.
+ * An actor that never kept a file is left as it is. Running it again after a crash adopts the same claims again and
+ * rewrites the same rows, so it finishes what the first run started.
+ */
+const adoptArchivedFiles = Effect.fnUntraced(function* adoptArchivedFiles<Failure, Services>(
+  owner: ArchivedOwner<Failure, Services>,
+) {
+  yield* prepareReceipts(owner.sql);
+
+  const origin = yield* recordedOwner(owner.sql);
+
+  if (Option.isNone(origin)) {
+    return yield* Effect.void;
+  }
+
+  const adopted = yield* adoptedFiles(owner, origin.value);
+
+  return yield* rewritten(owner.sql, owner.actor, adopted);
+});
+
+export { adoptArchivedFiles };
+
+export type { ArchivedOwner };

package/src/host/blob/export/blob-export-holds.sql.ts

@@ -0,0 +1,91 @@
+import { Array as Arr, Effect } from "effect";
+import type { Option } from "effect";
+import type { SqlClient, SqlError } from "effect/sql";
+
+import { prepareReceipts } from "#src/host/blob/reference/blob-receipts.sql";
+
+/** The actor's own connection, which every hold is read and written on. */
+interface OnActor {
+  readonly sql: SqlClient.SqlClient;
+}
+
+/** A snapshot's hold on the actor's files, kept until `now + lifetimeMillis`. */
+interface ExportHold extends OnActor {
+  readonly lifetimeMillis: number;
+  readonly now: number;
+  readonly snapshot: string;
+}
+
+/** A page of a snapshot's held files after the SHA-256 `after` (the start when missing). */
+interface HeldPage extends OnActor {
+  readonly after?: string;
+  readonly snapshot: string;
+}
+
+/** One held file as an export lists it: only its content's SHA-256 and size, never a claim or a grant. */
+interface ExportedFile {
+  readonly bytes: number;
+  readonly sha256: string;
+}
+
+interface HeldLookup extends OnActor {
+  readonly now: number;
+  readonly sha256: string;
+  readonly snapshot: string;
+}
+
+/** What an engine needs to open a held file for the actor: its blob, the claim holding it, and its grant. */
+interface HeldBlob {
+  readonly actor: string;
+  readonly blob: string;
+  readonly bytes: number;
+  readonly claim: string;
+  readonly grant: string;
+}
+
+const exportPageSize = 100;
+
+/**
+ * Holds every file the actor keeps, at the cut an export snapshot takes, for `lifetimeMillis` after `now`: its
+ * linked files, the files its ancestry keeps from a fork, and the receipts its command history keeps. While a hold
+ * lasts, a removed file's claim is not released, so its blob stays to be downloaded; once it expires, the release
+ * goes ahead. A new hold replaces the actor's previous one, whose snapshot is gone. Run it on the actor's own
+ * connection, in the same turn as the snapshot, so no file changes between the two. (`WHERE true` keeps SQLite from
+ * reading the upsert's `ON CONFLICT` as a join constraint.)
+ */
+const holdExportFiles = ({ lifetimeMillis, now, snapshot, sql }: ExportHold): Effect.Effect<void, SqlError.SqlError> =>
+  sql.withTransaction(
+    Effect.gen(function* holdFiles() {
+      const expiresAt = now + lifetimeMillis;
+
+      yield* prepareReceipts(sql);
+      yield* sql`DELETE FROM akter_blob_export_holds`;
+      yield* sql`INSERT INTO akter_blob_export_holds (snapshot, claim, actor, blob, bytes, sha256, grant, expires_at) SELECT ${snapshot}, claim, actor, blob, bytes, sha256, grant, ${expiresAt} FROM akter_blob_links WHERE true ON CONFLICT DO NOTHING`;
+      yield* sql`INSERT INTO akter_blob_export_holds (snapshot, claim, actor, blob, bytes, sha256, grant, expires_at) SELECT ${snapshot}, claim, actor, blob, bytes, sha256, grant, ${expiresAt} FROM akter_blob_ancestry WHERE true ON CONFLICT DO NOTHING`;
+      yield* sql`INSERT INTO akter_blob_export_holds (snapshot, claim, actor, blob, bytes, sha256, grant, expires_at) SELECT ${snapshot}, claim, actor, blob, bytes, sha256, grant, ${expiresAt} FROM akter_blob_receipts AS receipt WHERE EXISTS (SELECT 1 FROM akter_blob_history WHERE claim = receipt.claim) ON CONFLICT DO NOTHING`;
+    }),
+  );
+
+/** A page of a snapshot's held files, one entry per distinct SHA-256, in SHA-256 order. */
+const heldExportFiles = ({
+  after = "",
+  snapshot,
+  sql,
+}: HeldPage): Effect.Effect<readonly ExportedFile[], SqlError.SqlError> =>
+  sql<ExportedFile>`SELECT DISTINCT sha256, bytes FROM akter_blob_export_holds WHERE snapshot = ${snapshot} AND sha256 > ${after} ORDER BY sha256 LIMIT ${exportPageSize}`;
+
+/** The blob that holds a snapshot's file with this SHA-256, or none once the hold has expired at `now`. */
+const heldExportFile = ({
+  now,
+  sha256,
+  snapshot,
+  sql,
+}: HeldLookup): Effect.Effect<Option.Option<HeldBlob>, SqlError.SqlError> =>
+  Effect.map(
+    sql<HeldBlob>`SELECT actor, blob, bytes, claim, grant FROM akter_blob_export_holds WHERE snapshot = ${snapshot} AND sha256 = ${sha256} AND expires_at > ${now} ORDER BY claim LIMIT 1`,
+    Arr.head,
+  );
+
+export { heldExportFile, heldExportFiles, holdExportFiles };
+
+export type { ExportedFile, ExportHold, HeldBlob, HeldLookup, HeldPage };

package/src/host/blob/plain-blob-sealing.pure.ts

@@ -0,0 +1,70 @@
+import { Effect, FileSystem, Stream } from "effect";
+
+import type { BlobHashing } from "#src/host/hash/blob-hashing.service";
+
+import type { BlobSealingOperations, Opening, SealedBlob, Sealing } from "./blob-sealing.contract.js";
+import { rangeOf } from "./blob-range.pure.js";
+import { BlobTransferError } from "#src/contract/file/blob-transfer.errors";
+import { exactLength } from "./stream/blob-length.pure.js";
+import { blobWriter } from "./stream/blob-writing.pure.js";
+import type { WriteBlob } from "./stream/blob-writing.pure.js";
+
+const zeroBytes = 0;
+
+const failed = (blob: string, message: string): BlobTransferError => BlobTransferError.make({ blob, message });
+
+const sealingWith = (write: WriteBlob, hashing: BlobHashing["Service"]): BlobSealingOperations["seal"] =>
+  Effect.fnUntraced(function* seal<Failure, Services>(
+    request: Sealing<Failure, Services>,
+  ): Effect.fn.Return<SealedBlob, BlobTransferError | Failure, Services> {
+    const hash = yield* hashing.sha256;
+    let bytes = zeroBytes;
+
+    const source = Stream.tap(request.source, (chunk) => {
+      bytes += chunk.byteLength;
+
+      return hash.update(chunk);
+    });
+
+    yield* write({ ...request, source });
+
+    const sha256 = yield* hash.digest;
+
+    return { bytes, grant: "", sha256 };
+  });
+
+const openingWith =
+  (fileSystem: FileSystem.FileSystem): BlobSealingOperations["open"] =>
+  (request: Opening) =>
+    Stream.unwrap(
+      Effect.gen(function* open() {
+        const stored = yield* Effect.mapError(fileSystem.stat(request.filename), () =>
+          failed(request.blob, "the file could not be opened"),
+        );
+
+        const range = yield* rangeOf({ ...request, bytes: Number(stored.size) });
+
+        const source = fileSystem.stream(request.filename, { bytesToRead: range.length, offset: range.offset });
+
+        return exactLength({ blob: request.blob, bytes: range.length, source }).pipe(
+          Stream.map((bytes) => new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength)),
+          Stream.mapError(() => failed(request.blob, "the file could not be read")),
+        );
+      }),
+    );
+
+const plainBlobSealing = (
+  hashing: BlobHashing["Service"],
+): Effect.Effect<BlobSealingOperations, never, FileSystem.FileSystem> =>
+  Effect.gen(function* makePlainSealing() {
+    const fileSystem = yield* FileSystem.FileSystem;
+    const write = yield* blobWriter;
+
+    return {
+      open: openingWith(fileSystem),
+      rewrap: () => Effect.succeed(""),
+      seal: sealingWith(write, hashing),
+    };
+  });
+
+export { plainBlobSealing };