@smthrs/engine-store 0.0.0-stage → 1.0.0-rc.4

package/dist/cjs/ArtifactGc.js

@@ -0,0 +1,183 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var ArtifactGc_exports = {};
+__export(ArtifactGc_exports, {
+  ArtifactGc: () => ArtifactGc,
+  ArtifactGcError: () => ArtifactGcError,
+  ArtifactGcErrorCode: () => ArtifactGcErrorCode,
+  ArtifactGcPolicy: () => ArtifactGcPolicy,
+  defaultGraceMs: () => defaultGraceMs,
+  layer: () => layer,
+  layerFileSystem: () => layerFileSystem,
+  layerPolicy: () => layerPolicy,
+  make: () => make
+});
+module.exports = __toCommonJS(ArtifactGc_exports);
+var ArtifactSweep = __toESM(require("@smthrs/artifacts/ArtifactSweep"), 1);
+var Clock = __toESM(require("effect/Clock"), 1);
+var Context = __toESM(require("effect/Context"), 1);
+var Effect = __toESM(require("effect/Effect"), 1);
+var Layer = __toESM(require("effect/Layer"), 1);
+var Option = __toESM(require("effect/Option"), 1);
+var Schema = __toESM(require("effect/Schema"), 1);
+var SqlClient = __toESM(require("effect/unstable/sql/SqlClient"), 1);
+var ArtifactRoots = __toESM(require("./internal/ArtifactRoots.js"), 1);
+const ArtifactGcErrorCode = Schema.Literals(["invalid_options", "mark_failed", "sweep_failed"]);
+class ArtifactGcError extends Schema.TaggedError()(
+  "@smthrs/engine-store/ArtifactGcError",
+  {
+    code: ArtifactGcErrorCode,
+    message: Schema.String,
+    cause: Schema.optional(Schema.Unknown)
+  }
+) {
+}
+class ArtifactGcPolicy extends Context.Service()(
+  "@smthrs/engine-store/ArtifactGcPolicy"
+) {
+}
+const layerPolicy = (policy) => Layer.succeed(ArtifactGcPolicy)(policy);
+class ArtifactGc extends Context.Service()("@smthrs/engine-store/ArtifactGc") {
+}
+const defaultGraceMs = 14 * 24 * 60 * 60 * 1e3;
+const markFailed = (message, cause) => new ArtifactGcError({ code: "mark_failed", message, cause });
+const sweepFailed = (message, cause) => new ArtifactGcError({ code: "sweep_failed", message, cause });
+const invalidOptions = (message) => new ArtifactGcError({ code: "invalid_options", message });
+const rootDigests = (table, metaJson) => ArtifactRoots.rootDigests(table, metaJson).pipe(
+  Effect.mapError((cause) => markFailed(cause.message, cause.cause))
+);
+const checkpointDigests = (checkpointJson) => ArtifactRoots.checkpointDigests(checkpointJson).pipe(
+  Effect.mapError((cause) => markFailed(cause.message, cause.cause))
+);
+const make = (options = {}) => Effect.gen(function* () {
+  const sql = yield* Effect.service(SqlClient.SqlClient);
+  const sweeper = yield* ArtifactSweep.ArtifactSweep;
+  const policy = yield* Effect.serviceOption(ArtifactGcPolicy);
+  const pageSize = options.pageSize ?? 500;
+  const markCacheEntries = (live) => Effect.gen(function* () {
+    let after = "";
+    for (; ; ) {
+      const rows = yield* sql`
+            SELECT key_digest, meta_json FROM flows_step_cache
+            WHERE key_digest > ${after}
+            ORDER BY key_digest
+            LIMIT ${pageSize}
+          `.pipe(Effect.mapError((cause) => markFailed("the step cache could not be scanned", cause)));
+      for (const row of rows) {
+        for (const digest of yield* rootDigests("flows_step_cache", row.meta_json)) {
+          live.add(digest);
+        }
+      }
+      if (rows.length < pageSize) return;
+      after = rows[rows.length - 1].key_digest;
+    }
+  });
+  const markAttempts = (live) => Effect.gen(function* () {
+    let after = { runId: "", stepKeyDigest: "", attempt: -1 };
+    for (; ; ) {
+      const rows = yield* sql`
+            SELECT run_id, step_key_digest, attempt, checkpoint_json, meta_json FROM flows_attempts
+            WHERE (run_id, step_key_digest, attempt) > (${after.runId}, ${after.stepKeyDigest}, ${after.attempt})
+            ORDER BY run_id, step_key_digest, attempt
+            LIMIT ${pageSize}
+          `.pipe(Effect.mapError((cause) => markFailed("the attempt table could not be scanned", cause)));
+      for (const row of rows) {
+        for (const digest of yield* rootDigests("flows_attempts", row.meta_json)) {
+          live.add(digest);
+        }
+        for (const digest of yield* checkpointDigests(row.checkpoint_json)) {
+          live.add(digest);
+        }
+      }
+      if (rows.length < pageSize) return;
+      const last = rows[rows.length - 1];
+      after = { runId: last.run_id, stepKeyDigest: last.step_key_digest, attempt: last.attempt };
+    }
+  });
+  const gc = Effect.fn("ArtifactGc.gc")(
+    (gcOptions) => Effect.gen(function* () {
+      if (!Number.isSafeInteger(pageSize) || pageSize < 1) {
+        return yield* Effect.fail(invalidOptions("artifact GC pageSize must be a positive safe integer"));
+      }
+      const grace = gcOptions?.graceMs ?? (Option.isSome(policy) ? policy.value.graceMs : void 0) ?? defaultGraceMs;
+      if (!Number.isSafeInteger(grace) || grace < 0) {
+        return yield* Effect.fail(invalidOptions("artifact GC graceMs must be a non-negative safe integer"));
+      }
+      const startedAtMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor));
+      const bound = startedAtMs - grace;
+      const live = new Set(gcOptions?.pins ?? []);
+      if (Option.isSome(policy) && policy.value.pins !== void 0) {
+        for (const digest of yield* policy.value.pins) live.add(digest);
+      }
+      yield* markCacheEntries(live);
+      yield* markAttempts(live);
+      const blobs = yield* sweeper.inventory.pipe(
+        Effect.mapError((cause) => sweepFailed(`the blob inventory refused: ${cause.message}`, cause))
+      );
+      const swept = [];
+      let reclaimedBytes = 0;
+      let keptByGrace = 0;
+      for (const blob of blobs) {
+        if (live.has(blob.digest)) continue;
+        if (blob.modifiedAtMs >= bound) {
+          keptByGrace++;
+          continue;
+        }
+        if (gcOptions?.dryRun === true) {
+          swept.push(blob.digest);
+          reclaimedBytes += blob.sizeBytes;
+          continue;
+        }
+        const removed = yield* sweeper.remove(blob.digest, { ifUnmodifiedSinceMs: bound - 1 }).pipe(
+          Effect.mapError((cause) => sweepFailed(`sweeping ${blob.digest} refused: ${cause.message}`, cause))
+        );
+        if (removed) {
+          swept.push(blob.digest);
+          reclaimedBytes += blob.sizeBytes;
+        } else {
+          keptByGrace++;
+        }
+      }
+      return {
+        scannedBlobs: blobs.length,
+        liveDigests: live.size,
+        sweptDigests: swept,
+        reclaimedBytes,
+        keptByGrace,
+        dryRun: gcOptions?.dryRun === true
+      };
+    })
+  );
+  return { gc };
+});
+const layer = (options = {}) => Layer.effect(ArtifactGc)(make(options));
+const layerFileSystem = (options = {}) => layer(options).pipe(
+  Layer.provide(ArtifactSweep.layerFileSystem({ directory: options.directory, coordination: options.coordination }))
+);
+//# sourceMappingURL=ArtifactGc.js.map

package/dist/cjs/ArtifactGc.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/ArtifactGc.ts"],
+  "sourcesContent": ["/**\n * Mark/sweep garbage collection for the content-addressed artifact store.\n *\n * The artifact tier is the one store in the composition with true *garbage*:\n * every output over the inline bound is spilled to `.flows/objects` by digest\n * (`StepBoundary`), nothing ever deletes a published blob, and a blob whose\n * every referencing row is gone is unreachable forever. The step cache is\n * deliberately NOT collected here \u2014 its rows are the *roots* of this graph,\n * they never become unreachable, and removing one is the policy decision\n * `CacheStore.evict` already serves.\n *\n * The design combines the two prior arts on the shelf:\n *\n * - **Reachability from durable roots** \u2014 git's `git gc` marks from refs;\n *   Skyframe deletes what a keyed graph walk cannot reach. Here the roots are\n *   every attempt row of every existing (never-deleted) run and every step\n *   cache entry; each carries boundary evidence, and\n *   `StepBoundary.referencedDigests` names the blobs that evidence needs.\n *   Attempt checkpoints are roots too: they are opaque executable state, so\n *   the mark keeps digest-shaped strings found in their JSON.\n * - **A grace period over mtime** \u2014 git refuses to prune unreachable objects\n *   younger than `gc.pruneExpire` (two weeks), jj keeps operations newer than\n *   its keep bound, and Bazel's disk-cache collector\n *   (`reference/bazel/.../remote/disk/DiskCacheGarbageCollector.java`) fences\n *   on mtime. The cutoff is the collection start time minus the grace period,\n *   sampled before resolving policy pins or scanning roots and held fixed\n *   through inventory and deletion. Only mtimes strictly before it are\n *   eligible: a blob published \u2014 or freshened by a dedupe `put` \u2014 at or after\n *   collection start survives, including same-millisecond writes at zero\n *   grace, no lock required.\n *\n * Collection NEVER runs automatically. `gc()` is an explicit verb because\n * deletion is irreversible, and a human approving a plan must approve the\n * deletions too. The opt-in {@link ArtifactGcPolicy} layer\n * configures *how* an invocation collects (grace bound, pinned digests); it\n * does not schedule anything.\n *\n * @since 0.1.0\n */\n\nimport * as ArtifactSweep from \"@smthrs/artifacts/ArtifactSweep\"\nimport * as Clock from \"effect/Clock\"\nimport * as Context from \"effect/Context\"\nimport * as Effect from \"effect/Effect\"\nimport type * as FileSystem from \"effect/FileSystem\"\nimport * as Layer from \"effect/Layer\"\nimport * as Option from \"effect/Option\"\nimport * as Schema from \"effect/Schema\"\nimport * as SqlClient from \"effect/unstable/sql/SqlClient\"\nimport * as ArtifactRoots from \"./internal/ArtifactRoots.ts\"\n\n/**\n * Stable error codes returned by artifact garbage collection.\n *\n * @category models\n * @since 0.1.0\n */\nexport const ArtifactGcErrorCode = Schema.Literals([\"invalid_options\", \"mark_failed\", \"sweep_failed\"])\n\n/**\n * Stable error codes returned by artifact garbage collection.\n *\n * @category models\n * @since 0.1.0\n */\nexport type ArtifactGcErrorCode = typeof ArtifactGcErrorCode.Type\n\n/**\n * A collection that could not complete. `mark_failed` means the live set\n * could not be computed \u2014 nothing was deleted. `sweep_failed` means the live\n * set held but a deletion refused; everything already swept was garbage, so a\n * re-run converges from wherever the failure stopped it.\n *\n * @category errors\n * @since 0.1.0\n */\nexport class ArtifactGcError extends Schema.TaggedError<ArtifactGcError>()(\n  \"@smthrs/engine-store/ArtifactGcError\",\n  {\n    code: ArtifactGcErrorCode,\n    message: Schema.String,\n    cause: Schema.optional(Schema.Unknown)\n  }\n) {}\n\n/**\n * The opt-in collection policy. Explicit {@link GcOptions} on a `gc()` call\n * override the installed policy; the policy overrides the defaults.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Policy {\n  /** Grace bound applied when a `gc()` call does not name one. */\n  readonly graceMs?: number | undefined\n  /**\n   * Digests held live regardless of reachability \u2014 the seam for references\n   * this composition's tables cannot see (an external index, an export a\n   * human wants kept). Resolved fresh on every collection.\n   */\n  readonly pins?: Effect.Effect<ReadonlyArray<string>> | undefined\n}\n\n/**\n * Service tag for the opt-in collection policy.\n *\n * @category services\n * @since 0.1.0\n */\nexport class ArtifactGcPolicy extends Context.Service<ArtifactGcPolicy, Policy>()(\n  \"@smthrs/engine-store/ArtifactGcPolicy\"\n) {}\n\n/**\n * Installs a collection policy.\n *\n * @category layers\n * @since 0.1.0\n */\nexport const layerPolicy = (policy: Policy): Layer.Layer<ArtifactGcPolicy> => Layer.succeed(ArtifactGcPolicy)(policy)\n\n/**\n * Options for one explicit collection.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface GcOptions {\n  /**\n   * How recently a blob must have been written or freshened to survive\n   * unreferenced. Defaults to the installed {@link ArtifactGcPolicy}'s bound,\n   * then to {@link defaultGraceMs}.\n   */\n  readonly graceMs?: number | undefined\n  /** Digests held live for this collection, unioned with the policy's pins. */\n  readonly pins?: ReadonlyArray<string> | undefined\n  /** Compute and report without deleting anything. */\n  readonly dryRun?: boolean | undefined\n}\n\n/**\n * What one collection did.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface GcReport {\n  /** Blobs the inventory enumerated. */\n  readonly scannedBlobs: number\n  /** Distinct digests the mark phase proved live, pins included. */\n  readonly liveDigests: number\n  /** Digests deleted \u2014 or, under `dryRun`, the ones a real run would delete. */\n  readonly sweptDigests: ReadonlyArray<string>\n  /** Bytes the swept blobs held. */\n  readonly reclaimedBytes: number\n  /** Unreferenced blobs retained by the grace bound or a freshen race. */\n  readonly keptByGrace: number\n  readonly dryRun: boolean\n}\n\n/**\n * Explicit artifact garbage collection.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface Service {\n  /**\n   * Marks the live set from the durable roots, then sweeps every blob that\n   * is both outside it and older than the grace bound. Explicit only \u2014\n   * nothing in the engine composition ever calls this.\n   */\n  readonly gc: (options?: GcOptions) => Effect.Effect<GcReport, ArtifactGcError>\n}\n\n/**\n * Service tag for artifact garbage collection.\n *\n * @category services\n * @since 0.1.0\n */\nexport class ArtifactGc extends Context.Service<ArtifactGc, Service>()(\"@smthrs/engine-store/ArtifactGc\") {}\n\n/**\n * Two weeks, git's `gc.pruneExpire` default. The bound is deliberately far\n * beyond any live attempt's duration: a blob spilled by a running step is\n * unreferenced until its attempt row finishes, and the grace period is the\n * only thing protecting it in that window.\n *\n * @category constants\n * @since 0.1.0\n */\nexport const defaultGraceMs = 14 * 24 * 60 * 60 * 1000\n\n/**\n * Construction options.\n *\n * @category models\n * @since 0.1.0\n */\nexport interface MakeOptions {\n  /** Rows per mark-phase page. Defaults to 500. */\n  readonly pageSize?: number | undefined\n}\n\nconst markFailed = (message: string, cause: unknown): ArtifactGcError =>\n  new ArtifactGcError({ code: \"mark_failed\", message, cause })\n\nconst sweepFailed = (message: string, cause: unknown): ArtifactGcError =>\n  new ArtifactGcError({ code: \"sweep_failed\", message, cause })\n\nconst invalidOptions = (message: string): ArtifactGcError => new ArtifactGcError({ code: \"invalid_options\", message })\n\n/**\n * Extracts the digests one root row keeps live, FAIL-SAFE: a row whose\n * metadata carries a `boundary` this build cannot decode aborts the\n * collection rather than contributing nothing. Silently reading such a row as\n * \"references no artifacts\" is exactly how a live blob gets collected; a\n * metadata shape with no `boundary` key at all is the ordinary\n * foreign-evidence case `referencedDigests` already defines as empty.\n */\nconst rootDigests = (table: string, metaJson: string): Effect.Effect<ReadonlyArray<string>, ArtifactGcError> =>\n  ArtifactRoots.rootDigests(table, metaJson).pipe(\n    Effect.mapError((cause) => markFailed(cause.message, cause.cause))\n  )\n\nconst checkpointDigests = (checkpointJson: string | null): Effect.Effect<ReadonlyArray<string>, ArtifactGcError> =>\n  ArtifactRoots.checkpointDigests(checkpointJson).pipe(\n    Effect.mapError((cause) => markFailed(cause.message, cause.cause))\n  )\n\n/**\n * Builds the collector over the composition's own durable tables and the\n * host-local sweep surface.\n *\n * The mark phase reads `flows_step_cache` and `flows_attempts` directly, the\n * same way `DurableEngineState` range-scans `flows_runs`: this package\n * composes every one of those migrations, so the schema is its own. Every run\n * present in `flows_runs` is a live root \u2014 there is no deleted state, and the\n * attempt table's foreign key guarantees each attempt's run exists \u2014 so the\n * scan is simply *all* attempt rows plus *all* cache entries, paged by\n * primary key so no page holds more than `pageSize` rows in memory.\n *\n * Ordering is the safety argument. The live set is computed BEFORE the\n * inventory, so a root recorded during the sweep can only be missed, never\n * half-seen \u2014 and the blob such a root references is protected anyway: a\n * fresh publication carries a fresh mtime, a re-publication of existing bytes\n * freshens the blob's mtime (`ArtifactStore.put`), and the sweep's deletion\n * is fenced on that mtime (`ArtifactSweep.remove`'s `ifUnmodifiedSinceMs`).\n * A crash mid-sweep deletes some garbage and no live blobs; re-running\n * converges.\n *\n * @category constructors\n * @since 0.1.0\n */\nexport const make = (\n  options: MakeOptions = {}\n): Effect.Effect<Service, never, SqlClient.SqlClient | ArtifactSweep.ArtifactSweep> =>\n  Effect.gen(function*() {\n    const sql = yield* Effect.service(SqlClient.SqlClient)\n    const sweeper = yield* ArtifactSweep.ArtifactSweep\n    const policy = yield* Effect.serviceOption(ArtifactGcPolicy)\n    const pageSize = options.pageSize ?? 500\n\n    const markCacheEntries = (live: Set<string>) =>\n      Effect.gen(function*() {\n        let after = \"\"\n        for (;;) {\n          const rows = yield* sql<{ readonly key_digest: string; readonly meta_json: string }>`\n            SELECT key_digest, meta_json FROM flows_step_cache\n            WHERE key_digest > ${after}\n            ORDER BY key_digest\n            LIMIT ${pageSize}\n          `.pipe(Effect.mapError((cause) => markFailed(\"the step cache could not be scanned\", cause)))\n          for (const row of rows) {\n            for (const digest of yield* rootDigests(\"flows_step_cache\", row.meta_json)) {\n              live.add(digest)\n            }\n          }\n          if (rows.length < pageSize) return\n          after = rows[rows.length - 1]!.key_digest\n        }\n      })\n\n    const markAttempts = (live: Set<string>) =>\n      Effect.gen(function*() {\n        let after = { runId: \"\", stepKeyDigest: \"\", attempt: -1 }\n        for (;;) {\n          const rows = yield* sql<{\n            readonly run_id: string\n            readonly step_key_digest: string\n            readonly attempt: number\n            readonly checkpoint_json: string | null\n            readonly meta_json: string\n          }>`\n            SELECT run_id, step_key_digest, attempt, checkpoint_json, meta_json FROM flows_attempts\n            WHERE (run_id, step_key_digest, attempt) > (${after.runId}, ${after.stepKeyDigest}, ${after.attempt})\n            ORDER BY run_id, step_key_digest, attempt\n            LIMIT ${pageSize}\n          `.pipe(Effect.mapError((cause) => markFailed(\"the attempt table could not be scanned\", cause)))\n          for (const row of rows) {\n            for (const digest of yield* rootDigests(\"flows_attempts\", row.meta_json)) {\n              live.add(digest)\n            }\n            for (const digest of yield* checkpointDigests(row.checkpoint_json)) {\n              live.add(digest)\n            }\n          }\n          if (rows.length < pageSize) return\n          const last = rows[rows.length - 1]!\n          after = { runId: last.run_id, stepKeyDigest: last.step_key_digest, attempt: last.attempt }\n        }\n      })\n\n    const gc: Service[\"gc\"] = Effect.fn(\"ArtifactGc.gc\")((gcOptions) =>\n      Effect.gen(function*() {\n        if (!Number.isSafeInteger(pageSize) || pageSize < 1) {\n          return yield* Effect.fail(invalidOptions(\"artifact GC pageSize must be a positive safe integer\"))\n        }\n        const grace = gcOptions?.graceMs ??\n          (Option.isSome(policy) ? policy.value.graceMs : undefined) ??\n          defaultGraceMs\n        if (!Number.isSafeInteger(grace) || grace < 0) {\n          return yield* Effect.fail(invalidOptions(\"artifact GC graceMs must be a non-negative safe integer\"))\n        }\n        const startedAtMs = yield* Clock.currentTimeMillis.pipe(Effect.map(Math.floor))\n        const bound = startedAtMs - grace\n        const live = new Set<string>(gcOptions?.pins ?? [])\n        if (Option.isSome(policy) && policy.value.pins !== undefined) {\n          for (const digest of yield* policy.value.pins) live.add(digest)\n        }\n        yield* markCacheEntries(live)\n        yield* markAttempts(live)\n        const blobs = yield* sweeper.inventory.pipe(\n          Effect.mapError((cause) => sweepFailed(`the blob inventory refused: ${cause.message}`, cause))\n        )\n        const swept: Array<string> = []\n        let reclaimedBytes = 0\n        let keptByGrace = 0\n        for (const blob of blobs) {\n          if (live.has(blob.digest)) continue\n          if (blob.modifiedAtMs >= bound) {\n            keptByGrace++\n            continue\n          }\n          if (gcOptions?.dryRun === true) {\n            swept.push(blob.digest)\n            reclaimedBytes += blob.sizeBytes\n            continue\n          }\n          // The fence re-reads the blob's age inside the deletion: an\n          // unreferenced-at-mark blob a concurrent `put` freshened since the\n          // inventory fails the fence and is retained, exactly as a fresher\n          // cache row survives a laggard's fenced evict (issue #119).\n          // remove's fence is inclusive; mtimes have millisecond resolution,\n          // so exclude cutoff equality there just as in the inventory check.\n          const removed = yield* sweeper.remove(blob.digest, { ifUnmodifiedSinceMs: bound - 1 }).pipe(\n            Effect.mapError((cause) => sweepFailed(`sweeping ${blob.digest} refused: ${cause.message}`, cause))\n          )\n          if (removed) {\n            swept.push(blob.digest)\n            reclaimedBytes += blob.sizeBytes\n          } else {\n            keptByGrace++\n          }\n        }\n        return {\n          scannedBlobs: blobs.length,\n          liveDigests: live.size,\n          sweptDigests: swept,\n          reclaimedBytes,\n          keptByGrace,\n          dryRun: gcOptions?.dryRun === true\n        }\n      })\n    )\n\n    return { gc }\n  })\n\n/**\n * Provides the artifact garbage collector. Install {@link layerPolicy} into\n * this layer's inputs to configure collection; nothing schedules `gc()` \u2014\n * invoking it stays an explicit caller decision.\n *\n * @category layers\n * @since 0.1.0\n */\nexport const layer = (\n  options: MakeOptions = {}\n): Layer.Layer<ArtifactGc, never, SqlClient.SqlClient | ArtifactSweep.ArtifactSweep> =>\n  Layer.effect(ArtifactGc)(make(options))\n\n/**\n * Provides the artifact garbage collector over a filesystem objects directory:\n * {@link layer} with `ArtifactSweep.layerFileSystem` beneath it. Pass the same\n * `directory` and `coordination` the paired `ArtifactStore.layerFileSystem`\n * publishes with.\n *\n * @category layers\n * @since 1.0.0\n */\nexport const layerFileSystem = (\n  options: MakeOptions & ArtifactSweep.SweepOptions = {}\n): Layer.Layer<ArtifactGc, never, SqlClient.SqlClient | FileSystem.FileSystem> =>\n  layer(options).pipe(\n    Layer.provide(ArtifactSweep.layerFileSystem({ directory: options.directory, coordination: options.coordination }))\n  )\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAwCA,oBAA+B;AAC/B,YAAuB;AACvB,cAAyB;AACzB,aAAwB;AAExB,YAAuB;AACvB,aAAwB;AACxB,aAAwB;AACxB,gBAA2B;AAC3B,oBAA+B;AAQxB,MAAM,sBAAsB,OAAO,SAAS,CAAC,mBAAmB,eAAe,cAAc,CAAC;AAmB9F,MAAM,wBAAwB,OAAO,YAA6B;AAAA,EACvE;AAAA,EACA;AAAA,IACE,MAAM;AAAA,IACN,SAAS,OAAO;AAAA,IAChB,OAAO,OAAO,SAAS,OAAO,OAAO;AAAA,EACvC;AACF,EAAE;AAAC;AA0BI,MAAM,yBAAyB,QAAQ,QAAkC;AAAA,EAC9E;AACF,EAAE;AAAC;AAQI,MAAM,cAAc,CAAC,WAAkD,MAAM,QAAQ,gBAAgB,EAAE,MAAM;AA8D7G,MAAM,mBAAmB,QAAQ,QAA6B,EAAE,iCAAiC,EAAE;AAAC;AAWpG,MAAM,iBAAiB,KAAK,KAAK,KAAK,KAAK;AAalD,MAAM,aAAa,CAAC,SAAiB,UACnC,IAAI,gBAAgB,EAAE,MAAM,eAAe,SAAS,MAAM,CAAC;AAE7D,MAAM,cAAc,CAAC,SAAiB,UACpC,IAAI,gBAAgB,EAAE,MAAM,gBAAgB,SAAS,MAAM,CAAC;AAE9D,MAAM,iBAAiB,CAAC,YAAqC,IAAI,gBAAgB,EAAE,MAAM,mBAAmB,QAAQ,CAAC;AAUrH,MAAM,cAAc,CAAC,OAAe,aAClC,cAAc,YAAY,OAAO,QAAQ,EAAE;AAAA,EACzC,OAAO,SAAS,CAAC,UAAU,WAAW,MAAM,SAAS,MAAM,KAAK,CAAC;AACnE;AAEF,MAAM,oBAAoB,CAAC,mBACzB,cAAc,kBAAkB,cAAc,EAAE;AAAA,EAC9C,OAAO,SAAS,CAAC,UAAU,WAAW,MAAM,SAAS,MAAM,KAAK,CAAC;AACnE;AA0BK,MAAM,OAAO,CAClB,UAAuB,CAAC,MAExB,OAAO,IAAI,aAAY;AACrB,QAAM,MAAM,OAAO,OAAO,QAAQ,UAAU,SAAS;AACrD,QAAM,UAAU,OAAO,cAAc;AACrC,QAAM,SAAS,OAAO,OAAO,cAAc,gBAAgB;AAC3D,QAAM,WAAW,QAAQ,YAAY;AAErC,QAAM,mBAAmB,CAAC,SACxB,OAAO,IAAI,aAAY;AACrB,QAAI,QAAQ;AACZ,eAAS;AACP,YAAM,OAAO,OAAO;AAAA;AAAA,iCAEG,KAAK;AAAA;AAAA,oBAElB,QAAQ;AAAA,YAChB,KAAK,OAAO,SAAS,CAAC,UAAU,WAAW,uCAAuC,KAAK,CAAC,CAAC;AAC3F,iBAAW,OAAO,MAAM;AACtB,mBAAW,UAAU,OAAO,YAAY,oBAAoB,IAAI,SAAS,GAAG;AAC1E,eAAK,IAAI,MAAM;AAAA,QACjB;AAAA,MACF;AACA,UAAI,KAAK,SAAS,SAAU;AAC5B,cAAQ,KAAK,KAAK,SAAS,CAAC,EAAG;AAAA,IACjC;AAAA,EACF,CAAC;AAEH,QAAM,eAAe,CAAC,SACpB,OAAO,IAAI,aAAY;AACrB,QAAI,QAAQ,EAAE,OAAO,IAAI,eAAe,IAAI,SAAS,GAAG;AACxD,eAAS;AACP,YAAM,OAAO,OAAO;AAAA;AAAA,0DAQ4B,MAAM,KAAK,KAAK,MAAM,aAAa,KAAK,MAAM,OAAO;AAAA;AAAA,oBAE3F,QAAQ;AAAA,YAChB,KAAK,OAAO,SAAS,CAAC,UAAU,WAAW,0CAA0C,KAAK,CAAC,CAAC;AAC9F,iBAAW,OAAO,MAAM;AACtB,mBAAW,UAAU,OAAO,YAAY,kBAAkB,IAAI,SAAS,GAAG;AACxE,eAAK,IAAI,MAAM;AAAA,QACjB;AACA,mBAAW,UAAU,OAAO,kBAAkB,IAAI,eAAe,GAAG;AAClE,eAAK,IAAI,MAAM;AAAA,QACjB;AAAA,MACF;AACA,UAAI,KAAK,SAAS,SAAU;AAC5B,YAAM,OAAO,KAAK,KAAK,SAAS,CAAC;AACjC,cAAQ,EAAE,OAAO,KAAK,QAAQ,eAAe,KAAK,iBAAiB,SAAS,KAAK,QAAQ;AAAA,IAC3F;AAAA,EACF,CAAC;AAEH,QAAM,KAAoB,OAAO,GAAG,eAAe;AAAA,IAAE,CAAC,cACpD,OAAO,IAAI,aAAY;AACrB,UAAI,CAAC,OAAO,cAAc,QAAQ,KAAK,WAAW,GAAG;AACnD,eAAO,OAAO,OAAO,KAAK,eAAe,sDAAsD,CAAC;AAAA,MAClG;AACA,YAAM,QAAQ,WAAW,YACtB,OAAO,OAAO,MAAM,IAAI,OAAO,MAAM,UAAU,WAChD;AACF,UAAI,CAAC,OAAO,cAAc,KAAK,KAAK,QAAQ,GAAG;AAC7C,eAAO,OAAO,OAAO,KAAK,eAAe,yDAAyD,CAAC;AAAA,MACrG;AACA,YAAM,cAAc,OAAO,MAAM,kBAAkB,KAAK,OAAO,IAAI,KAAK,KAAK,CAAC;AAC9E,YAAM,QAAQ,cAAc;AAC5B,YAAM,OAAO,IAAI,IAAY,WAAW,QAAQ,CAAC,CAAC;AAClD,UAAI,OAAO,OAAO,MAAM,KAAK,OAAO,MAAM,SAAS,QAAW;AAC5D,mBAAW,UAAU,OAAO,OAAO,MAAM,KAAM,MAAK,IAAI,MAAM;AAAA,MAChE;AACA,aAAO,iBAAiB,IAAI;AAC5B,aAAO,aAAa,IAAI;AACxB,YAAM,QAAQ,OAAO,QAAQ,UAAU;AAAA,QACrC,OAAO,SAAS,CAAC,UAAU,YAAY,+BAA+B,MAAM,OAAO,IAAI,KAAK,CAAC;AAAA,MAC/F;AACA,YAAM,QAAuB,CAAC;AAC9B,UAAI,iBAAiB;AACrB,UAAI,cAAc;AAClB,iBAAW,QAAQ,OAAO;AACxB,YAAI,KAAK,IAAI,KAAK,MAAM,EAAG;AAC3B,YAAI,KAAK,gBAAgB,OAAO;AAC9B;AACA;AAAA,QACF;AACA,YAAI,WAAW,WAAW,MAAM;AAC9B,gBAAM,KAAK,KAAK,MAAM;AACtB,4BAAkB,KAAK;AACvB;AAAA,QACF;AAOA,cAAM,UAAU,OAAO,QAAQ,OAAO,KAAK,QAAQ,EAAE,qBAAqB,QAAQ,EAAE,CAAC,EAAE;AAAA,UACrF,OAAO,SAAS,CAAC,UAAU,YAAY,YAAY,KAAK,MAAM,aAAa,MAAM,OAAO,IAAI,KAAK,CAAC;AAAA,QACpG;AACA,YAAI,SAAS;AACX,gBAAM,KAAK,KAAK,MAAM;AACtB,4BAAkB,KAAK;AAAA,QACzB,OAAO;AACL;AAAA,QACF;AAAA,MACF;AACA,aAAO;AAAA,QACL,cAAc,MAAM;AAAA,QACpB,aAAa,KAAK;AAAA,QAClB,cAAc;AAAA,QACd;AAAA,QACA;AAAA,QACA,QAAQ,WAAW,WAAW;AAAA,MAChC;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO,EAAE,GAAG;AACd,CAAC;AAUI,MAAM,QAAQ,CACnB,UAAuB,CAAC,MAExB,MAAM,OAAO,UAAU,EAAE,KAAK,OAAO,CAAC;AAWjC,MAAM,kBAAkB,CAC7B,UAAoD,CAAC,MAErD,MAAM,OAAO,EAAE;AAAA,EACb,MAAM,QAAQ,cAAc,gBAAgB,EAAE,WAAW,QAAQ,WAAW,cAAc,QAAQ,aAAa,CAAC,CAAC;AACnH;",
+  "names": []
+}

package/dist/cjs/ArtifactSync.d.ts

@@ -0,0 +1,164 @@
+/**
+ * The two-tier artifact protocol the step cache rides on: publish before the
+ * entry, fetch on demand after it.
+ *
+ * This is the seam that makes a shared cache safe. A cache entry references
+ * its large outputs by digest, so an entry that becomes observable while one
+ * of those artifacts is still missing hands the next machine a hit it cannot
+ * materialize. Bazel states the constraint at
+ * `reference/bazel/.../remote/UploadManifest.java:630-633` — "action results
+ * may fail to validate server-side if they are accessed before all blobs they
+ * refer to are present" — and enforces it by ordering:
+ * `findMissingDigests` → upload the missing → upload the `ActionResult` LAST.
+ * {@link Service.publish} is that ordering, and `ActionPersistence` calls it
+ * before `CacheStore.put`.
+ *
+ * The read direction is the same protocol in reverse, and how eagerly it runs
+ * is the {@link DownloadPolicy}, declared on the shared tier itself
+ * (`RemoteArtifacts.Options.downloadPolicy`) and split across two seams.
+ * `all` fetches every referenced artifact into this host's store while
+ * admitting the replay. `toplevel` and `minimal` fetch nothing here: one
+ * batched probe establishes that the shared tier can serve them and the
+ * transfer is left to the first read through `CombinedArtifacts`, which then
+ * writes the blob back under `toplevel` and does not under `minimal`. This is
+ * Bazel's `RemoteOutputChecker` dial
+ * (`--remote_download_{all,toplevel,minimal}`) split the way our two seams
+ * split: what a replay prefetches is this module's decision, what a read
+ * materializes is the combined store's.
+ *
+ * Governing design: `packages/smithers/flows/artifacts/docs/api.md`.
+ *
+ * @since 0.1.0
+ */
+import * as ArtifactStore from "@smthrs/artifacts/ArtifactStore";
+import * as RemoteArtifacts from "@smthrs/artifacts/RemoteArtifacts";
+import * as Context from "effect/Context";
+import type * as Crypto from "effect/Crypto";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Schema from "effect/Schema";
+declare const ArtifactPublicationFailed_base: Schema.Class<ArtifactPublicationFailed, Schema.TaggedStruct<"@smthrs/engine-store/ArtifactPublicationFailed", {
+    readonly code: Schema.Literal<"artifact_publication_failed">;
+    readonly digests: Schema.$Array<Schema.String>;
+    readonly message: Schema.String;
+    readonly cause: Schema.optional<Schema.Union<readonly [typeof ArtifactStore.ArtifactStoreError, typeof ArtifactStore.ArtifactMissing, typeof ArtifactStore.ArtifactCorruption]>>;
+}>, import("effect/Cause").YieldableError>;
+/**
+ * A referenced artifact could not be made durable in the shared tier, so the
+ * cache entry that references it must not be published.
+ *
+ * Failing here is the point: silently publishing the entry anyway is exactly
+ * the state the REAPI ordering constraint exists to prevent.
+ *
+ * @since 0.1.0
+ * @category errors
+ */
+export declare class ArtifactPublicationFailed extends ArtifactPublicationFailed_base {
+}
+/**
+ * How eagerly a replay materializes the artifacts it references.
+ *
+ * `all` downloads every referenced artifact into this host's local store while
+ * the replay is admitted, so every later read is local and a shared tier that
+ * goes down afterwards costs nothing. `minimal` downloads none of them: it
+ * establishes that each digest is resolvable — locally, or from the shared tier
+ * — and lets the first actual read fetch the bytes.
+ *
+ * `minimal` is only sound when the store the replay reads through can reach the
+ * shared tier, which means `CombinedArtifacts` with the same remote tier. Under
+ * a purely local `ArtifactStore` an admitted `minimal` replay would later read
+ * an artifact this host never fetched.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export type DownloadPolicy = RemoteArtifacts.DownloadPolicy;
+/**
+ * The two directions of the artifact protocol: {@link Service.publish} before a
+ * cache entry becomes observable, {@link Service.hydrate} when a replay needs
+ * bytes this host does not have.
+ *
+ * The asymmetry in their error types is the design: publishing may fail,
+ * because publishing an entry whose artifacts are missing is the unsafe state;
+ * hydrating never fails, because a shared tier that is down should cost a run
+ * a cache hit, not the run itself.
+ *
+ * @since 0.1.0
+ * @category services
+ */
+export interface Service {
+    /**
+     * Makes every digest durable in the shared tier, in REAPI order:
+     * `findMissing`, upload what is missing, then confirm nothing is missing
+     * any more. Returns only once the shared tier can serve all of them.
+     */
+    readonly publish: (digests: ReadonlyArray<string>) => Effect.Effect<void, ArtifactPublicationFailed, Crypto.Crypto>;
+    /**
+     * Fetches the digests this host is missing from the shared tier and writes
+     * them into the local store, reporting whether every digest is now locally
+     * resolvable.
+     *
+     * Best-effort by construction: `false` means "replay still cannot be
+     * satisfied", which the caller turns into a real execution. A shared tier
+     * that is down must never fail a run that can simply do the work.
+     */
+    readonly hydrate: (digests: ReadonlyArray<string>) => Effect.Effect<boolean, never, Crypto.Crypto>;
+}
+declare const ArtifactSync_base: Context.ServiceClass<ArtifactSync, "@smthrs/engine-store/ArtifactSync", Service>;
+/**
+ * Service tag for the two-tier artifact protocol.
+ *
+ * @since 0.1.0
+ * @category services
+ */
+export declare class ArtifactSync extends ArtifactSync_base {
+}
+/**
+ * The single-tier implementation: there is no shared artifact store, so there
+ * is nothing to publish and nothing to fetch.
+ *
+ * This is the default `ActionPersistence` falls back to when the tag is
+ * absent, which is what keeps a purely local composition free of any
+ * remote-cache machinery.
+ *
+ * @since 0.1.0
+ * @category constructors
+ */
+export declare const makeLocal: () => Service;
+/**
+ * Provides the single-tier (no shared artifact store) protocol.
+ *
+ * @since 0.1.0
+ * @category layers
+ */
+export declare const layerLocal: Layer.Layer<ArtifactSync>;
+/**
+ * Builds the two-tier protocol over a local and a shared artifact store.
+ *
+ * @since 0.1.0
+ * @category constructors
+ */
+export declare const make: (options: {
+    readonly local: ArtifactStore.Service;
+    readonly remote: ArtifactStore.Service;
+    /**
+     * Defaults to the policy the shared tier declares, and to `all` for a tier
+     * that declares none: every referenced artifact is fetched at hydrate.
+     */
+    readonly downloadPolicy?: DownloadPolicy | undefined;
+}) => Service;
+/**
+ * Provides the two-tier protocol, taking the local store from the
+ * `ArtifactStore` tag and the shared store from the supplied effect.
+ *
+ * The two tiers cannot both be layers: they inhabit the same tag, so composing
+ * them would shadow one with the other.
+ *
+ * @since 0.1.0
+ * @category layers
+ */
+export declare const layer: <E, R>(remote: Effect.Effect<ArtifactStore.Service, E, R>, options?: {
+    readonly downloadPolicy?: DownloadPolicy | undefined;
+} | undefined) => Layer.Layer<ArtifactSync, E, ArtifactStore.ArtifactStore | R>;
+export {};
+//# sourceMappingURL=ArtifactSync.d.ts.map

package/dist/cjs/ArtifactSync.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ArtifactSync.d.ts","sourceRoot":"","sources":["../../src/ArtifactSync.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,KAAK,aAAa,MAAM,iCAAiC,CAAA;AAChE,OAAO,KAAK,eAAe,MAAM,mCAAmC,CAAA;AACpE,OAAO,KAAK,OAAO,MAAM,gBAAgB,CAAA;AACzC,OAAO,KAAK,KAAK,MAAM,MAAM,eAAe,CAAA;AAC5C,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAA;AACrC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;;;;;;;AAEvC;;;;;;;;;GASG;AACH,qBAAa,yBAA0B,SAAQ,8BAc9C;CAAG;AAEJ;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,MAAM,cAAc,GAAG,eAAe,CAAC,cAAc,CAAA;AAE3D;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,OAAO;IACtB;;;;OAIG;IACH,QAAQ,CAAC,OAAO,EAAE,CAChB,OAAO,EAAE,aAAa,CAAC,MAAM,CAAC,KAC3B,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,yBAAyB,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;IAClE;;;;;;;;OAQG;IACH,QAAQ,CAAC,OAAO,EAAE,CAChB,OAAO,EAAE,aAAa,CAAC,MAAM,CAAC,KAC3B,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;CAClD;;AAED;;;;;GAKG;AACH,qBAAa,YAAa,SAAQ,iBAA6E;CAAG;AAElH;;;;;;;;;;GAUG;AACH,eAAO,MAAM,SAAS,QAAO,OAG3B,CAAA;AAEF;;;;;GAKG;AACH,eAAO,MAAM,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,YAAY,CAA4C,CAAA;AAE7F;;;;;GAKG;AACH,eAAO,MAAM,IAAI,GAAI,SAAS;IAC5B,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,OAAO,CAAA;IACrC,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAA;IACtC;;;OAGG;IACH,QAAQ,CAAC,cAAc,CAAC,EAAE,cAAc,GAAG,SAAS,CAAA;CACrD,KAAG,OAmFH,CAAA;AAED;;;;;;;;;GASG;AACH,eAAO,MAAM,KAAK,GAAI,CAAC,EAAE,CAAC,EACxB,QAAQ,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,EAClD,UAAU;IAAE,QAAQ,CAAC,cAAc,CAAC,EAAE,cAAc,GAAG,SAAS,CAAA;CAAE,GAAG,SAAS,KAC7E,KAAK,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,EAAE,aAAa,CAAC,aAAa,GAAG,CAAC,CAW5D,CAAA"}

package/dist/cjs/ArtifactSync.js

@@ -0,0 +1,144 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var ArtifactSync_exports = {};
+__export(ArtifactSync_exports, {
+  ArtifactPublicationFailed: () => ArtifactPublicationFailed,
+  ArtifactSync: () => ArtifactSync,
+  layer: () => layer,
+  layerLocal: () => layerLocal,
+  make: () => make,
+  makeLocal: () => makeLocal
+});
+module.exports = __toCommonJS(ArtifactSync_exports);
+var ArtifactStore = __toESM(require("@smthrs/artifacts/ArtifactStore"), 1);
+var RemoteArtifacts = __toESM(require("@smthrs/artifacts/RemoteArtifacts"), 1);
+var Context = __toESM(require("effect/Context"), 1);
+var Effect = __toESM(require("effect/Effect"), 1);
+var Layer = __toESM(require("effect/Layer"), 1);
+var Schema = __toESM(require("effect/Schema"), 1);
+class ArtifactPublicationFailed extends Schema.TaggedError()(
+  "@smthrs/engine-store/ArtifactPublicationFailed",
+  {
+    code: Schema.Literal("artifact_publication_failed"),
+    digests: Schema.Array(Schema.String),
+    message: Schema.String,
+    cause: Schema.optional(
+      Schema.Union([
+        ArtifactStore.ArtifactStoreError,
+        ArtifactStore.ArtifactMissing,
+        ArtifactStore.ArtifactCorruption
+      ])
+    )
+  }
+) {
+}
+class ArtifactSync extends Context.Service()("@smthrs/engine-store/ArtifactSync") {
+}
+const makeLocal = () => ({
+  publish: Effect.fn("ArtifactSync.publish")(() => Effect.void),
+  hydrate: Effect.fn("ArtifactSync.hydrate")((digests) => Effect.succeed(digests.length === 0))
+});
+const layerLocal = Layer.succeed(ArtifactSync)(makeLocal());
+const make = (options) => {
+  const { local, remote } = options;
+  const downloadPolicy = options.downloadPolicy ?? RemoteArtifacts.downloadPolicyOf(remote) ?? "all";
+  const publicationFailed = (digests, message, cause) => new ArtifactPublicationFailed({ code: "artifact_publication_failed", digests, message, cause });
+  return {
+    publish: Effect.fn("ArtifactSync.publish")(function* (digests) {
+      if (digests.length === 0) return;
+      const missing = yield* remote.findMissing(digests).pipe(
+        Effect.mapError((cause) => publicationFailed(digests, "the shared tier refused a probe", cause))
+      );
+      for (const digest of missing) {
+        const bytes = yield* local.get(digest).pipe(
+          Effect.mapError(
+            (cause) => (
+              // The blob is referenced by evidence this host just produced, so
+              // it should be in this host's own store. If it is not, the
+              // evidence is not publishable — and saying so is far better than
+              // publishing an entry nobody can replay.
+              publicationFailed([digest], "this host cannot read the artifact it recorded", cause)
+            )
+          )
+        );
+        yield* remote.put(bytes).pipe(
+          Effect.mapError((cause) => publicationFailed([digest], "the shared tier refused an upload", cause))
+        );
+      }
+      if (missing.length === 0) return;
+      const stillMissing = yield* remote.findMissing(missing).pipe(
+        Effect.mapError((cause) => publicationFailed(missing, "the shared tier refused a probe", cause))
+      );
+      if (stillMissing.length > 0) {
+        return yield* Effect.fail(
+          publicationFailed(stillMissing, "the shared tier did not retain the uploaded artifacts")
+        );
+      }
+    }),
+    hydrate: Effect.fn("ArtifactSync.hydrate")(function* (digests) {
+      const fallBack = (message, digests2) => (cause) => Effect.logWarning(`artifact hydration falls back to re-execution: ${message}: ${digests2.join(", ")}`).pipe(
+        Effect.annotateLogs({ reason: cause.message })
+      );
+      const missing = yield* local.findMissing(digests).pipe(
+        Effect.tapError(fallBack("the local tier refused a probe", digests)),
+        Effect.option
+      );
+      if (missing._tag === "None") return false;
+      if (downloadPolicy !== "all") {
+        if (missing.value.length === 0) return true;
+        const elsewhere = yield* remote.findMissing(missing.value).pipe(
+          Effect.tapError(fallBack("the shared tier refused a probe", missing.value)),
+          Effect.option
+        );
+        return elsewhere._tag === "Some" && elsewhere.value.length === 0;
+      }
+      for (const digest of missing.value) {
+        const written = yield* remote.get(digest).pipe(
+          Effect.flatMap((bytes) => local.put(bytes)),
+          Effect.tapError(fallBack("the artifact could not be fetched", [digest])),
+          Effect.option
+        );
+        if (written._tag === "None") return false;
+      }
+      return true;
+    })
+  };
+};
+const layer = (remote, options) => Layer.effect(ArtifactSync)(
+  Effect.gen(function* () {
+    const local = yield* ArtifactStore.ArtifactStore;
+    const shared = yield* remote;
+    return make({
+      local,
+      remote: shared,
+      ...options?.downloadPolicy === void 0 ? {} : { downloadPolicy: options.downloadPolicy }
+    });
+  })
+);
+//# sourceMappingURL=ArtifactSync.js.map

package/dist/cjs/ArtifactSync.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/ArtifactSync.ts"],
+  "sourcesContent": ["/**\n * The two-tier artifact protocol the step cache rides on: publish before the\n * entry, fetch on demand after it.\n *\n * This is the seam that makes a shared cache safe. A cache entry references\n * its large outputs by digest, so an entry that becomes observable while one\n * of those artifacts is still missing hands the next machine a hit it cannot\n * materialize. Bazel states the constraint at\n * `reference/bazel/.../remote/UploadManifest.java:630-633` \u2014 \"action results\n * may fail to validate server-side if they are accessed before all blobs they\n * refer to are present\" \u2014 and enforces it by ordering:\n * `findMissingDigests` \u2192 upload the missing \u2192 upload the `ActionResult` LAST.\n * {@link Service.publish} is that ordering, and `ActionPersistence` calls it\n * before `CacheStore.put`.\n *\n * The read direction is the same protocol in reverse, and how eagerly it runs\n * is the {@link DownloadPolicy}, declared on the shared tier itself\n * (`RemoteArtifacts.Options.downloadPolicy`) and split across two seams.\n * `all` fetches every referenced artifact into this host's store while\n * admitting the replay. `toplevel` and `minimal` fetch nothing here: one\n * batched probe establishes that the shared tier can serve them and the\n * transfer is left to the first read through `CombinedArtifacts`, which then\n * writes the blob back under `toplevel` and does not under `minimal`. This is\n * Bazel's `RemoteOutputChecker` dial\n * (`--remote_download_{all,toplevel,minimal}`) split the way our two seams\n * split: what a replay prefetches is this module's decision, what a read\n * materializes is the combined store's.\n *\n * Governing design: `packages/smithers/flows/artifacts/docs/api.md`.\n *\n * @since 0.1.0\n */\n\nimport * as ArtifactStore from \"@smthrs/artifacts/ArtifactStore\"\nimport * as RemoteArtifacts from \"@smthrs/artifacts/RemoteArtifacts\"\nimport * as Context from \"effect/Context\"\nimport type * as Crypto from \"effect/Crypto\"\nimport * as Effect from \"effect/Effect\"\nimport * as Layer from \"effect/Layer\"\nimport * as Schema from \"effect/Schema\"\n\n/**\n * A referenced artifact could not be made durable in the shared tier, so the\n * cache entry that references it must not be published.\n *\n * Failing here is the point: silently publishing the entry anyway is exactly\n * the state the REAPI ordering constraint exists to prevent.\n *\n * @since 0.1.0\n * @category errors\n */\nexport class ArtifactPublicationFailed extends Schema.TaggedError<ArtifactPublicationFailed>()(\n  \"@smthrs/engine-store/ArtifactPublicationFailed\",\n  {\n    code: Schema.Literal(\"artifact_publication_failed\"),\n    digests: Schema.Array(Schema.String),\n    message: Schema.String,\n    cause: Schema.optional(\n      Schema.Union([\n        ArtifactStore.ArtifactStoreError,\n        ArtifactStore.ArtifactMissing,\n        ArtifactStore.ArtifactCorruption\n      ])\n    )\n  }\n) {}\n\n/**\n * How eagerly a replay materializes the artifacts it references.\n *\n * `all` downloads every referenced artifact into this host's local store while\n * the replay is admitted, so every later read is local and a shared tier that\n * goes down afterwards costs nothing. `minimal` downloads none of them: it\n * establishes that each digest is resolvable \u2014 locally, or from the shared tier\n * \u2014 and lets the first actual read fetch the bytes.\n *\n * `minimal` is only sound when the store the replay reads through can reach the\n * shared tier, which means `CombinedArtifacts` with the same remote tier. Under\n * a purely local `ArtifactStore` an admitted `minimal` replay would later read\n * an artifact this host never fetched.\n *\n * @since 0.1.0\n * @category models\n */\nexport type DownloadPolicy = RemoteArtifacts.DownloadPolicy\n\n/**\n * The two directions of the artifact protocol: {@link Service.publish} before a\n * cache entry becomes observable, {@link Service.hydrate} when a replay needs\n * bytes this host does not have.\n *\n * The asymmetry in their error types is the design: publishing may fail,\n * because publishing an entry whose artifacts are missing is the unsafe state;\n * hydrating never fails, because a shared tier that is down should cost a run\n * a cache hit, not the run itself.\n *\n * @since 0.1.0\n * @category services\n */\nexport interface Service {\n  /**\n   * Makes every digest durable in the shared tier, in REAPI order:\n   * `findMissing`, upload what is missing, then confirm nothing is missing\n   * any more. Returns only once the shared tier can serve all of them.\n   */\n  readonly publish: (\n    digests: ReadonlyArray<string>\n  ) => Effect.Effect<void, ArtifactPublicationFailed, Crypto.Crypto>\n  /**\n   * Fetches the digests this host is missing from the shared tier and writes\n   * them into the local store, reporting whether every digest is now locally\n   * resolvable.\n   *\n   * Best-effort by construction: `false` means \"replay still cannot be\n   * satisfied\", which the caller turns into a real execution. A shared tier\n   * that is down must never fail a run that can simply do the work.\n   */\n  readonly hydrate: (\n    digests: ReadonlyArray<string>\n  ) => Effect.Effect<boolean, never, Crypto.Crypto>\n}\n\n/**\n * Service tag for the two-tier artifact protocol.\n *\n * @since 0.1.0\n * @category services\n */\nexport class ArtifactSync extends Context.Service<ArtifactSync, Service>()(\"@smthrs/engine-store/ArtifactSync\") {}\n\n/**\n * The single-tier implementation: there is no shared artifact store, so there\n * is nothing to publish and nothing to fetch.\n *\n * This is the default `ActionPersistence` falls back to when the tag is\n * absent, which is what keeps a purely local composition free of any\n * remote-cache machinery.\n *\n * @since 0.1.0\n * @category constructors\n */\nexport const makeLocal = (): Service => ({\n  publish: Effect.fn(\"ArtifactSync.publish\")(() => Effect.void),\n  hydrate: Effect.fn(\"ArtifactSync.hydrate\")((digests) => Effect.succeed(digests.length === 0))\n})\n\n/**\n * Provides the single-tier (no shared artifact store) protocol.\n *\n * @since 0.1.0\n * @category layers\n */\nexport const layerLocal: Layer.Layer<ArtifactSync> = Layer.succeed(ArtifactSync)(makeLocal())\n\n/**\n * Builds the two-tier protocol over a local and a shared artifact store.\n *\n * @since 0.1.0\n * @category constructors\n */\nexport const make = (options: {\n  readonly local: ArtifactStore.Service\n  readonly remote: ArtifactStore.Service\n  /**\n   * Defaults to the policy the shared tier declares, and to `all` for a tier\n   * that declares none: every referenced artifact is fetched at hydrate.\n   */\n  readonly downloadPolicy?: DownloadPolicy | undefined\n}): Service => {\n  const { local, remote } = options\n  const downloadPolicy = options.downloadPolicy ?? RemoteArtifacts.downloadPolicyOf(remote) ?? \"all\"\n  const publicationFailed = (\n    digests: ReadonlyArray<string>,\n    message: string,\n    cause?: ArtifactStore.ArtifactStoreError | ArtifactStore.ArtifactMissing | ArtifactStore.ArtifactCorruption\n  ) => new ArtifactPublicationFailed({ code: \"artifact_publication_failed\", digests, message, cause })\n  return {\n    publish: Effect.fn(\"ArtifactSync.publish\")(function*(digests) {\n      if (digests.length === 0) return\n      const missing = yield* remote.findMissing(digests).pipe(\n        Effect.mapError((cause) => publicationFailed(digests, \"the shared tier refused a probe\", cause))\n      )\n      for (const digest of missing) {\n        const bytes = yield* local.get(digest).pipe(\n          Effect.mapError((cause) =>\n            // The blob is referenced by evidence this host just produced, so\n            // it should be in this host's own store. If it is not, the\n            // evidence is not publishable \u2014 and saying so is far better than\n            // publishing an entry nobody can replay.\n            publicationFailed([digest], \"this host cannot read the artifact it recorded\", cause)\n          )\n        )\n        yield* remote.put(bytes).pipe(\n          Effect.mapError((cause) => publicationFailed([digest], \"the shared tier refused an upload\", cause))\n        )\n      }\n      if (missing.length === 0) return\n      // The confirming re-probe is not paranoia: a tier that accepted the\n      // upload but did not durably store it is exactly the state that makes a\n      // published entry unreplayable, and it is cheap to rule out \u2014 one\n      // batched round trip over the set we just wrote.\n      const stillMissing = yield* remote.findMissing(missing).pipe(\n        Effect.mapError((cause) => publicationFailed(missing, \"the shared tier refused a probe\", cause))\n      )\n      if (stillMissing.length > 0) {\n        return yield* Effect.fail(\n          publicationFailed(stillMissing, \"the shared tier did not retain the uploaded artifacts\")\n        )\n      }\n    }),\n    hydrate: Effect.fn(\"ArtifactSync.hydrate\")(function*(digests) {\n      // Every refusal below falls back to re-executing the step, which is\n      // correct and silent. The warning is the operator's only signal that a\n      // tier stopped serving replays.\n      const fallBack = (message: string, digests: ReadonlyArray<string>) => (cause: { readonly message: string }) =>\n        Effect.logWarning(`artifact hydration falls back to re-execution: ${message}: ${digests.join(\", \")}`).pipe(\n          Effect.annotateLogs({ reason: cause.message })\n        )\n      const missing = yield* local.findMissing(digests).pipe(\n        Effect.tapError(fallBack(\"the local tier refused a probe\", digests)),\n        Effect.option\n      )\n      if (missing._tag === \"None\") return false\n      if (downloadPolicy !== \"all\") {\n        if (missing.value.length === 0) return true\n        // One batched probe, no body transfer: the answer these policies owe\n        // the caller is \"can this replay be satisfied\", and `findMissing`\n        // answers exactly that. A tier that refuses the probe is\n        // indistinguishable from one that holds nothing, so the replay is\n        // refused either way.\n        const elsewhere = yield* remote.findMissing(missing.value).pipe(\n          Effect.tapError(fallBack(\"the shared tier refused a probe\", missing.value)),\n          Effect.option\n        )\n        return elsewhere._tag === \"Some\" && elsewhere.value.length === 0\n      }\n      for (const digest of missing.value) {\n        // Read-through, one artifact at a time and only on demand: the remote\n        // read is digest-verified by `RemoteArtifacts`, and the local `put`\n        // re-addresses the bytes, so a tier serving wrong content cannot get\n        // them written into this workspace.\n        const written = yield* remote.get(digest).pipe(\n          Effect.flatMap((bytes) => local.put(bytes)),\n          Effect.tapError(fallBack(\"the artifact could not be fetched\", [digest])),\n          Effect.option\n        )\n        if (written._tag === \"None\") return false\n      }\n      return true\n    })\n  }\n}\n\n/**\n * Provides the two-tier protocol, taking the local store from the\n * `ArtifactStore` tag and the shared store from the supplied effect.\n *\n * The two tiers cannot both be layers: they inhabit the same tag, so composing\n * them would shadow one with the other.\n *\n * @since 0.1.0\n * @category layers\n */\nexport const layer = <E, R>(\n  remote: Effect.Effect<ArtifactStore.Service, E, R>,\n  options?: { readonly downloadPolicy?: DownloadPolicy | undefined } | undefined\n): Layer.Layer<ArtifactSync, E, ArtifactStore.ArtifactStore | R> =>\n  Layer.effect(ArtifactSync)(\n    Effect.gen(function*() {\n      const local = yield* ArtifactStore.ArtifactStore\n      const shared = yield* remote\n      return make({\n        local,\n        remote: shared,\n        ...(options?.downloadPolicy === undefined ? {} : { downloadPolicy: options.downloadPolicy })\n      })\n    })\n  )\n"],
+  "mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAiCA,oBAA+B;AAC/B,sBAAiC;AACjC,cAAyB;AAEzB,aAAwB;AACxB,YAAuB;AACvB,aAAwB;AAYjB,MAAM,kCAAkC,OAAO,YAAuC;AAAA,EAC3F;AAAA,EACA;AAAA,IACE,MAAM,OAAO,QAAQ,6BAA6B;AAAA,IAClD,SAAS,OAAO,MAAM,OAAO,MAAM;AAAA,IACnC,SAAS,OAAO;AAAA,IAChB,OAAO,OAAO;AAAA,MACZ,OAAO,MAAM;AAAA,QACX,cAAc;AAAA,QACd,cAAc;AAAA,QACd,cAAc;AAAA,MAChB,CAAC;AAAA,IACH;AAAA,EACF;AACF,EAAE;AAAC;AA+DI,MAAM,qBAAqB,QAAQ,QAA+B,EAAE,mCAAmC,EAAE;AAAC;AAa1G,MAAM,YAAY,OAAgB;AAAA,EACvC,SAAS,OAAO,GAAG,sBAAsB,EAAE,MAAM,OAAO,IAAI;AAAA,EAC5D,SAAS,OAAO,GAAG,sBAAsB,EAAE,CAAC,YAAY,OAAO,QAAQ,QAAQ,WAAW,CAAC,CAAC;AAC9F;AAQO,MAAM,aAAwC,MAAM,QAAQ,YAAY,EAAE,UAAU,CAAC;AAQrF,MAAM,OAAO,CAAC,YAQN;AACb,QAAM,EAAE,OAAO,OAAO,IAAI;AAC1B,QAAM,iBAAiB,QAAQ,kBAAkB,gBAAgB,iBAAiB,MAAM,KAAK;AAC7F,QAAM,oBAAoB,CACxB,SACA,SACA,UACG,IAAI,0BAA0B,EAAE,MAAM,+BAA+B,SAAS,SAAS,MAAM,CAAC;AACnG,SAAO;AAAA,IACL,SAAS,OAAO,GAAG,sBAAsB,EAAE,WAAU,SAAS;AAC5D,UAAI,QAAQ,WAAW,EAAG;AAC1B,YAAM,UAAU,OAAO,OAAO,YAAY,OAAO,EAAE;AAAA,QACjD,OAAO,SAAS,CAAC,UAAU,kBAAkB,SAAS,mCAAmC,KAAK,CAAC;AAAA,MACjG;AACA,iBAAW,UAAU,SAAS;AAC5B,cAAM,QAAQ,OAAO,MAAM,IAAI,MAAM,EAAE;AAAA,UACrC,OAAO;AAAA,YAAS,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA,cAKf,kBAAkB,CAAC,MAAM,GAAG,kDAAkD,KAAK;AAAA;AAAA,UACrF;AAAA,QACF;AACA,eAAO,OAAO,IAAI,KAAK,EAAE;AAAA,UACvB,OAAO,SAAS,CAAC,UAAU,kBAAkB,CAAC,MAAM,GAAG,qCAAqC,KAAK,CAAC;AAAA,QACpG;AAAA,MACF;AACA,UAAI,QAAQ,WAAW,EAAG;AAK1B,YAAM,eAAe,OAAO,OAAO,YAAY,OAAO,EAAE;AAAA,QACtD,OAAO,SAAS,CAAC,UAAU,kBAAkB,SAAS,mCAAmC,KAAK,CAAC;AAAA,MACjG;AACA,UAAI,aAAa,SAAS,GAAG;AAC3B,eAAO,OAAO,OAAO;AAAA,UACnB,kBAAkB,cAAc,uDAAuD;AAAA,QACzF;AAAA,MACF;AAAA,IACF,CAAC;AAAA,IACD,SAAS,OAAO,GAAG,sBAAsB,EAAE,WAAU,SAAS;AAI5D,YAAM,WAAW,CAAC,SAAiBA,aAAmC,CAAC,UACrE,OAAO,WAAW,kDAAkD,OAAO,KAAKA,SAAQ,KAAK,IAAI,CAAC,EAAE,EAAE;AAAA,QACpG,OAAO,aAAa,EAAE,QAAQ,MAAM,QAAQ,CAAC;AAAA,MAC/C;AACF,YAAM,UAAU,OAAO,MAAM,YAAY,OAAO,EAAE;AAAA,QAChD,OAAO,SAAS,SAAS,kCAAkC,OAAO,CAAC;AAAA,QACnE,OAAO;AAAA,MACT;AACA,UAAI,QAAQ,SAAS,OAAQ,QAAO;AACpC,UAAI,mBAAmB,OAAO;AAC5B,YAAI,QAAQ,MAAM,WAAW,EAAG,QAAO;AAMvC,cAAM,YAAY,OAAO,OAAO,YAAY,QAAQ,KAAK,EAAE;AAAA,UACzD,OAAO,SAAS,SAAS,mCAAmC,QAAQ,KAAK,CAAC;AAAA,UAC1E,OAAO;AAAA,QACT;AACA,eAAO,UAAU,SAAS,UAAU,UAAU,MAAM,WAAW;AAAA,MACjE;AACA,iBAAW,UAAU,QAAQ,OAAO;AAKlC,cAAM,UAAU,OAAO,OAAO,IAAI,MAAM,EAAE;AAAA,UACxC,OAAO,QAAQ,CAAC,UAAU,MAAM,IAAI,KAAK,CAAC;AAAA,UAC1C,OAAO,SAAS,SAAS,qCAAqC,CAAC,MAAM,CAAC,CAAC;AAAA,UACvE,OAAO;AAAA,QACT;AACA,YAAI,QAAQ,SAAS,OAAQ,QAAO;AAAA,MACtC;AACA,aAAO;AAAA,IACT,CAAC;AAAA,EACH;AACF;AAYO,MAAM,QAAQ,CACnB,QACA,YAEA,MAAM,OAAO,YAAY;AAAA,EACvB,OAAO,IAAI,aAAY;AACrB,UAAM,QAAQ,OAAO,cAAc;AACnC,UAAM,SAAS,OAAO;AACtB,WAAO,KAAK;AAAA,MACV;AAAA,MACA,QAAQ;AAAA,MACR,GAAI,SAAS,mBAAmB,SAAY,CAAC,IAAI,EAAE,gBAAgB,QAAQ,eAAe;AAAA,IAC5F,CAAC;AAAA,EACH,CAAC;AACH;",
+  "names": ["digests"]
+}