@smthrs/artifacts 0.0.0-stage → 1.0.0-rc.12

package/package.json

@@ -1,6 +1,182 @@
 {
   "name": "@smthrs/artifacts",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "type": "module",
+  "version": "1.0.0-rc.12",
+  "license": "MIT",
+  "smthrs": {
+    "group": "engine"
+  },
+  "description": "Effect service for the Smithers content-addressed artifact store, local and remote",
+  "homepage": "https://github.com/smithersai/smithers",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/smithersai/smithers.git",
+    "directory": "packages/smithers/flows/artifacts"
+  },
+  "bugs": {
+    "url": "https://github.com/smithersai/smithers/issues"
+  },
+  "tags": [
+    "typescript",
+    "effect",
+    "cache",
+    "content-addressed"
+  ],
+  "keywords": [
+    "typescript",
+    "effect",
+    "cache",
+    "content-addressed"
+  ],
+  "engines": {
+    "node": ">=26.4.0"
+  },
+  "sideEffects": [],
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "import": {
+        "types": "./dist/esm/index.d.ts",
+        "default": "./dist/esm/index.js"
+      },
+      "require": {
+        "types": "./dist/cjs/index.d.ts",
+        "default": "./dist/cjs/index.js"
+      }
+    },
+    "./internal/*": null,
+    "./*/index": null,
+    "./ArtifactBackupLease": {
+      "import": {
+        "types": "./dist/esm/ArtifactBackupLease.d.ts",
+        "default": "./dist/esm/ArtifactBackupLease.js"
+      },
+      "require": {
+        "types": "./dist/cjs/ArtifactBackupLease.d.ts",
+        "default": "./dist/cjs/ArtifactBackupLease.js"
+      }
+    },
+    "./ArtifactStore": {
+      "import": {
+        "types": "./dist/esm/ArtifactStore.d.ts",
+        "default": "./dist/esm/ArtifactStore.js"
+      },
+      "require": {
+        "types": "./dist/cjs/ArtifactStore.d.ts",
+        "default": "./dist/cjs/ArtifactStore.js"
+      }
+    },
+    "./ArtifactStoreMetrics": {
+      "import": {
+        "types": "./dist/esm/ArtifactStoreMetrics.d.ts",
+        "default": "./dist/esm/ArtifactStoreMetrics.js"
+      },
+      "require": {
+        "types": "./dist/cjs/ArtifactStoreMetrics.d.ts",
+        "default": "./dist/cjs/ArtifactStoreMetrics.js"
+      }
+    },
+    "./ArtifactSweep": {
+      "import": {
+        "types": "./dist/esm/ArtifactSweep.d.ts",
+        "default": "./dist/esm/ArtifactSweep.js"
+      },
+      "require": {
+        "types": "./dist/cjs/ArtifactSweep.d.ts",
+        "default": "./dist/cjs/ArtifactSweep.js"
+      }
+    },
+    "./CombinedArtifacts": {
+      "import": {
+        "types": "./dist/esm/CombinedArtifacts.d.ts",
+        "default": "./dist/esm/CombinedArtifacts.js"
+      },
+      "require": {
+        "types": "./dist/cjs/CombinedArtifacts.d.ts",
+        "default": "./dist/cjs/CombinedArtifacts.js"
+      }
+    },
+    "./FileLease": {
+      "import": {
+        "types": "./dist/esm/FileLease.d.ts",
+        "default": "./dist/esm/FileLease.js"
+      },
+      "require": {
+        "types": "./dist/cjs/FileLease.d.ts",
+        "default": "./dist/cjs/FileLease.js"
+      }
+    },
+    "./RemoteArtifacts": {
+      "import": {
+        "types": "./dist/esm/RemoteArtifacts.d.ts",
+        "default": "./dist/esm/RemoteArtifacts.js"
+      },
+      "require": {
+        "types": "./dist/cjs/RemoteArtifacts.d.ts",
+        "default": "./dist/cjs/RemoteArtifacts.js"
+      }
+    },
+    "./index": {
+      "import": {
+        "types": "./dist/esm/index.d.ts",
+        "default": "./dist/esm/index.js"
+      },
+      "require": {
+        "types": "./dist/cjs/index.d.ts",
+        "default": "./dist/cjs/index.js"
+      }
+    }
+  },
+  "files": [
+    "src/**/*.ts",
+    "dist/**/*.js",
+    "dist/**/*.js.map",
+    "dist/**/*.d.ts",
+    "dist/**/*.d.ts.map",
+    "dist/**/package.json",
+    "LICENSE",
+    "README.md",
+    "CHANGELOG.md",
+    "docs/"
+  ],
+  "publishConfig": {
+    "access": "public",
+    "provenance": true,
+    "tag": "next"
+  },
+  "dependencies": {
+    "@smthrs/crypto": "1.0.0-rc.12"
+  },
+  "devDependencies": {
+    "@effect/language-service": "0.87.2",
+    "@effect/platform-node": "4.0.4",
+    "@effect/vitest": "4.0.4",
+    "@eslint/js": "9.39.5",
+    "@types/node": "26.4.1",
+    "@vitest/coverage-v8": "5.0.0",
+    "dprint": "0.57.1",
+    "effect": "4.0.4",
+    "esbuild": "0.28.2",
+    "eslint": "9.39.5",
+    "eslint-import-resolver-typescript": "4.4.5",
+    "eslint-plugin-import": "2.32.0",
+    "eslint-plugin-jsdoc": "64.3.4",
+    "eslint-plugin-unicorn": "65.0.0",
+    "madge": "8.0.0",
+    "typescript": "5.9.3",
+    "typescript-eslint": "8.69.0",
+    "vitest": "5.0.0"
+  },
+  "peerDependencies": {
+    "effect": "^4.0.4"
+  },
+  "scripts": {
+    "lint": "eslint src --max-warnings=0 && dprint check",
+    "format": "dprint fmt",
+    "build": "node scripts/build.mjs",
+    "check": "tsc -b tsconfig.json && tsc -p tsconfig.test.json --noEmit",
+    "circular": "node scripts/circular.mjs",
+    "test": "vitest",
+    "coverage": "vitest --coverage"
+  }
 }

package/src/ArtifactBackupLease.ts

@@ -0,0 +1,223 @@
+/**
+ * Cross-process exclusion between a filesystem artifact backup and garbage
+ * collection.
+ *
+ * A backup holds one heartbeat-backed marker for the duration of its database
+ * snapshot and blob copy. Sweep deletion checks that marker while holding the
+ * same short-lived global gate. Publication remains unconstrained: new blobs
+ * may appear during a backup, but a blob already referenced by the frozen
+ * database cannot disappear underneath it.
+ *
+ * @since 1.0.0-rc.0
+ */
+
+import * as Clock from "effect/Clock"
+import * as Effect from "effect/Effect"
+import type * as FileSystem from "effect/FileSystem"
+import * as Option from "effect/Option"
+import * as PlatformError from "effect/PlatformError"
+import * as Random from "effect/Random"
+import * as ArtifactLocks from "./internal/ArtifactLocks.ts"
+
+const markerName = ".backup-lease"
+const gateName = ".backup-lease-gate"
+const heartbeatEvery = "10 seconds"
+const staleAfterMs = 60_000
+const acquireWithin = "2 minutes"
+const retryEvery = "25 millis"
+
+const isReason = (cause: unknown, tag: PlatformError.SystemErrorTag): boolean =>
+  cause instanceof PlatformError.PlatformError && cause.reason._tag === tag
+
+const token = Effect.gen(function*() {
+  const first = yield* Random.nextIntBetween(0, Number.MAX_SAFE_INTEGER, { halfOpen: true })
+  const second = yield* Random.nextIntBetween(0, Number.MAX_SAFE_INTEGER, { halfOpen: true })
+  return `${first.toString(36)}-${second.toString(36)}-${yield* Clock.currentTimeMillis}`
+})
+
+const gate = <A, E, R, E2>(
+  fs: FileSystem.FileSystem,
+  directory: string,
+  effect: Effect.Effect<A, E, R>,
+  failure: (cause: unknown) => E2
+): Effect.Effect<A, E | E2, R> => ArtifactLocks.withDigest(fs, directory, gateName, effect, failure)
+
+const activeMarker = <E>(
+  fs: FileSystem.FileSystem,
+  directory: string,
+  failure: (cause: unknown) => E
+): Effect.Effect<boolean, E> =>
+  Effect.gen(function*() {
+    const path = `${directory}/${markerName}`
+    const info = yield* fs.stat(path).pipe(
+      Effect.map(Option.some),
+      Effect.catch((cause): Effect.Effect<Option.Option<FileSystem.File.Info>, E> =>
+        isReason(cause, "NotFound") ? Effect.succeed(Option.none()) : Effect.fail(failure(cause))
+      )
+    )
+    if (Option.isNone(info)) return false
+    const modified = Option.getOrUndefined(info.value.mtime)
+    const now = yield* Clock.currentTimeMillis
+    if (modified === undefined || now - modified.getTime() <= staleAfterMs) return true
+    yield* fs.remove(path).pipe(
+      Effect.catch((cause): Effect.Effect<void, E> =>
+        isReason(cause, "NotFound") ? Effect.void : Effect.fail(failure(cause))
+      )
+    )
+    return false
+  })
+
+/**
+ * Runs `effect` while sweep deletion is fenced across every process using
+ * this objects directory. A crashed holder is recovered after its heartbeat
+ * becomes stale.
+ *
+ * @category combinators
+ * @since 1.0.0-rc.0
+ */
+export const withLease = <A, E, R, E2>(
+  fs: FileSystem.FileSystem,
+  directory: string,
+  effect: Effect.Effect<A, E, R>,
+  failure: (cause: unknown) => E2
+): Effect.Effect<A, E | E2, R> =>
+  Effect.gen(function*() {
+    const owner = yield* token
+    const marker = `${directory}/${markerName}`
+    yield* fs.makeDirectory(directory, { recursive: true }).pipe(Effect.mapError(failure))
+
+    let ownsMarker = false
+    const acquire = Effect.gen(function*() {
+      while (true) {
+        const acquired = yield* gate(
+          fs,
+          directory,
+          Effect.gen(function*() {
+            if (yield* activeMarker(fs, directory, failure)) return false
+            return yield* Effect.uninterruptible(
+              fs.writeFileString(marker, owner, { flag: "wx", mode: 0o600 }).pipe(
+                Effect.andThen(Effect.sync(() => {
+                  ownsMarker = true
+                  return true
+                })),
+                Effect.catch((cause): Effect.Effect<boolean, E2> =>
+                  isReason(cause, "AlreadyExists") ? Effect.succeed(false) : Effect.fail(failure(cause))
+                )
+              )
+            )
+          }),
+          failure
+        )
+        if (acquired) return
+        yield* Effect.sleep(retryEvery)
+      }
+    }).pipe(
+      Effect.timeout(acquireWithin),
+      Effect.catchTag("TimeoutError", (cause) => Effect.fail(failure(cause)))
+    )
+
+    // Install marker cleanup before acquisition, including gate release. The
+    // marker write and ownership bookkeeping complete in one interruption mask.
+    return yield* Effect.acquireUseRelease(
+      Effect.void,
+      () =>
+        acquire.pipe(Effect.andThen(
+          Effect.scoped(Effect.gen(function*() {
+            yield* Effect.forkScoped(
+              Effect.gen(function*() {
+                while (true) {
+                  yield* Effect.sleep(heartbeatEvery)
+                  const reclaimed = yield* gate(
+                    fs,
+                    directory,
+                    fs.readFileString(marker).pipe(
+                      Effect.map(Option.some),
+                      Effect.catch((cause) =>
+                        isReason(cause, "NotFound") ? Effect.succeed(Option.none<string>()) : Effect.fail(cause)
+                      ),
+                      Effect.flatMap((found) => {
+                        // A missing or foreign marker means a sweep judged this
+                        // lease stale and reaped it: deletion is no longer
+                        // fenced while the backup still copies. The sibling
+                        // `internal/ArtifactLocks.ts` heartbeat reports the same
+                        // loss the same way. A reaped marker is never handed
+                        // back, so the loss is reported once and the heartbeat
+                        // ends instead of repeating it every tick.
+                        if (Option.getOrUndefined(found) !== owner) {
+                          return Effect.as(
+                            Effect.logWarning(
+                              "Artifact backup lease was reclaimed while its backup was still running"
+                            ),
+                            true
+                          )
+                        }
+                        return Effect.flatMap(Clock.currentTimeMillis, (now) => {
+                          const timestamp = new Date(now)
+                          return Effect.as(fs.utimes(marker, timestamp, timestamp), false)
+                        })
+                      }),
+                      Effect.mapError(failure)
+                    ),
+                    failure
+                  ).pipe(
+                    // A failed heartbeat is retried on the next tick, but a
+                    // minute of them lets the marker go stale, so each one is
+                    // reported rather than dropped.
+                    Effect.catch((cause) =>
+                      Effect.as(Effect.logWarning("Artifact backup lease heartbeat failed", cause), false)
+                    )
+                  )
+                  if (reclaimed) return
+                }
+              })
+            )
+            return yield* effect
+          }))
+        )),
+      () =>
+        ownsMarker
+          ? gate(
+            fs,
+            directory,
+            fs.readFileString(marker).pipe(
+              Effect.flatMap((found) => found === owner ? fs.remove(marker) : Effect.void),
+              // A backup whose heartbeat lapsed has its marker reaped by whoever
+              // noticed, so `NotFound` is the ordinary end of a slow lease, not a
+              // fault: there is nothing left to release and nothing left to leak.
+              // The sibling release in `internal/ArtifactLocks.ts` classifies the
+              // same cause the same way.
+              Effect.catch((cause): Effect.Effect<void, PlatformError.PlatformError> =>
+                isReason(cause, "NotFound") ? Effect.void : Effect.fail(cause)
+              )
+            ),
+            failure
+          ).pipe(
+            Effect.catch((cause) => Effect.logWarning("Artifact backup lease release failed", cause))
+          )
+          : Effect.void
+    )
+  })
+
+/**
+ * Runs one sweep deletion only when no live backup lease exists. The marker
+ * check and `effect` share the global gate, closing the check/delete race.
+ * `None` means a backup deliberately fenced the operation.
+ *
+ * @category combinators
+ * @since 1.0.0-rc.0
+ */
+export const unlessActive = <A, E, R, E2>(
+  fs: FileSystem.FileSystem,
+  directory: string,
+  effect: Effect.Effect<A, E, R>,
+  failure: (cause: unknown) => E2
+): Effect.Effect<Option.Option<A>, E | E2, R> =>
+  gate(
+    fs,
+    directory,
+    Effect.gen(function*() {
+      if (yield* activeMarker(fs, directory, failure)) return Option.none<A>()
+      return Option.some(yield* effect)
+    }),
+    failure
+  )

package/src/ArtifactStore.ts

@@ -0,0 +1,152 @@
+/**
+ * The content-addressed artifact store: bytes addressed by their own SHA-256
+ * digest.
+ *
+ * It is deliberately *not* the step cache: the step cache maps a step key to a
+ * recorded result, while large result bytes live here under their digest. The
+ * two tiers remain separate because artifacts must be published before a cache
+ * record may reference them. See the package README and
+ * {@link https://github.com/smithersai/smithers/blob/main/packages/smithers/flows/plan/docs/concepts/step-keys.md | step-key documentation}.
+ *
+ * This module defines the contract: the address schema, the service interface
+ * and tag, and one layer per local implementation. The errors, byte helpers,
+ * and implementations are defined in the modules named for them and re-exported
+ * here unchanged, so this subpath and the root `ArtifactStore` namespace keep
+ * their whole surface.
+ *
+ * @since 1.0.0-rc.0
+ */
+
+import { Sha256 } from "@smthrs/crypto"
+import * as Context from "effect/Context"
+import type * as Crypto from "effect/Crypto"
+import * as Effect from "effect/Effect"
+import * as FileSystem from "effect/FileSystem"
+import * as Layer from "effect/Layer"
+import * as Path from "effect/Path"
+import type { ArtifactCorruption, ArtifactMissing, ArtifactStoreError } from "./ArtifactStoreError.ts"
+import { type FileSystemOptions, makeFileSystem } from "./FileSystemArtifactStore.ts"
+import { makeMemory } from "./MemoryArtifactStore.ts"
+import { makeNoop } from "./NoopArtifactStore.ts"
+
+export {
+  ArtifactCorruption,
+  ArtifactMissing,
+  ArtifactStoreError,
+  ArtifactStoreErrorCode
+} from "./ArtifactStoreError.ts"
+export { defaultDirectory, type FileSystemOptions, makeFileSystem } from "./FileSystemArtifactStore.ts"
+export { measureBytes } from "./measureBytes.ts"
+export { makeMemory } from "./MemoryArtifactStore.ts"
+export { makeNoop } from "./NoopArtifactStore.ts"
+export { snapshotBytes } from "./snapshotBytes.ts"
+export { validateDigest } from "./validateDigest.ts"
+
+/**
+ * Schema for a content address: exactly 64 lowercase hexadecimal SHA-256
+ * characters, branded by `@smthrs/crypto`. Re-exported so a consumer never has
+ * to reach past this package for the address type it stores under.
+ *
+ * @category schemas
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export const Digest = Sha256.Digest
+
+/**
+ * A content address produced by this store.
+ *
+ * Read operations accept a plain `string` rather than this brand on purpose: a
+ * digest read back out of a durable row is untrusted input, so the store
+ * validates it (see {@link ArtifactStoreError}'s `invalid_digest` code) instead
+ * of asking every caller to re-brand a persisted column. `put` returns the
+ * brand, because it measured the bytes itself.
+ *
+ * @category models
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export type Digest = typeof Sha256.Digest.Type
+
+/**
+ * Content-addressed blob storage.
+ *
+ * The contract's ergonomics follow Effect's own `KeyValueStore`
+ * (`effect/persistence/KeyValueStore`): a small set of total
+ * operations over one address space, with a single typed error family, so a
+ * memory, filesystem, or network implementation is the same shape.
+ * `findMissing` is Bazel's `MissingDigestsFinder` — one batched round trip
+ * whose result is guaranteed to be a subset of its input — because a
+ * per-digest existence probe over a network tier is the wrong shape entirely.
+ *
+ * @category models
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export interface Service {
+  /**
+   * Stores `bytes` under their own SHA-256 digest and returns that address.
+   * Storing the same bytes twice is idempotent.
+   */
+  readonly put: (bytes: Uint8Array) => Effect.Effect<Digest, ArtifactStoreError, Crypto.Crypto>
+  /**
+   * Reads the bytes stored at `digest`, verifying that they still hash to it.
+   */
+  readonly get: (
+    digest: string
+  ) => Effect.Effect<Uint8Array, ArtifactMissing | ArtifactCorruption | ArtifactStoreError, Crypto.Crypto>
+  /** Whether this tier holds an artifact at `digest`. */
+  readonly has: (digest: string) => Effect.Effect<boolean, ArtifactStoreError>
+  /**
+   * Which of `digests` this tier does not hold. The returned array is
+   * guaranteed to be a subset of the input and free of duplicates.
+   */
+  readonly findMissing: (
+    digests: Iterable<string>
+  ) => Effect.Effect<Array<string>, ArtifactStoreError>
+}
+
+/**
+ * Service tag for the content-addressed artifact store.
+ *
+ * The identity string equals this module's package path, per the house rule
+ * that an identity is the defining module path.
+ *
+ * @category services
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export class ArtifactStore extends Context.Service<ArtifactStore, Service>()("@smthrs/artifacts/ArtifactStore") {}
+
+/**
+ * Provides the filesystem-backed artifact store.
+ *
+ * @category layers
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export const layerFileSystem = (
+  options: FileSystemOptions = {}
+): Layer.Layer<ArtifactStore, never, FileSystem.FileSystem | Path.Path> =>
+  Layer.effect(ArtifactStore)(Effect.gen(function*() {
+    return makeFileSystem(yield* FileSystem.FileSystem, yield* Path.Path, options)
+  }))
+
+/**
+ * Provides an in-memory artifact store.
+ *
+ * @category layers
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export const layerMemory: Layer.Layer<ArtifactStore> = Layer.effect(ArtifactStore)(Effect.sync(makeMemory))
+
+/**
+ * Provides a no-op artifact store.
+ *
+ * @category layers
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export const layerNoop = (overrides: Partial<Service> = {}): Layer.Layer<ArtifactStore> =>
+  Layer.succeed(ArtifactStore)(makeNoop(overrides))

package/src/ArtifactStoreError.ts

@@ -0,0 +1,96 @@
+/**
+ * The artifact store's typed failures: a refusal of the store itself, the
+ * typed miss, and a digest mismatch, with the stable codes a refusal carries.
+ *
+ * @since 1.0.0-rc.0
+ */
+
+import { Sha256 } from "@smthrs/crypto"
+import * as Schema from "effect/Schema"
+
+/**
+ * Stable error codes returned by artifact store operations.
+ *
+ * @category models
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export const ArtifactStoreErrorCode = Schema.Literals([
+  "digest_failed",
+  "invalid_configuration",
+  "invalid_digest",
+  "unavailable",
+  "transport_failed"
+])
+
+/**
+ * Stable error codes returned by artifact store operations.
+ *
+ * @category models
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export type ArtifactStoreErrorCode = typeof ArtifactStoreErrorCode.Type
+
+/**
+ * A typed failure of the store itself: the host or crypto provider refused an
+ * operation, the remote tier refused a request, or the caller supplied invalid
+ * configuration or an invalid content address.
+ *
+ * Distinct from {@link ArtifactMissing} and {@link ArtifactCorruption} on
+ * purpose. A miss is an ordinary, expected outcome that a second tier may
+ * still satisfy; corruption is an integrity violation of the store's strongest
+ * invariant. `invalid_configuration` and `invalid_digest` are permanent;
+ * retryability of host, crypto, and transport failures depends on the cause.
+ *
+ * @category errors
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export class ArtifactStoreError extends Schema.TaggedError<ArtifactStoreError>()(
+  "@smthrs/artifacts/ArtifactStoreError",
+  {
+    code: ArtifactStoreErrorCode,
+    message: Schema.String,
+    cause: Schema.optional(Schema.Unknown)
+  }
+) {}
+
+/**
+ * The typed miss: this tier holds no bytes at the requested address.
+ *
+ * A miss is not a failure of the store — it is the answer a read-through
+ * composition is built to act on, so it is a distinct tag rather than an
+ * `unavailable` code that a caller would have to string-match.
+ *
+ * @category errors
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export class ArtifactMissing extends Schema.TaggedError<ArtifactMissing>()(
+  "@smthrs/artifacts/ArtifactMissing",
+  {
+    code: Schema.Literal("artifact_missing"),
+    digest: Sha256.Digest
+  }
+) {}
+
+/**
+ * Bytes stored at a content address no longer hash to it.
+ *
+ * Every read is digest-verified, so a truncated blob left by a crashing writer
+ * or by disk corruption is refused rather than handed back as if it were the
+ * recorded artifact.
+ *
+ * @category errors
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export class ArtifactCorruption extends Schema.TaggedError<ArtifactCorruption>()(
+  "@smthrs/artifacts/ArtifactCorruption",
+  {
+    code: Schema.Literal("artifact_corruption"),
+    recordedDigest: Sha256.Digest,
+    measuredDigest: Sha256.Digest
+  }
+) {}