@smthrs/artifacts 0.0.0-stage → 1.0.0-rc.4

package/src/ArtifactBackupLease.ts

@@ -0,0 +1,213 @@
+/**
+ * Cross-process exclusion between a filesystem artifact backup and garbage
+ * collection.
+ *
+ * A backup holds one heartbeat-backed marker for the duration of its database
+ * snapshot and blob copy. Sweep deletion checks that marker while holding the
+ * same short-lived global gate. Publication remains unconstrained: new blobs
+ * may appear during a backup, but a blob already referenced by the frozen
+ * database cannot disappear underneath it.
+ *
+ * @since 1.0.0-rc.0
+ */
+
+import * as Clock from "effect/Clock"
+import * as Effect from "effect/Effect"
+import type * as FileSystem from "effect/FileSystem"
+import * as Option from "effect/Option"
+import * as PlatformError from "effect/PlatformError"
+import * as Random from "effect/Random"
+import * as ArtifactLocks from "./internal/ArtifactLocks.ts"
+
+const markerName = ".backup-lease"
+const gateName = ".backup-lease-gate"
+const heartbeatEvery = "10 seconds"
+const staleAfterMs = 60_000
+const acquireWithin = "2 minutes"
+const retryEvery = "25 millis"
+
+const isReason = (cause: unknown, tag: PlatformError.SystemErrorTag): boolean =>
+  cause instanceof PlatformError.PlatformError && cause.reason._tag === tag
+
+const token = Effect.gen(function*() {
+  const first = yield* Random.nextIntBetween(0, Number.MAX_SAFE_INTEGER, { halfOpen: true })
+  const second = yield* Random.nextIntBetween(0, Number.MAX_SAFE_INTEGER, { halfOpen: true })
+  return `${first.toString(36)}-${second.toString(36)}-${yield* Clock.currentTimeMillis}`
+})
+
+const gate = <A, E, R, E2>(
+  fs: FileSystem.FileSystem,
+  directory: string,
+  effect: Effect.Effect<A, E, R>,
+  failure: (cause: unknown) => E2
+): Effect.Effect<A, E | E2, R> => ArtifactLocks.withDigest(fs, directory, gateName, effect, failure)
+
+const activeMarker = <E>(
+  fs: FileSystem.FileSystem,
+  directory: string,
+  failure: (cause: unknown) => E
+): Effect.Effect<boolean, E> =>
+  Effect.gen(function*() {
+    const path = `${directory}/${markerName}`
+    const info = yield* fs.stat(path).pipe(
+      Effect.map(Option.some),
+      Effect.catch((cause): Effect.Effect<Option.Option<FileSystem.File.Info>, E> =>
+        isReason(cause, "NotFound") ? Effect.succeed(Option.none()) : Effect.fail(failure(cause))
+      )
+    )
+    if (Option.isNone(info)) return false
+    const modified = Option.getOrUndefined(info.value.mtime)
+    const now = yield* Clock.currentTimeMillis
+    if (modified === undefined || now - modified.getTime() <= staleAfterMs) return true
+    yield* fs.remove(path).pipe(
+      Effect.catch((cause): Effect.Effect<void, E> =>
+        isReason(cause, "NotFound") ? Effect.void : Effect.fail(failure(cause))
+      )
+    )
+    return false
+  })
+
+/**
+ * Runs `effect` while sweep deletion is fenced across every process using
+ * this objects directory. A crashed holder is recovered after its heartbeat
+ * becomes stale.
+ *
+ * @category combinators
+ * @since 1.0.0-rc.0
+ */
+export const withLease = <A, E, R, E2>(
+  fs: FileSystem.FileSystem,
+  directory: string,
+  effect: Effect.Effect<A, E, R>,
+  failure: (cause: unknown) => E2
+): Effect.Effect<A, E | E2, R> =>
+  Effect.gen(function*() {
+    const owner = yield* token
+    const marker = `${directory}/${markerName}`
+    yield* fs.makeDirectory(directory, { recursive: true }).pipe(Effect.mapError(failure))
+
+    let ownsMarker = false
+    const acquire = Effect.gen(function*() {
+      while (true) {
+        const acquired = yield* gate(
+          fs,
+          directory,
+          Effect.gen(function*() {
+            if (yield* activeMarker(fs, directory, failure)) return false
+            return yield* Effect.uninterruptible(
+              fs.writeFileString(marker, owner, { flag: "wx", mode: 0o600 }).pipe(
+                Effect.andThen(Effect.sync(() => {
+                  ownsMarker = true
+                  return true
+                })),
+                Effect.catch((cause): Effect.Effect<boolean, E2> =>
+                  isReason(cause, "AlreadyExists") ? Effect.succeed(false) : Effect.fail(failure(cause))
+                )
+              )
+            )
+          }),
+          failure
+        )
+        if (acquired) return
+        yield* Effect.sleep(retryEvery)
+      }
+    }).pipe(
+      Effect.timeout(acquireWithin),
+      Effect.catchTag("TimeoutError", (cause) => Effect.fail(failure(cause)))
+    )
+
+    // Install marker cleanup before acquisition, including gate release. The
+    // marker write and ownership bookkeeping complete in one interruption mask.
+    return yield* Effect.acquireUseRelease(
+      Effect.void,
+      () =>
+        acquire.pipe(Effect.andThen(
+          Effect.scoped(Effect.gen(function*() {
+            yield* Effect.forkScoped(
+              Effect.forever(
+                Effect.sleep(heartbeatEvery).pipe(
+                  Effect.andThen(gate(
+                    fs,
+                    directory,
+                    fs.readFileString(marker).pipe(
+                      Effect.map(Option.some),
+                      Effect.catch((cause) =>
+                        isReason(cause, "NotFound") ? Effect.succeed(Option.none<string>()) : Effect.fail(cause)
+                      ),
+                      Effect.flatMap((found) => {
+                        // A missing or foreign marker means a sweep judged this
+                        // lease stale and reaped it: deletion is no longer
+                        // fenced while the backup still copies. The sibling
+                        // `internal/ArtifactLocks.ts` heartbeat reports the same
+                        // loss the same way.
+                        if (Option.getOrUndefined(found) !== owner) {
+                          return Effect.logWarning(
+                            "Artifact backup lease was reclaimed while its backup was still running"
+                          )
+                        }
+                        return Effect.flatMap(Clock.currentTimeMillis, (now) => {
+                          const timestamp = new Date(now)
+                          return fs.utimes(marker, timestamp, timestamp)
+                        })
+                      }),
+                      Effect.mapError(failure)
+                    ),
+                    failure
+                  )),
+                  // A failed heartbeat is retried on the next tick, but a
+                  // minute of them lets the marker go stale, so each one is
+                  // reported rather than dropped.
+                  Effect.catch((cause) => Effect.logWarning("Artifact backup lease heartbeat failed", cause))
+                )
+              )
+            )
+            return yield* effect
+          }))
+        )),
+      () =>
+        ownsMarker
+          ? gate(
+            fs,
+            directory,
+            fs.readFileString(marker).pipe(
+              Effect.flatMap((found) => found === owner ? fs.remove(marker) : Effect.void),
+              // A backup whose heartbeat lapsed has its marker reaped by whoever
+              // noticed, so `NotFound` is the ordinary end of a slow lease, not a
+              // fault: there is nothing left to release and nothing left to leak.
+              // The sibling release in `internal/ArtifactLocks.ts` classifies the
+              // same cause the same way.
+              Effect.catch((cause): Effect.Effect<void, PlatformError.PlatformError> =>
+                isReason(cause, "NotFound") ? Effect.void : Effect.fail(cause)
+              )
+            ),
+            failure
+          ).pipe(
+            Effect.catch((cause) => Effect.logWarning("Artifact backup lease release failed", cause))
+          )
+          : Effect.void
+    )
+  })
+
+/**
+ * Runs one sweep deletion only when no live backup lease exists. The marker
+ * check and `effect` share the global gate, closing the check/delete race.
+ * `None` means a backup deliberately fenced the operation.
+ *
+ * @category combinators
+ * @since 1.0.0-rc.0
+ */
+export const unlessActive = <A, E, R, E2>(
+  fs: FileSystem.FileSystem,
+  directory: string,
+  effect: Effect.Effect<A, E, R>,
+  failure: (cause: unknown) => E2
+): Effect.Effect<Option.Option<A>, E | E2, R> =>
+  gate(
+    fs,
+    directory,
+    Effect.gen(function*() {
+      if (yield* activeMarker(fs, directory, failure)) return Option.none<A>()
+      return Option.some(yield* effect)
+    }),
+    failure
+  )

package/src/ArtifactStore.ts

@@ -0,0 +1,152 @@
+/**
+ * The content-addressed artifact store: bytes addressed by their own SHA-256
+ * digest.
+ *
+ * It is deliberately *not* the step cache: the step cache maps a step key to a
+ * recorded result, while large result bytes live here under their digest. The
+ * two tiers remain separate because artifacts must be published before a cache
+ * record may reference them. See the package README and
+ * {@link https://smithers.sh/docs/concepts/content-addressing | step-key documentation}.
+ *
+ * This module defines the contract: the address schema, the service interface
+ * and tag, and one layer per local implementation. The errors, byte helpers,
+ * and implementations are defined in the modules named for them and re-exported
+ * here unchanged, so this subpath and the root `ArtifactStore` namespace keep
+ * their whole surface.
+ *
+ * @since 1.0.0-rc.0
+ */
+
+import { Sha256 } from "@smthrs/crypto"
+import * as Context from "effect/Context"
+import type * as Crypto from "effect/Crypto"
+import * as Effect from "effect/Effect"
+import * as FileSystem from "effect/FileSystem"
+import * as Layer from "effect/Layer"
+import * as Path from "effect/Path"
+import type { ArtifactCorruption, ArtifactMissing, ArtifactStoreError } from "./ArtifactStoreError.ts"
+import { type FileSystemOptions, makeFileSystem } from "./FileSystemArtifactStore.ts"
+import { makeMemory } from "./MemoryArtifactStore.ts"
+import { makeNoop } from "./NoopArtifactStore.ts"
+
+export {
+  ArtifactCorruption,
+  ArtifactMissing,
+  ArtifactStoreError,
+  ArtifactStoreErrorCode
+} from "./ArtifactStoreError.ts"
+export { defaultDirectory, type FileSystemOptions, makeFileSystem } from "./FileSystemArtifactStore.ts"
+export { measureBytes } from "./measureBytes.ts"
+export { makeMemory } from "./MemoryArtifactStore.ts"
+export { makeNoop } from "./NoopArtifactStore.ts"
+export { snapshotBytes } from "./snapshotBytes.ts"
+export { validateDigest } from "./validateDigest.ts"
+
+/**
+ * Schema for a content address: exactly 64 lowercase hexadecimal SHA-256
+ * characters, branded by `@smthrs/crypto`. Re-exported so a consumer never has
+ * to reach past this package for the address type it stores under.
+ *
+ * @category schemas
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export const Digest = Sha256.Digest
+
+/**
+ * A content address produced by this store.
+ *
+ * Read operations accept a plain `string` rather than this brand on purpose: a
+ * digest read back out of a durable row is untrusted input, so the store
+ * validates it (see {@link ArtifactStoreError}'s `invalid_digest` code) instead
+ * of asking every caller to re-brand a persisted column. `put` returns the
+ * brand, because it measured the bytes itself.
+ *
+ * @category models
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export type Digest = typeof Sha256.Digest.Type
+
+/**
+ * Content-addressed blob storage.
+ *
+ * The contract's ergonomics follow Effect's own `KeyValueStore`
+ * (`effect/unstable/persistence/KeyValueStore`): a small set of total
+ * operations over one address space, with a single typed error family, so a
+ * memory, filesystem, or network implementation is the same shape.
+ * `findMissing` is Bazel's `MissingDigestsFinder` — one batched round trip
+ * whose result is guaranteed to be a subset of its input — because a
+ * per-digest existence probe over a network tier is the wrong shape entirely.
+ *
+ * @category models
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export interface Service {
+  /**
+   * Stores `bytes` under their own SHA-256 digest and returns that address.
+   * Storing the same bytes twice is idempotent.
+   */
+  readonly put: (bytes: Uint8Array) => Effect.Effect<Digest, ArtifactStoreError, Crypto.Crypto>
+  /**
+   * Reads the bytes stored at `digest`, verifying that they still hash to it.
+   */
+  readonly get: (
+    digest: string
+  ) => Effect.Effect<Uint8Array, ArtifactMissing | ArtifactCorruption | ArtifactStoreError, Crypto.Crypto>
+  /** Whether this tier holds an artifact at `digest`. */
+  readonly has: (digest: string) => Effect.Effect<boolean, ArtifactStoreError>
+  /**
+   * Which of `digests` this tier does not hold. The returned array is
+   * guaranteed to be a subset of the input and free of duplicates.
+   */
+  readonly findMissing: (
+    digests: Iterable<string>
+  ) => Effect.Effect<Array<string>, ArtifactStoreError>
+}
+
+/**
+ * Service tag for the content-addressed artifact store.
+ *
+ * The identity string equals this module's package path, per the house rule
+ * that an identity is the defining module path.
+ *
+ * @category services
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export class ArtifactStore extends Context.Service<ArtifactStore, Service>()("@smthrs/artifacts/ArtifactStore") {}
+
+/**
+ * Provides the filesystem-backed artifact store.
+ *
+ * @category layers
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export const layerFileSystem = (
+  options: FileSystemOptions = {}
+): Layer.Layer<ArtifactStore, never, FileSystem.FileSystem | Path.Path> =>
+  Layer.effect(ArtifactStore)(Effect.gen(function*() {
+    return makeFileSystem(yield* FileSystem.FileSystem, yield* Path.Path, options)
+  }))
+
+/**
+ * Provides an in-memory artifact store.
+ *
+ * @category layers
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export const layerMemory: Layer.Layer<ArtifactStore> = Layer.effect(ArtifactStore)(Effect.sync(makeMemory))
+
+/**
+ * Provides a no-op artifact store.
+ *
+ * @category layers
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export const layerNoop = (overrides: Partial<Service> = {}): Layer.Layer<ArtifactStore> =>
+  Layer.succeed(ArtifactStore)(makeNoop(overrides))

package/src/ArtifactStoreError.ts

@@ -0,0 +1,96 @@
+/**
+ * The artifact store's typed failures: a refusal of the store itself, the
+ * typed miss, and a digest mismatch, with the stable codes a refusal carries.
+ *
+ * @since 1.0.0-rc.0
+ */
+
+import { Sha256 } from "@smthrs/crypto"
+import * as Schema from "effect/Schema"
+
+/**
+ * Stable error codes returned by artifact store operations.
+ *
+ * @category models
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export const ArtifactStoreErrorCode = Schema.Literals([
+  "digest_failed",
+  "invalid_configuration",
+  "invalid_digest",
+  "unavailable",
+  "transport_failed"
+])
+
+/**
+ * Stable error codes returned by artifact store operations.
+ *
+ * @category models
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export type ArtifactStoreErrorCode = typeof ArtifactStoreErrorCode.Type
+
+/**
+ * A typed failure of the store itself: the host or crypto provider refused an
+ * operation, the remote tier refused a request, or the caller supplied invalid
+ * configuration or an invalid content address.
+ *
+ * Distinct from {@link ArtifactMissing} and {@link ArtifactCorruption} on
+ * purpose. A miss is an ordinary, expected outcome that a second tier may
+ * still satisfy; corruption is an integrity violation of the store's strongest
+ * invariant. `invalid_configuration` and `invalid_digest` are permanent;
+ * retryability of host, crypto, and transport failures depends on the cause.
+ *
+ * @category errors
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export class ArtifactStoreError extends Schema.TaggedError<ArtifactStoreError>()(
+  "@smthrs/artifacts/ArtifactStoreError",
+  {
+    code: ArtifactStoreErrorCode,
+    message: Schema.String,
+    cause: Schema.optional(Schema.Unknown)
+  }
+) {}
+
+/**
+ * The typed miss: this tier holds no bytes at the requested address.
+ *
+ * A miss is not a failure of the store — it is the answer a read-through
+ * composition is built to act on, so it is a distinct tag rather than an
+ * `unavailable` code that a caller would have to string-match.
+ *
+ * @category errors
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export class ArtifactMissing extends Schema.TaggedError<ArtifactMissing>()(
+  "@smthrs/artifacts/ArtifactMissing",
+  {
+    code: Schema.Literal("artifact_missing"),
+    digest: Sha256.Digest
+  }
+) {}
+
+/**
+ * Bytes stored at a content address no longer hash to it.
+ *
+ * Every read is digest-verified, so a truncated blob left by a crashing writer
+ * or by disk corruption is refused rather than handed back as if it were the
+ * recorded artifact.
+ *
+ * @category errors
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export class ArtifactCorruption extends Schema.TaggedError<ArtifactCorruption>()(
+  "@smthrs/artifacts/ArtifactCorruption",
+  {
+    code: Schema.Literal("artifact_corruption"),
+    recordedDigest: Sha256.Digest,
+    measuredDigest: Sha256.Digest
+  }
+) {}

package/src/ArtifactStoreMetrics.ts

@@ -0,0 +1,83 @@
+/**
+ * Standard metric definitions for content-addressed artifact storage.
+ *
+ * This module only defines the metric handles, following the shape of Effect's
+ * `ClusterMetrics`. No exporter ships in this package; provide one — for
+ * example `@smthrs/observability` — and these counters appear in it.
+ *
+ * Only the local `ArtifactStore` implementations, filesystem and memory, update
+ * them. `RemoteArtifacts` is deliberately uninstrumented, and the counters carry
+ * no tier attribute, so read them as *local artifact store traffic* rather than
+ * as artifact operations:
+ *
+ * - a `CombinedArtifacts` read the local tier serves counts one get;
+ * - a read the shared tier serves counts NO get, because no local store
+ *   answered it;
+ * - the write-back that materializes such a read counts a put, indistinguishable
+ *   from a producer publishing new bytes.
+ *
+ * Attributing operations per tier needs the tier in the metric, which would
+ * change the published counter shape; until then this is what the numbers mean.
+ *
+ * The one shared-tier signal is {@link remoteFailure}: a `CombinedArtifacts`
+ * upload or write-back that was refused, or abandoned at its deadline.
+ *
+ * @since 1.0.0-rc.0
+ */
+
+import * as Metric from "effect/Metric"
+
+/**
+ * Counter over successful artifact puts. A put that deduplicated against an
+ * existing verified blob still counts: the caller stored bytes and received
+ * an address either way.
+ *
+ * @category metrics
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export const puts = Metric.counter("flows_artifact_puts", {
+  description: "Successful artifact puts, including deduplicated ones"
+})
+
+/**
+ * Counter over successful artifact gets. Missing and corrupt reads fail with
+ * their typed errors and are deliberately not counted here; they are error
+ * evidence, not throughput.
+ *
+ * @category metrics
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export const gets = Metric.counter("flows_artifact_gets", {
+  description: "Successful artifact gets"
+})
+
+/**
+ * Counter over opportunistic `CombinedArtifacts` transfers that failed or hit
+ * their deadline and were dropped: `put`'s upload to the shared tier and
+ * `get`'s write-back into the local tier.
+ *
+ * Read the attributed views on {@link remoteFailure}: every update carries an
+ * `operation` attribute, so this bare handle aggregates nothing and always
+ * reads zero.
+ *
+ * @category metrics
+ * @since 1.0.0-rc.1
+ */
+export const remoteFailures = Metric.counter("flows_artifact_remote_failures", {
+  description: "Dropped combined-artifact transfers by operation"
+})
+
+/**
+ * `remoteFailures` views keyed by the dropped transfer.
+ *
+ * @category metrics
+ * @since 1.0.0-rc.1
+ */
+export const remoteFailure: {
+  readonly [Operation in "put" | "write_back"]: Metric.Metric<number, Metric.CounterState<number>>
+} = {
+  put: Metric.withAttributes(remoteFailures, { operation: "put" }),
+  write_back: Metric.withAttributes(remoteFailures, { operation: "write_back" })
+}