@smthrs/artifacts 0.0.0-stage → 1.0.0-rc.3

package/src/index.ts

@@ -0,0 +1,71 @@
+/**
+ * The content-addressed artifact store: bytes addressed by their own digest.
+ *
+ * `ArtifactStore` is the byte half of the cache. `@smthrs/step-cache` maps a
+ * step key to a recorded result, while large result bytes live here under their
+ * digest. The package depends on `effect` and `@smthrs/crypto`, owns no SQL,
+ * and bundles for the browser: host access is Effect's `FileSystem` and
+ * `HttpClient` tags, both of which the capability kernel decorates in place.
+ *
+ * ```ts
+ * import { ArtifactStore, CombinedArtifacts, RemoteArtifacts } from "@smthrs/artifacts"
+ * import * as Effect from "effect/Effect"
+ * import * as FileSystem from "effect/FileSystem"
+ * import * as Path from "effect/Path"
+ *
+ * const layer = CombinedArtifacts.layer({
+ *   local: Effect.map(Effect.all([FileSystem.FileSystem, Path.Path]), ([fs, path]) =>
+ *     ArtifactStore.makeFileSystem(fs, path)),
+ *   remote: RemoteArtifacts.make({ endpoint: "https://cache.example.com" })
+ * })
+ * ```
+ *
+ * @since 1.0.0-rc.0
+ */
+
+/**
+ * @category services
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export * as ArtifactStore from "./ArtifactStore.ts"
+
+/**
+ * @category metrics
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export * as ArtifactStoreMetrics from "./ArtifactStoreMetrics.ts"
+
+/**
+ * @category services
+ * @since 1.0.0-rc.0
+ */
+export * as ArtifactBackupLease from "./ArtifactBackupLease.ts"
+
+/**
+ * @category services
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export * as ArtifactSweep from "./ArtifactSweep.ts"
+
+/**
+ * @category services
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export * as CombinedArtifacts from "./CombinedArtifacts.ts"
+
+/**
+ * @category coordination
+ * @since 1.0.0-rc.1
+ */
+export * as FileLease from "./FileLease.ts"
+
+/**
+ * @category services
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export * as RemoteArtifacts from "./RemoteArtifacts.ts"

package/src/internal/ArtifactLocks.ts

@@ -0,0 +1,110 @@
+/**
+ * Per-digest coordination for filesystem publication and removal.
+ *
+ * The semaphore is the cheap in-process path. A `FileLease` lock file is the
+ * actual workspace-wide fence: every process that can mutate the object
+ * directory observes it, and a crashed owner is recovered after its heartbeat
+ * expires.
+ *
+ * @since 1.0.0-rc.0
+ */
+
+import * as Deferred from "effect/Deferred"
+import * as Effect from "effect/Effect"
+import type * as FileSystem from "effect/FileSystem"
+import * as Semaphore from "effect/Semaphore"
+import * as FileLease from "../FileLease.ts"
+import * as ArtifactPath from "./ArtifactPath.ts"
+
+interface Entry {
+  readonly semaphore: Semaphore.Semaphore
+  users: number
+}
+
+const locks = new WeakMap<FileSystem.FileSystem, Map<string, Entry>>()
+
+/**
+ * The subdirectory of the objects directory that holds lock files and their
+ * stale-owner tombstones. Named here rather than spelled in each caller so the
+ * store's crash-orphan sweep reclaims exactly what this module creates.
+ *
+ * @category constants
+ * @since 1.0.0-rc.0
+ */
+export const directoryName = ".locks"
+
+/**
+ * Coordinates publication, freshening, and sweep deletion for one digest.
+ * `process` is the explicit weaker mode for hosts without atomic create.
+ *
+ * @category combinators
+ * @since 1.0.0-rc.0
+ */
+export const withDigest = <A, E, R, E2>(
+  fs: FileSystem.FileSystem,
+  directory: string,
+  digest: string,
+  effect: Effect.Effect<A, E, R>,
+  failure: (cause: unknown) => E2,
+  coordination: "required" | "process" = "required"
+): Effect.Effect<A, E | E2, R> =>
+  // Every execution of the returned effect claims its own share of the entry,
+  // which is why the whole body is suspended rather than run when `withDigest`
+  // is called. A count taken at construction is wrong in both directions: an
+  // effect built and discarded pins an entry nothing will ever release, and one
+  // built once and run repeatedly — `ArtifactBackupLease` builds its gate once
+  // and runs it on every heartbeat — retires the entry underneath a live holder,
+  // so the next caller mints a second semaphore for the same digest and the two
+  // serialize against nothing.
+  Effect.suspend(() => {
+    let byKey = locks.get(fs)
+    if (byKey === undefined) {
+      byKey = new Map()
+      locks.set(fs, byKey)
+    }
+    const key = JSON.stringify([directory, digest])
+    let entry = byKey.get(key)
+    if (entry === undefined) {
+      entry = { semaphore: Semaphore.makeUnsafe(1), users: 0 }
+      byKey.set(key, entry)
+    }
+    entry.users += 1
+    const held = entry
+    const table = byKey
+
+    // The deadline starts before the semaphore wait and ends once both locks
+    // are held. The protected operation and its release are never timed out.
+    const ready = Deferred.makeUnsafe<void>()
+    const deadline = Deferred.await(ready).pipe(
+      Effect.timeout(FileLease.defaultAcquireWithin),
+      Effect.catchTag("TimeoutError", (cause) => Effect.fail(failure(cause))),
+      Effect.andThen(Effect.never)
+    )
+    const coordinated = coordination === "process"
+      ? Deferred.succeed(ready, undefined).pipe(Effect.andThen(effect))
+      : Effect.gen(function*() {
+        // The lock directory is refused as a symlink before and after it is
+        // created, and its identity is re-checked before every lock-file
+        // mutation, so a swap of `.locks` for a link cannot redirect lock,
+        // claim, or tombstone writes and removals elsewhere. The portable
+        // filesystem cannot make a check atomic with the mutation after it.
+        const lockDirectory = `${directory}/${directoryName}`
+        yield* ArtifactPath.guard(fs, lockDirectory).pipe(Effect.mapError(failure))
+        yield* fs.makeDirectory(lockDirectory, { recursive: true, mode: 0o700 }).pipe(Effect.mapError(failure))
+        const checkLockDirectory = yield* ArtifactPath.guard(fs, lockDirectory).pipe(Effect.mapError(failure))
+        return yield* FileLease.hold(fs, `${lockDirectory}/${digest}.lock`, effect, failure, {
+          label: "Artifact lock",
+          annotations: { digest },
+          onAcquired: Deferred.succeed(ready, undefined).pipe(Effect.asVoid),
+          guard: checkLockDirectory
+        })
+      })
+
+    return held.semaphore.withPermit(coordinated).pipe(
+      Effect.raceFirst(deadline),
+      Effect.ensuring(Effect.sync(() => {
+        held.users -= 1
+        if (held.users === 0 && table.get(key) === held) table.delete(key)
+      }))
+    )
+  })

package/src/internal/ArtifactPath.ts

@@ -0,0 +1,72 @@
+/**
+ * Path checks for the portable filesystem adapter. These detect symlinks and
+ * observed replacements, but cannot make pathname mutations atomic with the
+ * check. Effect's FileSystem has no descriptor-relative rename or unlink.
+ *
+ * @since 1.0.0-rc.0
+ */
+
+import * as Effect from "effect/Effect"
+import type * as FileSystem from "effect/FileSystem"
+import * as Option from "effect/Option"
+import * as PlatformError from "effect/PlatformError"
+
+/**
+ * Checks one entry without accepting a symbolic link and returns a check that
+ * also refuses a changed identity. Missing entries are allowed for creation.
+ *
+ * @category utilities
+ * @since 1.0.0-rc.0
+ */
+export const guard = (
+  fs: FileSystem.FileSystem,
+  path: string,
+  type: "Directory" | "File" = "Directory",
+  expected?: FileSystem.File.Info
+): Effect.Effect<Effect.Effect<void, PlatformError.PlatformError>, PlatformError.PlatformError> => {
+  const refused = () =>
+    PlatformError.systemError({
+      _tag: "PermissionDenied",
+      module: "ArtifactPath",
+      method: "guard",
+      pathOrDescriptor: path,
+      description: "artifact path is a symlink, has the wrong type, or was replaced"
+    })
+  const inspect = Effect.gen(function*() {
+    const linked = yield* fs.readLink(path).pipe(
+      Effect.as(true),
+      Effect.catch((cause) => {
+        // Node readlink uses EINVAL for an existing entry that is not a link.
+        // Other failures, including an unsupported host capability, fail closed.
+        const underlying = cause.cause
+        return cause.reason._tag === "NotFound" ||
+            (typeof underlying === "object" && underlying !== null && "code" in underlying &&
+              underlying.code === "EINVAL")
+          ? Effect.succeed(false)
+          : Effect.fail(cause)
+      })
+    )
+    if (linked) return yield* Effect.fail(refused())
+    const info = yield* fs.stat(path).pipe(
+      Effect.map(Option.some),
+      Effect.catch((cause) => cause.reason._tag === "NotFound" ? Effect.succeed(Option.none()) : Effect.fail(cause))
+    )
+    if (Option.isSome(info) && info.value.type !== type) return yield* Effect.fail(refused())
+    return info
+  })
+  const unchanged = (before: Option.Option<FileSystem.File.Info>, after: Option.Option<FileSystem.File.Info>) => {
+    if (Option.isNone(before)) return Effect.void
+    if (
+      Option.isNone(after) || before.value.dev !== after.value.dev ||
+      Option.getOrUndefined(before.value.ino) !== Option.getOrUndefined(after.value.ino)
+    ) {
+      return Effect.fail(refused())
+    }
+    return Effect.void
+  }
+  return Effect.gen(function*() {
+    const before = yield* inspect
+    yield* unchanged(Option.fromUndefinedOr(expected), before)
+    return Effect.flatMap(inspect, (after) => unchanged(before, after))
+  })
+}

package/src/measureBytes.ts

@@ -0,0 +1,28 @@
+/**
+ * Measuring artifact bytes: the SHA-256 content address of one immutable
+ * snapshot, computed through the injected `Crypto` service.
+ *
+ * @since 1.0.0-rc.0
+ */
+
+import { Sha256 } from "@smthrs/crypto"
+import type * as Crypto from "effect/Crypto"
+import * as Effect from "effect/Effect"
+import type { Digest } from "./ArtifactStore.ts"
+import { ArtifactStoreError } from "./ArtifactStoreError.ts"
+
+const digestFailure = (cause: unknown): ArtifactStoreError =>
+  new ArtifactStoreError({
+    code: "digest_failed",
+    message: "the Crypto service failed to compute an artifact digest",
+    cause
+  })
+
+/**
+ * Measures one immutable byte snapshot without ever retaining it in an error.
+ *
+ * @category utilities
+ * @since 1.0.0-rc.0
+ */
+export const measureBytes = (bytes: Uint8Array): Effect.Effect<Digest, ArtifactStoreError, Crypto.Crypto> =>
+  Sha256.digest(bytes).pipe(Effect.mapError(digestFailure))

package/src/snapshotBytes.ts

@@ -0,0 +1,34 @@
+/**
+ * Snapshotting caller-owned artifact bytes, so a caller that reuses its buffer
+ * cannot change what is measured or stored.
+ *
+ * @since 1.0.0-rc.0
+ */
+
+import * as Effect from "effect/Effect"
+import { ArtifactStoreError } from "./ArtifactStoreError.ts"
+
+/**
+ * A refused byte copy is the host declining an allocation, not a crypto
+ * failure: the Crypto service has not been consulted when this fires, so
+ * reporting `digest_failed` would point an operator diagnosing memory pressure
+ * at the wrong subsystem. The message is constant, and the buffer is never
+ * attached.
+ */
+const snapshotFailure = (cause: unknown): ArtifactStoreError =>
+  new ArtifactStoreError({ code: "unavailable", message: "the host could not copy the artifact bytes", cause })
+
+/**
+ * Copies a caller-owned buffer when the returned Effect begins.
+ *
+ * A host that refuses the copy — a detached buffer, an allocation past what the
+ * runtime will give — fails as `unavailable`, the code for a host refusal.
+ *
+ * @category utilities
+ * @since 1.0.0-rc.0
+ */
+export const snapshotBytes = (bytes: Uint8Array): Effect.Effect<Uint8Array, ArtifactStoreError> =>
+  Effect.try({
+    try: () => new Uint8Array(bytes),
+    catch: (cause) => snapshotFailure(cause)
+  })

package/src/validateDigest.ts

@@ -0,0 +1,31 @@
+/**
+ * Validating a content address before any implementation logs it or
+ * interpolates it into a path or URL.
+ *
+ * @since 1.0.0-rc.0
+ */
+
+import * as Effect from "effect/Effect"
+import type { Digest } from "./ArtifactStore.ts"
+import { ArtifactStoreError } from "./ArtifactStoreError.ts"
+
+/**
+ * Refuses anything other than the canonical SHA-256 address representation.
+ *
+ * Every implementation validates before logging or interpolating an untrusted
+ * value into a path or URL. The failure text is constant and bounded, so even a
+ * hostile multi-megabyte value cannot be copied into logs or durable errors.
+ *
+ * @category predicates
+ * @since 1.0.0-rc.0
+ * @slop
+ */
+export const validateDigest = (digest: string): Effect.Effect<Digest, ArtifactStoreError> =>
+  typeof digest === "string" && /^[0-9a-f]{64}$/.test(digest)
+    ? Effect.succeed(digest as Digest)
+    : Effect.fail(
+      new ArtifactStoreError({
+        code: "invalid_digest",
+        message: "artifact digest must be exactly 64 lowercase hexadecimal characters"
+      })
+    )