@smthrs/engine-store 0.0.0-stage → 1.0.0-rc.4

package/src/internal/CacheOutputPolicy.ts

@@ -0,0 +1,82 @@
+/**
+ * Replay authority comes from the current descriptor, never the recorded paths.
+ * This preflight performs no host I/O; every path is checked before replay can prune.
+ * @since 1.0.0-rc.0
+ */
+
+import * as ArtifactStore from "@smthrs/artifacts/ArtifactStore"
+import type { FileBoundary } from "@smthrs/flow/FileBoundary"
+import * as FileSet from "@smthrs/plan/FileSet"
+import * as Schema from "effect/Schema"
+import type * as StepBoundary from "../StepBoundary.ts"
+
+const Outputs = Schema.Struct({
+  outputs: Schema.Array(Schema.Struct({ path: Schema.String, digest: Schema.NullOr(ArtifactStore.Digest) })),
+  trees: Schema.optional(Schema.Array(Schema.Struct({ path: Schema.String, identity: Schema.String })))
+})
+// The existing abstract boundary protocol records its exact declared path set.
+// Validate that format separately; it does not claim a filesystem blob manifest.
+const DeclaredPaths = Schema.Struct({ paths: Schema.Array(FileSet.Entry) })
+const samePaths = Schema.toEquivalence(Schema.Array(FileSet.Entry))
+
+/** A typed refusal leaves both the evidence and the workspace intact.
+ * @category models
+ * @since 1.0.0-rc.0
+ */
+export type Decision =
+  | { readonly _tag: "ReplayOutputs" }
+  | { readonly _tag: "Refused"; readonly reason: "output-boundary-mismatch" | "unsupported-output-evidence" }
+
+/** Validate all recorded writes, removals and pruning roots before materialization.
+ * Unknown output formats cannot establish current filesystem authority.
+ * @category classifiers
+ * @since 1.0.0-rc.0
+ */
+export const replay = (
+  descriptor: FileBoundary | undefined,
+  evidence: StepBoundary.BoundaryEvidence
+): Decision => {
+  const mismatch = { _tag: "Refused", reason: "output-boundary-mismatch" } as const
+  if (descriptor === undefined) return mismatch
+  const decoded = Schema.decodeUnknownResult(Outputs)(evidence.declaredOutputs)
+  if (decoded._tag === "Failure") {
+    const paths = Schema.decodeUnknownResult(DeclaredPaths)(evidence.declaredOutputs)
+    // Malformed production evidence cannot downgrade itself to the abstract
+    // protocol by also supplying a matching path list.
+    if (paths._tag === "Failure" || Object.hasOwn(evidence.declaredOutputs as object, "outputs")) {
+      return { _tag: "Refused", reason: "unsupported-output-evidence" }
+    }
+    return (descriptor.removes ?? []).length === 0 && samePaths(paths.success.paths, descriptor.writeSet)
+      ? { _tag: "ReplayOutputs" }
+      : mismatch
+  }
+  const { outputs, trees = [] } = decoded.success
+  const paths = [...outputs.map((output) => output.path), ...trees.map((tree) => tree.path)]
+  // Use the exact coordinate spelling that replay passes to the filesystem.
+  if (paths.some((path) => !FileSet.workspaceRelative(path) || FileSet.canonical(path) !== path)) return mismatch
+  if (new Set(outputs.map((output) => output.path)).size !== outputs.length) return mismatch
+  if (new Set(trees.map((tree) => tree.path)).size !== trees.length) return mismatch
+  const removes = descriptor.removes ?? []
+  for (const output of outputs) {
+    if (output.digest === null) {
+      if (!removes.includes(output.path)) return mismatch
+    } else if (
+      removes.includes(output.path) ||
+      !descriptor.writeSet.some((entry) => FileSet.overlaps(entry, output.path))
+    ) return mismatch
+  }
+  // A tree root authorizes pruning its entire subtree. A glob or a narrower
+  // tree is never sufficient authority for a recorded broader pruning root.
+  const declaredTrees = descriptor.writeSet.filter(FileSet.isTreeArtifact)
+  // A declared tree root is a directory, never a replayable file or deletion.
+  if (declaredTrees.some((entry) => outputs.some((output) => output.path === entry.path))) return mismatch
+  if (trees.some((tree) => !declaredTrees.some((entry) => entry.path === tree.path))) return mismatch
+  if (declaredTrees.some((entry) => !trees.some((tree) => tree.path === entry.path))) return mismatch
+  if (removes.some((path) => !outputs.some((output) => output.path === path && output.digest === null))) return mismatch
+  if (
+    descriptor.writeSet.some((entry) =>
+      typeof entry === "string" && !outputs.some((output) => output.path === entry && output.digest !== null)
+    )
+  ) return mismatch
+  return { _tag: "ReplayOutputs" }
+}

package/src/internal/CachePublication.ts

@@ -0,0 +1,153 @@
+/**
+ * The two-tier step-cache protocol, extracted out of `ActionPersistence`.
+ *
+ * `CacheAdmission` classifies the evidence that permits publication;
+ * `ActionPersistence` supplies the measured facts and durable ordering. This module owns *how* it becomes
+ * shareable and how a shared hit is made replayable, which is the half a
+ * two-tier composition has to be able to inject into:
+ *
+ * - {@link publishArtifacts} runs before the cache entry is written, so an
+ *   entry is never observable while an artifact it references is missing
+ *   (`reference/bazel/.../remote/UploadManifest.java:630-633`).
+ * - {@link publishEntry} runs after the transaction that made the local row
+ *   durable, so the shared entry lands last of all — and outside the write
+ *   transaction.
+ * - {@link hydrateArtifacts} runs after a replay refused for a *missing*
+ *   artifact — the one refusal a shared tier can repair — so the replay is
+ *   retried once against fetched bytes instead of falling straight through to
+ *   a real execution.
+ *
+ * The seams are `ArtifactSync` and `CacheSync`, both resolved optionally: a
+ * composition with no shared tier gets the `makeLocal()` implementations, whose
+ * publish steps are no-ops and whose `hydrate` reports nothing was fetched.
+ * That is why a purely local engine pays nothing for this file.
+ *
+ * Every host call here stays OUTSIDE the `DurableWriter` transaction, like the
+ * Jj snapshot and the boundary prepare/settle: a host call must never be held
+ * across a write transaction.
+ *
+ * Neither publication step can fail a run. A shared cache is an accelerator,
+ * and by the time these run the result is already durably recorded on this
+ * host — failing a completed run because an optional tier is unreachable trades
+ * a real result for an unavailable one. Both report the refusal instead, and
+ * the caller journals it, exactly as an unverified read set is journaled rather
+ * than swallowed.
+ *
+ * @since 0.1.0
+ */
+
+import type { CacheStore } from "@smthrs/step-cache"
+import * as Cause from "effect/Cause"
+import type * as Crypto from "effect/Crypto"
+import * as Effect from "effect/Effect"
+import * as Option from "effect/Option"
+import * as ArtifactSync from "../ArtifactSync.ts"
+import * as CacheSync from "../CacheSync.ts"
+import * as StepBoundary from "../StepBoundary.ts"
+
+/** Resolves the configured protocol, defaulting to the single-tier one. */
+const sync: Effect.Effect<ArtifactSync.Service> = Effect.map(
+  Effect.serviceOption(ArtifactSync.ArtifactSync),
+  (configured) => Option.isSome(configured) ? configured.value : ArtifactSync.makeLocal()
+)
+
+/** Resolves the configured entry publisher, defaulting to the single-tier one. */
+const entrySync: Effect.Effect<CacheSync.Service> = Effect.map(
+  Effect.serviceOption(CacheSync.CacheSync),
+  (configured) => Option.isSome(configured) ? configured.value : CacheSync.makeLocal()
+)
+
+/**
+ * Why a recorded result could not be shared, when it could not be.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export interface Unshareable {
+  /** Which half of the two-tier publication refused. */
+  readonly stage: "artifacts" | "entry"
+  readonly message: string
+}
+
+/**
+ * Makes every artifact the evidence references durable in the shared tier,
+ * reporting a refusal rather than failing.
+ *
+ * Called immediately before the transaction that records the cache entry, and
+ * never inside it. A refusal means the entry must not be published to the
+ * shared tier — the whole point of the ordering constraint — but says nothing
+ * about the *local* row, which stays as valid as it ever was.
+ *
+ * @since 0.1.0
+ * @category protocol
+ */
+export const publishArtifacts = (
+  evidence: StepBoundary.BoundaryEvidence | undefined
+): Effect.Effect<Unshareable | undefined, never, Crypto.Crypto> =>
+  Effect.gen(function*() {
+    if (evidence === undefined) return undefined
+    const digests = StepBoundary.referencedDigests(evidence)
+    if (digests.length === 0) return undefined
+    const protocol = yield* sync
+    return yield* protocol.publish(digests).pipe(
+      Effect.as(undefined),
+      Effect.catch((failure) => Effect.succeed<Unshareable>({ stage: "artifacts", message: failure.message }))
+    )
+  })
+
+/**
+ * Publishes the recorded entry to the shared step-result tier, reporting a
+ * refusal rather than failing.
+ *
+ * Called only after {@link publishArtifacts} reported success and only after
+ * the transaction that made the local row durable has committed: blobs first,
+ * entry last, and no network round trip held across a write transaction.
+ *
+ * @since 0.1.0
+ * @category protocol
+ */
+export const publishEntry = (
+  entry: CacheStore.CacheEntry
+): Effect.Effect<Unshareable | undefined> =>
+  Effect.gen(function*() {
+    const protocol = yield* entrySync
+    const refused = yield* protocol.publishEntry(entry)
+    return Option.isSome(refused) ? { stage: "entry" as const, message: refused.value.message } : undefined
+  })
+
+/**
+ * Fetches the artifacts a replay could not resolve locally, reporting whether
+ * the replay is now worth retrying.
+ *
+ * @since 0.1.0
+ * @category protocol
+ */
+export const hydrateArtifacts = (
+  evidence: StepBoundary.BoundaryEvidence
+): Effect.Effect<boolean, never, Crypto.Crypto> =>
+  Effect.gen(function*() {
+    const digests = StepBoundary.referencedDigests(evidence)
+    if (digests.length === 0) return false
+    const protocol = yield* sync
+    return yield* protocol.hydrate(digests)
+  })
+
+/**
+ * The classification a `replayOutputs` failure needs before it can be
+ * repaired: a {@link StepBoundary.MissingArtifact} means the bytes are simply
+ * not on this host, which a shared tier may fix. Anything else — a corrupt
+ * address, a host refusal — is not a fetchable condition.
+ *
+ * @since 0.1.0
+ * @category protocol
+ */
+export const replayMissingArtifact = (
+  cause: Cause.Cause<unknown>
+): StepBoundary.MissingArtifact | undefined => {
+  for (const reason of cause.reasons) {
+    if (Cause.isFailReason(reason) && reason.error instanceof StepBoundary.MissingArtifact) {
+      return reason.error
+    }
+  }
+  return undefined
+}

package/src/internal/CallFacts.ts

@@ -0,0 +1,87 @@
+/** Native call facts written by the action owner, never a harness stream consumer.
+ * @since 1.0.0
+ */
+
+import { CallFact, JournalEvent } from "@smthrs/journal"
+import { DerivedKey, digest } from "@smthrs/keys"
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as Option from "effect/Option"
+import * as Schema from "effect/Schema"
+
+const maxBytes = 65_536
+const encoder = new TextEncoder()
+const bounded = (value: Schema.Json) =>
+  Effect.gen(function*() {
+    const bytes = encoder.encode(typeof value === "string" ? value : JSON.stringify(value)).byteLength
+    return bytes <= maxBytes ? value : {
+      truncated: true,
+      bytes,
+      digest: digest(yield* Schema.decodeUnknownEffect(DerivedKey)(value).pipe(Effect.orDie))
+    }
+  })
+
+/** No new action identity, callback, table or delivery queue is introduced.
+ * @category constructors
+ * @since 1.0.0
+ */
+export const make = (action: unknown, runId: string) => {
+  const annotations = typeof action === "object" && action !== null && "annotations" in action
+    ? action.annotations as Context.Context<never>
+    : undefined
+  const annotation = annotations === undefined ? Option.none() : Context.getOption(annotations, CallFact.Annotation)
+  const resultSchema =
+    typeof action === "object" && action !== null && "successSchema" in action && Schema.isSchema(action.successSchema)
+      ? action.successSchema as Schema.Codec<unknown, unknown>
+      : undefined
+  const record = (phase: "invoked" | "settled", outcome?: unknown) =>
+    Effect.gen(function*() {
+      if (Option.isNone(annotation) || annotation.value.phase !== phase) return undefined
+      const call = yield* Schema.decodeUnknownEffect(CallFact.Call)(annotation.value.call).pipe(Effect.orDie)
+      // Production record boundaries carry a service-free success codec. Check
+      // it before publishing an older encoded row as a deliverable result: the
+      // reader would refuse a corrupt row even when its broad JSON shape fits.
+      if (phase === "settled" && resultSchema !== undefined) {
+        yield* Schema.decodeUnknownEffect(Schema.toCodecJson(resultSchema))(outcome).pipe(Effect.orDie)
+      }
+      const result = phase === "settled"
+        ? yield* Schema.decodeUnknownEffect(CallFact.Result)(outcome).pipe(Effect.orDie)
+        : undefined
+      const coordinates = {
+        version: 1 as const,
+        callId: call.callId,
+        identity: call.identity,
+        flowName: call.flowName
+      }
+      const payload: CallFact.Fact = result === undefined
+        ? {
+          ...coordinates,
+          phase: "invoked",
+          input: yield* bounded(call.input),
+          // Never bounded: the display projection is four short fields its
+          // owner already validated, and a truncation marker in its place
+          // would read as a declaration that claimed something unreadable.
+          ...(call.descriptor === undefined ? {} : { descriptor: call.descriptor })
+        }
+        : {
+          ...coordinates,
+          phase: "settled",
+          outcome: result.outcome,
+          value: yield* bounded(result.value),
+          ...(result.message === undefined ? {} : { message: yield* bounded(result.message) }),
+          ...(result.code === undefined ? {} : { code: result.code })
+        }
+      return new JournalEvent.Input({
+        runId: JournalEvent.RunId.make(runId),
+        sourceId: JournalEvent.SourceId.make(`call-fact-v1:${call.callId}:${phase}`),
+        sourceSeq: JournalEvent.SourceSeq.make(0),
+        eventType: CallFact.eventType,
+        payload
+      })
+    })
+  return {
+    settles: Option.isSome(annotation) && annotation.value.phase === "settled",
+    invoked: record("invoked"),
+    settled: (outcome: unknown) => record("settled", outcome)
+  }
+}

package/src/internal/CopiedRecord.ts

@@ -0,0 +1,201 @@
+/**
+ * Read-only validation of a journal record another lineage recorded under
+ * this run's producer identity.
+ *
+ * A time-travel fork copies the parent's journal rows verbatim: the copied
+ * record keeps the producer identity, but its payload names the parent run and
+ * its `meta.lineageId` is the parent's lineage, so this run's re-emission of
+ * the same record raises `idempotency_conflict` instead of collapsing into a
+ * `Duplicate`. That conflict is benign ONLY when the occupying record is the
+ * record being emitted, recorded by a validated ancestor. Anything else in the
+ * slot — an unrelated event type, another attempt's coordinates, the opposite
+ * terminal state, a record with no retained fork ancestry — is the conflict
+ * the journal reported, and it fails closed.
+ * @since 1.0.0-rc.0
+ */
+
+import { canonicalize, isRecord } from "@smthrs/canonical"
+import { FlowEngine } from "@smthrs/engine"
+import { Journal, JournalEvent } from "@smthrs/journal"
+import type { RunStore } from "@smthrs/run-store"
+import * as Effect from "effect/Effect"
+import * as Schema from "effect/Schema"
+import { RunState } from "../RunState.ts"
+
+/** Paged lookup retains only the matching record, including unexpected event types.
+ * @category accessors
+ * @since 1.0.0-rc.0
+ */
+export const find = (
+  journal: Journal.Service,
+  runId: string,
+  predicate: (entry: JournalEvent.Entry) => boolean
+): Effect.Effect<JournalEvent.Entry | undefined, Journal.JournalError> =>
+  Effect.gen(function*() {
+    let after: JournalEvent.Seq | undefined
+    while (true) {
+      const page = yield* journal.entries({
+        runId: JournalEvent.RunId.make(runId),
+        limit: 128,
+        ...after === undefined ? {} : { after }
+      })
+      const entry = page.entries.find(predicate)
+      if (entry !== undefined) return entry
+      if (!page.hasMore) return undefined
+      const next = page.entries.at(-1)?.seq
+      if (next === undefined || (after !== undefined && next <= after)) {
+        return yield* Effect.fail(
+          new Journal.JournalError({ code: "read_failed", message: "cache history cursor did not advance" })
+        )
+      }
+      after = next
+    }
+  })
+
+const sameJson = (left: unknown, right: unknown): boolean =>
+  left === undefined || right === undefined
+    ? left === right
+    : canonicalize(left, { loneSurrogates: "escape" }) === canonicalize(right, { loneSurrogates: "escape" })
+
+/** The record as the named ancestor would have emitted it: the payload's own
+ * `runId` and the lineage in `meta` name that run instead of this one. */
+const asEmittedBy = (record: JournalEvent.Input, runId: string, ancestor: string) => ({
+  payload: isRecord(record.payload) && record.payload["runId"] === runId
+    ? { ...record.payload, runId: ancestor }
+    : record.payload,
+  meta: isRecord(record.meta) ? { ...record.meta, lineageId: FlowEngine.Lineage.root(ancestor) } : record.meta
+})
+
+/**
+ * Proves every retained fork prefix between a copied entry and its original.
+ * Callers retain their payload and metadata equality rules.
+ * @category accessors
+ * @since 1.0.0
+ */
+export const prove = (options: {
+  readonly journal: Journal.Service
+  readonly runs: RunStore.Service
+  readonly runId: string
+  readonly entry: JournalEvent.Entry
+  readonly conflict: Journal.JournalError
+  readonly same?: (left: unknown, right: unknown) => boolean
+}) =>
+  Effect.gen(function*() {
+    const { conflict, journal, runs, same = sameJson } = options
+    let entry = options.entry
+    const source = (candidate: JournalEvent.Entry) =>
+      candidate.sourceId === options.entry.sourceId && candidate.sourceSeq === options.entry.sourceSeq
+    let child = options.runId
+    const visited = new Set<string>()
+    while (!visited.has(child)) {
+      visited.add(child)
+      const run = yield* runs.get(child).pipe(Effect.mapError((cause) =>
+        new Journal.JournalError({
+          code: "idempotency_conflict",
+          message: conflict.message,
+          cause
+        })
+      ))
+      const parent = run.parentRunId
+      if (parent === null) return yield* Effect.fail(conflict)
+      const copied = entry
+      const marker = yield* find(journal, child, (candidate) => {
+        const value = candidate.payload as {
+          readonly childRunId?: unknown
+          readonly parentRunId?: unknown
+          readonly forkJournalOffset?: unknown
+          readonly copiedFromRunId?: unknown
+        } | null
+        return candidate.eventType === "flows.time-travel.fork-created" &&
+          candidate.sourceId === "flows/time-travel/fork" && Number(candidate.sourceSeq) === candidate.seq &&
+          value?.childRunId === child &&
+          value.parentRunId === parent && typeof value.forkJournalOffset === "number" &&
+          Number.isSafeInteger(value.forkJournalOffset) && value.forkJournalOffset >= copied.seq &&
+          candidate.seq === value.forkJournalOffset + 1 &&
+          (value.copiedFromRunId === undefined || typeof value.copiedFromRunId === "string") &&
+          same(value, {
+            childRunId: child,
+            parentRunId: parent,
+            forkJournalOffset: value.forkJournalOffset,
+            ...(value.copiedFromRunId === undefined ? {} : { copiedFromRunId: value.copiedFromRunId })
+          })
+      })
+      if (marker === undefined) return yield* Effect.fail(conflict)
+      const copiedFrom = (marker.payload as { copiedFromRunId?: string }).copiedFromRunId
+      const sourceRunId = copiedFrom ?? parent
+      if (copiedFrom !== undefined) {
+        const decodeState = Schema.decodeUnknownEffect(Schema.fromJsonString(RunState))
+        const state = yield* decodeState(run.stateJson).pipe(Effect.mapError(() => conflict))
+        const sourceRun = yield* runs.get(copiedFrom).pipe(Effect.mapError(() => conflict))
+        const sourceState = yield* decodeState(sourceRun.stateJson).pipe(Effect.mapError(() => conflict))
+        const parentRun = yield* runs.get(parent).pipe(Effect.mapError(() => conflict))
+        if (
+          !state.forkKeyRunIds?.includes(copiedFrom) || state.parentExecutionId !== parent ||
+          sourceState.parentExecutionId === undefined || parentRun.parentRunId !== sourceState.parentExecutionId
+        ) return yield* Effect.fail(conflict)
+        // A carried origin must belong to the run the new parent forked.
+        // Its receipt alone never grants reuse of an unrelated run's records.
+        const parentFork = yield* find(journal, parent, (candidate) => {
+          const value = candidate.payload as { childRunId?: unknown; parentRunId?: unknown } | null
+          return candidate.eventType === "flows.time-travel.fork-created" &&
+            candidate.sourceId === "flows/time-travel/fork" &&
+            Number(candidate.sourceSeq) === candidate.seq &&
+            value?.childRunId === parent && value.parentRunId === sourceState.parentExecutionId
+        })
+        if (parentFork === undefined) return yield* Effect.fail(conflict)
+      }
+      const cutoff = yield* find(journal, sourceRunId, (candidate) => candidate.seq === marker.seq - 1)
+      if (
+        cutoff === undefined ||
+        !same(marker.meta, { lineageId: (cutoff.meta as { lineageId?: unknown } | null)?.lineageId })
+      ) {
+        return yield* Effect.fail(conflict)
+      }
+      const original = yield* find(journal, sourceRunId, source)
+      if (
+        original === undefined || original.seq !== copied.seq || original.emittedAtMs !== copied.emittedAtMs ||
+        original.eventType !== copied.eventType || !same(original.payload, copied.payload) ||
+        !same(original.meta, copied.meta)
+      ) return yield* Effect.fail(conflict)
+      if (isRecord(original.meta) && original.meta["lineageId"] === FlowEngine.Lineage.root(sourceRunId)) {
+        return { ancestor: sourceRunId, record: original }
+      }
+      child = sourceRunId
+      entry = original
+    }
+    return yield* Effect.fail(conflict)
+  })
+
+/**
+ * Accepts the occupying record only when it is `record` as a validated
+ * copied-lineage ancestor emitted it; otherwise fails with `conflict`.
+ *
+ * The shared ancestry proof requires, at each hop, the child's fork marker above the copied prefix, the parent row the
+ * marker was cut at, and the parent's own copy of the record at the same
+ * `seq`, identical in content. The walk ends at the run whose lineage the
+ * record carries, and that run's original must be the record being emitted
+ * under that run's identity: same event type, same payload with the ancestor
+ * in place of this run, same meta.
+ * @category accessors
+ * @since 1.0.0-rc.0
+ */
+export const accept = (options: {
+  readonly journal: Journal.Service
+  readonly runs: RunStore.Service
+  readonly runId: string
+  readonly record: JournalEvent.Input
+  readonly conflict: Journal.JournalError
+}): Effect.Effect<void, Journal.JournalError> =>
+  Effect.gen(function*() {
+    const { conflict, journal, record } = options
+    const sourceSeq = record.sourceSeq ?? 0
+    const source = (entry: JournalEvent.Entry) =>
+      entry.sourceId === record.sourceId && Number(entry.sourceSeq) === Number(sourceSeq)
+    const entry = yield* find(journal, options.runId, source)
+    if (entry === undefined || entry.eventType !== record.eventType) return yield* Effect.fail(conflict)
+    const original = yield* prove({ ...options, entry })
+    const expected = asEmittedBy(record, options.runId, original.ancestor)
+    return sameJson(original.record.payload, expected.payload) && sameJson(original.record.meta, expected.meta)
+      ? undefined
+      : yield* Effect.fail(conflict)
+  })