@smthrs/engine-store 0.0.0-stage → 1.0.0-rc.3

package/src/internal/NodeJournal.ts

@@ -0,0 +1,340 @@
+/**
+ * The interpreter's node records, as journal entries.
+ *
+ * `@smthrs/flow` builds a record for the graph it drove and for every node it
+ * scheduled and settled, and says nothing about where they go. This module is
+ * the durable store's answer: one journal entry per record, addressed by the
+ * record's own `sourceId`, typed by the payload schemas `@smthrs/journal`
+ * publishes, and carrying the same event types the plan scheduler writes so a
+ * reader folds both executors with one decoder.
+ *
+ * Two facts live here rather than in the interpreter. The journal's producer
+ * identity is `(runId, sourceId, sourceSeq)`, so a resumed walk that
+ * re-derives the same `sourceId` collapses onto the row the first walk wrote:
+ * that is the exactly-once contract, and it holds because the identity is the
+ * record's, never a fresh one minted per observation. And a declaration site
+ * is made repo-relative here, because only the writer knows the root: a
+ * journal is read on machines that did not write it, and an absolute path is
+ * at best noise and at worst an operator's home directory published into a
+ * run's history.
+ *
+ * @since 0.1.0
+ */
+
+import type { FlowRuntime } from "@smthrs/flow"
+import { EngineEvent, JournalEvent, Redaction } from "@smthrs/journal"
+import type * as Schema from "effect/Schema"
+import * as JournalRecords from "./JournalRecords.ts"
+
+/**
+ * Where the records of one run are addressed from.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export interface Scope {
+  readonly runId: string
+  /** The host's journal source, which every record of this run hangs under. */
+  readonly sourceId: string
+  readonly lineageId: string
+  /**
+   * The directory a declaration path is made relative to, when one is known.
+   *
+   * `undefined` means no path is recorded at all. That is the honest answer
+   * for a host that cannot say where its sources live, and it is better than
+   * a path a reader would have to guess the meaning of.
+   */
+  readonly root?: string | undefined
+  /**
+   * The revision the sources those declaration paths point into were read
+   * at, when the host can name one.
+   *
+   * It lives here for the reason `root` does: a journal is read on machines
+   * that did not write it, and only the writer knows which tree it loaded.
+   * A path alone says where a node was declared and never which bytes were
+   * at that line, so a reader without this opens no code at all rather than
+   * whatever the working tree holds when they look (D-068).
+   */
+  readonly sourceRevision?: string | undefined
+}
+
+/**
+ * A declaration path as a journal may carry it: relative to the root, or
+ * nothing.
+ *
+ * The rule lives with the schema that refuses an absolute path
+ * (`@smthrs/journal`'s `EngineEvent.DeclaredAt`), so every writer of a
+ * declaration site obeys it the same way. It is named here because this is
+ * the module that applies it, and because the store's own tests are written
+ * against it.
+ *
+ * @since 0.1.0
+ * @category accessors
+ */
+export const relativePath = EngineEvent.relativePath
+
+/**
+ * A value the journal will serialize, typed as what it will be.
+ *
+ * A node's effects are JSON by construction, but an optional member spelled
+ * `undefined` is not assignable to `Json` under `exactOptionalPropertyTypes`,
+ * and serialization is exactly what drops it. This is the journal's own
+ * encoding performed one step early, so the type says what the stored value
+ * is rather than asserting it. @private
+ */
+const asJson = (value: unknown): typeof Schema.Json.Type => JSON.parse(JSON.stringify(value))
+
+/**
+ * The directory declaration paths are recorded relative to.
+ *
+ * `process` is absent in a worker and in a browser, and a host that cannot
+ * name its own working directory records no declaration path at all rather
+ * than one a reader would have to guess the meaning of. `host` is the global
+ * object by default and is named here so both answers are reachable without
+ * taking the real one away from the runtime.
+ *
+ * @since 0.1.0
+ * @category accessors
+ */
+export const hostRoot = (
+  explicit: string | undefined,
+  host: { readonly process?: { readonly cwd?: () => string } | undefined } = globalThis
+): string | undefined => {
+  if (explicit !== undefined) return explicit
+  return typeof host.process?.cwd === "function" ? host.process.cwd() : undefined
+}
+
+const summary = (scope: Scope, node: FlowRuntime.NodeSummary) => {
+  const path = node.declaredAt === undefined ? undefined : relativePath(scope.root, node.declaredAt.path)
+  return {
+    id: node.id,
+    kind: node.kind,
+    dependsOn: node.dependsOn,
+    tier: node.tier,
+    ...(node.action === undefined ? {} : { action: node.action }),
+    ...(node.effects === undefined ? {} : { effects: asJson(node.effects) }),
+    ...(path === undefined ? {} : { declaredAt: { path, line: node.declaredAt!.line } })
+  }
+}
+
+const graph = (scope: Scope, record: FlowRuntime.PlanRecorded | FlowRuntime.SubgraphAppended) => ({
+  nodes: record.nodes.map((node) => summary(scope, node)),
+  edges: record.edges,
+  // On every page, because every page carries its own sites.
+  ...(scope.sourceRevision === undefined ? {} : { sourceRevision: scope.sourceRevision })
+})
+
+/**
+ * How many bytes of a node's settled value a journal row keeps.
+ *
+ * A node record is a small row beside the plan record's 12,000-byte page
+ * budget, and it is written once per node of every graph: a value kept whole
+ * would put an agent transcript in a permanent row and a projection would
+ * clip it without saying so. Two kibibytes is enough to read what a node
+ * produced and small enough that the settlement stays a summary.
+ *
+ * @since 0.1.0
+ * @category constants
+ */
+export const maximumResultBytes = 2_048
+
+/**
+ * How large a settled value may be before it is summarized at all.
+ *
+ * Redacting a value rebuilds every member of it and runs the textual rules
+ * over every string, on the journal write path, for every node of every
+ * graph. Past this ceiling the walk is the cost rather than the row, and the
+ * honest answer is the size alone: the encoding is measured, `preview` is
+ * empty, and `truncated` says the text was cut to nothing. Anything below it
+ * is redacted whole and then cut.
+ *
+ * @since 0.1.0
+ * @category constants
+ */
+export const maximumSummarizedBytes = 65_536
+
+/** The redactor a node summary is scrubbed with, built once. @private */
+const redactor = Redaction.make({ onTooDeep: "name" })
+
+/** UTF-8 bytes of a string, without allocating the encoding. @private */
+const utf8Bytes = (text: string): number => {
+  let bytes = 0
+  for (let index = 0; index < text.length; index++) {
+    const code = text.charCodeAt(index)
+    if (code < 0x80) bytes += 1
+    else if (code < 0x800) bytes += 2
+    else if (code >= 0xd800 && code <= 0xdbff && index + 1 < text.length) {
+      bytes += 4
+      index++
+    } else bytes += 3
+  }
+  return bytes
+}
+
+/**
+ * The longest prefix of `text` within `limit` UTF-8 bytes, never splitting a
+ * pair.
+ *
+ * One cut, taken where the walk stopped, because the walk stops for two
+ * reasons and only one of them can happen: the caller asks only for text it
+ * has already measured as over the limit, so a second exit for text that fits
+ * whole would be a line no run reaches. @private
+ */
+const clampToBytes = (text: string, limit: number): string => {
+  let bytes = 0
+  let index = 0
+  while (index < text.length) {
+    const code = text.charCodeAt(index)
+    const pair = code >= 0xd800 && code <= 0xdbff && index + 1 < text.length
+    const size = code < 0x80 ? 1 : code < 0x800 ? 2 : pair ? 4 : 3
+    if (bytes + size > limit) break
+    bytes += size
+    index += pair ? 2 : 1
+  }
+  return text.slice(0, index)
+}
+
+/**
+ * What a node settled with, redacted and then bounded.
+ *
+ * REDACTION COMES FIRST. `Redaction.redact` is the journal's own write-path
+ * scrubber, and it is the one that reads a field NAME: `{ apiKey: "sk-..." }`
+ * is a credential because of the key, which no rule over encoded text can see
+ * once the value is a string inside a larger string. Running it over the
+ * decoded value also collapses cycles and names functions, so the encoding
+ * that follows always succeeds. Cutting the text first would be worse than
+ * useless: a credential split across the boundary is a credential the textual
+ * rules no longer recognise, and half of it would be kept.
+ *
+ * A value that will not encode even after redaction — a bigint, a value whose
+ * own `toJSON` throws — gets no summary at all rather than a fabricated one.
+ *
+ * `bytes` is the size of the encoding `preview` was cut from, which for a
+ * value past {@link maximumSummarizedBytes} is the value's own encoding: that
+ * branch keeps nothing, so there is no second number to disagree with.
+ *
+ * @since 0.1.0
+ * @category constructors
+ */
+export const resultSummary = (value: unknown): {
+  readonly preview: string
+  readonly bytes: number
+  readonly truncated: boolean
+} | undefined => {
+  // The size first, from the value's own encoding, because it is the cheap
+  // half: a value this row will not keep is named by its size and never
+  // walked. A value that will not encode at all — a cycle, a bigint, a
+  // throwing `toJSON` — says nothing here, and the redactor below is asked,
+  // because it collapses cycles and names what it cannot rewrite.
+  let raw: string | undefined
+  try {
+    raw = JSON.stringify(value)
+  } catch {
+    raw = undefined
+  }
+  if (raw !== undefined) {
+    const rawBytes = utf8Bytes(raw)
+    if (rawBytes > maximumSummarizedBytes) return { preview: "", bytes: rawBytes, truncated: true }
+  }
+  let encoded: string | undefined
+  try {
+    encoded = JSON.stringify(redactor(value))
+  } catch {
+    return undefined
+  }
+  if (encoded === undefined) return undefined
+  const bytes = utf8Bytes(encoded)
+  if (bytes > maximumSummarizedBytes) return { preview: "", bytes, truncated: true }
+  if (bytes <= maximumResultBytes) return { preview: encoded, bytes, truncated: false }
+  return { preview: clampToBytes(encoded, maximumResultBytes), bytes, truncated: true }
+}
+
+/**
+ * The journal entry one node record is written as.
+ *
+ * @since 0.1.0
+ * @category constructors
+ */
+export const entry = (scope: Scope, record: FlowRuntime.NodeRecord): JournalEvent.Input => {
+  const options = {
+    runId: scope.runId,
+    sourceId: `${scope.sourceId}/${record.sourceId}`,
+    lineageId: scope.lineageId,
+    // Without an explicit sequence the journal ALLOCATES the next one for the
+    // source, so a resumed walk would write a second row for every node it
+    // re-derived. Pinning it to zero makes the re-emission an exact producer
+    // retry, which is the whole of the exactly-once contract.
+    sourceSeq: 0,
+    // The sequence is derived from the record's own identity, so a collision
+    // is the same node observed twice rather than two nodes wearing one
+    // address. What differs between the observations is metadata ABOUT the
+    // observation — a node the first walk BUILT settles CLEAN on the walk
+    // that replayed it — and the first row, the one that says what actually
+    // happened, stands.
+    dedupe: "identity" as const
+  }
+  switch (record._tag) {
+    case "PlanRecorded":
+      return JournalRecords.planRecorded(options, {
+        flow: record.flow,
+        generation: record.generation,
+        // The node COUNT of the whole graph, which is what this field has
+        // always meant; the node LIST rides beside it as `graph`.
+        nodes: record.nodeCount,
+        page: record.page,
+        pages: record.pages,
+        graph: graph(scope, record)
+      })
+    case "SubgraphAppended":
+      return JournalRecords.subgraphAppended(options, {
+        flow: record.flow,
+        generation: record.generation,
+        nodeIds: record.nodes.map((node) => node.id),
+        page: record.page,
+        pages: record.pages,
+        graph: graph(scope, record)
+      })
+    case "NodeScheduled":
+      return JournalRecords.nodeScheduled(options, {
+        nodeId: record.nodeId,
+        kind: record.kind,
+        attempt: record.attempt,
+        ...(record.action === undefined ? {} : { action: record.action })
+      })
+    case "NodeSettled": {
+      const result = record.value === undefined ? undefined : resultSummary(record.value)
+      return JournalRecords.nodeSettled(options, {
+        nodeId: record.nodeId,
+        outcome: record.outcome,
+        attempts: record.attempts,
+        stepKeyDigests: record.stepKeyDigests,
+        ...(record.action === undefined ? {} : { action: record.action }),
+        ...(result === undefined ? {} : { result })
+      })
+    }
+  }
+}
+
+/**
+ * UTF-8 upper bound for the durable entry, including allocated envelope fields.
+ *
+ * Sequence/time reserve their largest JSON encodings so page boundaries never
+ * depend on the current journal position or clock, including after a resume.
+ * @since 1.0.0
+ * @category accessors
+ */
+export const encodedBytes = (scope: Scope, record: FlowRuntime.NodeRecord): number => {
+  const input = entry(scope, record)
+  const sourceSeq = JournalEvent.SourceSeq.make(0)
+  return new TextEncoder().encode(JSON.stringify({
+    runId: input.runId,
+    seq: Number.MAX_SAFE_INTEGER - 1,
+    eventId: JournalEvent.makeEventId(input.runId, input.sourceId, sourceSeq),
+    sourceId: input.sourceId,
+    sourceSeq,
+    emittedAtMs: Number.MAX_VALUE,
+    eventType: input.eventType,
+    payload: redactor(input.payload),
+    meta: redactor(input.meta)
+  })).byteLength
+}

package/src/internal/Ordering.ts

@@ -0,0 +1,14 @@
+/**
+ * Host-locale-independent UTF-16 code-unit ordering for durable material.
+ *
+ * @since 1.0.0
+ */
+
+/**
+ * Compares two strings by JavaScript's locale-independent UTF-16 relational
+ * order.
+ *
+ * @category ordering
+ * @since 1.0.0
+ */
+export const compareText = (left: string, right: string): number => left < right ? -1 : left > right ? 1 : 0

package/src/internal/PlanInputs.ts

@@ -0,0 +1,62 @@
+/**
+ * Cross-check stored or newly measured observations against a verified plan.
+ * @since 1.0.0
+ */
+
+import { FileSet, type Plan } from "@smthrs/plan"
+import { Effect, Schema } from "effect"
+import * as PlanInputStore from "../PlanInputStore.ts"
+
+/**
+ * Validates without publishing or mutating either the old pins or observations.
+ * @since 1.0.0
+ * @category validation
+ */
+export const validate = (
+  snapshot: PlanInputStore.Snapshot,
+  nodes: ReadonlyArray<Plan.PlanNode>,
+  pinned: ReadonlyMap<string, string>,
+  writerScopesBefore: (node: Plan.PlanNode) => ReadonlyArray<FileSet.Entry>
+): Effect.Effect<Map<string, string>, PlanInputStore.PlanInputError> =>
+  Effect.gen(function*() {
+    const corrupt = (message: string) =>
+      Effect.fail(new PlanInputStore.PlanInputError({ code: "corrupt_state", message }))
+    if (snapshot.nodes.length !== nodes.length) return yield* corrupt("source observation node count changed")
+    const nextPins = new Map(pinned)
+    const newPins = new Set<string>()
+    for (const pin of snapshot.pins) {
+      if (nextPins.has(pin.path)) return yield* corrupt("a later generation replaced an existing source pin")
+      nextPins.set(pin.path, pin.digest)
+      newPins.add(pin.path)
+    }
+    for (let index = 0; index < nodes.length; index++) {
+      const node = nodes[index]!
+      const recorded = snapshot.nodes[index]!
+      const reads = FileSet.expandReads(node.effects.reads)
+      const writerScopes = reads.length === 0 ? [] : writerScopesBefore(node)
+      if (recorded.id !== node.id || recorded.key !== node.key || recorded.reads.length !== reads.length) {
+        return yield* corrupt("source observation does not match its plan node")
+      }
+      for (let readIndex = 0; readIndex < reads.length; readIndex++) {
+        const expected = Schema.decodeUnknownSync(FileSet.ReadEntry)(reads[readIndex]!)
+        const read = recorded.reads[readIndex]!
+        if (JSON.stringify(read.entry) !== JSON.stringify(expected)) {
+          return yield* corrupt("source observation declaration changed")
+        }
+        if (typeof expected === "string") {
+          const source = !writerScopes.some((scope) => FileSet.overlaps(scope, expected))
+          if (read.sourcePaths.length !== (source ? 1 : 0)) {
+            return yield* corrupt("source observation selects the wrong file version")
+          }
+        }
+        for (const path of read.sourcePaths) {
+          if (!nextPins.has(path) || writerScopes.some((scope) => FileSet.overlaps(scope, path))) {
+            return yield* corrupt("source observation has a missing pin or preceding writer")
+          }
+          newPins.delete(path)
+        }
+      }
+    }
+    if (newPins.size > 0) return yield* corrupt("source observation contains unreferenced pins")
+    return nextPins
+  })

package/src/internal/PlanMerges.ts

@@ -0,0 +1,123 @@
+/** Durable merge recovery and generated identity, independent of dispatch.
+ * @since 1.0.0
+ */
+
+import { KeyMaterial, Plan } from "@smthrs/plan"
+import { Effect } from "effect"
+import * as PlanMergeStore from "../PlanMergeStore.ts"
+
+const invalid = (message: string, cause?: unknown) =>
+  new PlanMergeStore.PlanMergeError({
+    code: "corrupt_state",
+    message,
+    ...(cause === undefined ? {} : { cause })
+  })
+/** Chooses a free display ID; ownership comes from the durable decision, not this spelling.
+ * @since 1.0.0
+ * @category identity
+ */
+export const allocateId = (plan: Plan.Plan, nodeId: string): string => {
+  const occupied = new Set(plan.nodes.map((node) => node.id))
+  const base = `${nodeId}+merge`
+  let candidate = base
+  for (let suffix = 1; occupied.has(candidate); suffix++) candidate = `${base}#${suffix}`
+  return candidate
+}
+/** The single declaration used both when appending and when recovering an extension.
+ * @since 1.0.0
+ * @category constructors
+ */
+export const draft = (node: Plan.PlanNode, mergeId: string, winners: ReadonlyArray<string>): Plan.NodeDraft => ({
+  id: mergeId,
+  kind: "merge",
+  material: {
+    version: KeyMaterial.version,
+    kind: node.material.kind,
+    body: { merge: { stopped: node.id, winners } },
+    inputs: winners.map((id) => ({ _tag: "Pending", from: id })),
+    layers: [],
+    capabilities: []
+  },
+  effects: node.effects
+})
+/** Checks that an intent names a permissible stopped node and its actual conflict peers.
+ * @since 1.0.0
+ * @category validation
+ */
+export const validateIntent = (plan: Plan.Plan, intent: PlanMergeStore.Intent) =>
+  Effect.gen(function*() {
+    const node = plan.nodes.find((candidate) => candidate.id === intent.nodeId)
+    if (
+      node === undefined || node.key !== intent.nodeKey || node.material.kind === "irreversible" ||
+      (node.runtime !== "stop-merge" && !node.conflicts.some((conflict) => conflict.runtime === "stop-merge")) ||
+      intent.peers.some((id) => !node.conflicts.some((conflict) => conflict.with === id))
+    ) {
+      return yield* Effect.fail(invalid("merge intent does not match its approved stopped node"))
+    }
+    return node
+  })
+/** Recovers only recorded merge extensions, never unrelated/unapproved appended work.
+ * @since 1.0.0
+ * @category recovery
+ */
+export const recover = (candidate: Plan.Plan, decisions: ReadonlyArray<PlanMergeStore.Decision>) =>
+  Effect.gen(function*() {
+    const byGeneration = new Map(
+      decisions.flatMap((decision) =>
+        decision.completion === undefined ? [] : [[decision.completion.generation, decision] as const]
+      )
+    )
+    let plan = candidate
+    if (byGeneration.size > 0) {
+      // Rebuild each prefix once. Re-verifying every complete prefix separately
+      // would multiply the compiler's overlap work across a long merge history.
+      plan = yield* Plan.verify({
+        ...candidate,
+        generation: 0,
+        digest: candidate.baseDigest,
+        nodes: candidate.nodes.filter((node) => node.generation === 0)
+      })
+        .pipe(Effect.mapError((cause) => invalid("merge recovery has an invalid approved base", cause)))
+      const last = Math.max(candidate.generation, ...byGeneration.keys())
+      if (last > Plan.maximumPlanNodes) return yield* Effect.fail(invalid("merge generation exceeds the plan limit"))
+      for (let generation = 1; generation <= last; generation++) {
+        const decision = byGeneration.get(generation)
+        let drafts: ReadonlyArray<Plan.NodeDraft>
+        if (decision?.completion !== undefined) {
+          const stopped = yield* validateIntent(plan, decision.intent)
+          if (decision.completion.parentDigest !== plan.digest) {
+            return yield* Effect.fail(invalid("merge parent digest changed"))
+          }
+          drafts = [draft(stopped, decision.completion.mergeId, decision.completion.winners)]
+        } else {
+          drafts = candidate.nodes.filter((node) => node.generation === generation).map((node) => ({
+            id: node.id,
+            kind: node.kind,
+            material: node.material,
+            effects: node.effects,
+            priority: node.priority,
+            conflictStrategy: node.strategy,
+            runtimeStrategy: node.runtime
+          }))
+          if (drafts.length === 0) {
+            return yield* Effect.fail(invalid("merge recovery requires the approved intervening plan generations"))
+          }
+        }
+        plan = yield* Plan.append(plan, drafts).pipe(
+          Effect.mapError((cause) => invalid("stored merge extension cannot be reconstructed", cause))
+        )
+        if (
+          decision?.completion !== undefined &&
+          (plan.digest !== decision.completion.planDigest ||
+            Plan.generationNodes(plan)[0]!.key !== decision.completion.mergeKey)
+        ) {
+          return yield* Effect.fail(invalid("merge decision does not identify its generated node"))
+        }
+        if (generation === candidate.generation && plan.digest !== candidate.digest) {
+          return yield* Effect.fail(invalid("merge decisions disagree with the supplied plan"))
+        }
+      }
+    }
+    for (const decision of decisions) yield* validateIntent(plan, decision.intent)
+    return plan
+  })

package/src/internal/ProvenanceSlot.ts

@@ -0,0 +1,52 @@
+/**
+ * Read-only validation of the cache-provenance record occupying a producer
+ * slot this run re-emits into.
+ *
+ * A provenance producer identity (`cache:<digest>:<action>[:<run>:<seq>]`,
+ * `sourceSeq` 0) names ONE fact about one cache row — this key expired, this
+ * row was served, this replay failed — so a re-emission after a resume or in
+ * a fork child records the same fact with fresh measurements (`ageMs`, the
+ * clock, a later `reason`) and collapses onto the row already there. That
+ * collapse is legitimate only while the occupying record IS that fact: the
+ * same event type, the same key, the same action, and the same recorded row
+ * when the record names one. Anything else in the slot is the conflict the
+ * journal reported.
+ * @since 1.0.0-rc.0
+ */
+
+import type { Journal, JournalEvent } from "@smthrs/journal"
+import * as Effect from "effect/Effect"
+import * as CacheAgeHistory from "./CacheAgeHistory.ts"
+
+const identity = ["keyDigest", "action", "recordedRunId", "recordedEventSeq"] as const
+
+const isRecord = (value: unknown): value is Readonly<Record<string, unknown>> =>
+  typeof value === "object" && value !== null && !Array.isArray(value)
+
+/**
+ * Accepts the occupying record only when it states the same provenance fact
+ * as `record`; otherwise fails with `conflict`.
+ * @category accessors
+ * @since 1.0.0-rc.0
+ */
+export const accept = (options: {
+  readonly journal: Journal.Service
+  readonly runId: string
+  readonly record: JournalEvent.Input
+  readonly conflict: Journal.JournalError
+}): Effect.Effect<void, Journal.JournalError> =>
+  Effect.gen(function*() {
+    const { conflict, journal, record } = options
+    const sourceSeq = record.sourceSeq ?? 0
+    const entry = yield* CacheAgeHistory.find(
+      journal,
+      options.runId,
+      (candidate) => candidate.sourceId === record.sourceId && Number(candidate.sourceSeq) === Number(sourceSeq)
+    )
+    if (entry === undefined || entry.eventType !== record.eventType) return yield* Effect.fail(conflict)
+    const attempted = record.payload
+    const occupying = entry.payload
+    if (!isRecord(attempted) || !isRecord(occupying)) return yield* Effect.fail(conflict)
+    const same = identity.every((field) => !(field in attempted) || attempted[field] === occupying[field])
+    return same ? undefined : yield* Effect.fail(conflict)
+  })