@smthrs/engine-store 0.0.0-stage → 1.0.0-rc.4

package/src/internal/RunLifecycle.ts

@@ -0,0 +1,53 @@
+/**
+ * Logical-run resolution shared by cancellation and child lifetime decisions.
+ * Round membership comes from RunStore; child ownership comes from the durable
+ * DAG. A parent_run_id alone is not membership (forks also use that column).
+ * Call these reads inside the same write transaction as the lifecycle change.
+ *
+ * @since 1.0.0
+ */
+
+import type { RunStore } from "@smthrs/run-store"
+import * as Effect from "effect/Effect"
+import type * as DurableEngineState from "../DurableEngineState.ts"
+
+/**
+ * Binds logical-run reads to the same stores as the owning engine.
+ *
+ * @category constructors
+ * @since 1.0.0
+ */
+export const make = (store: RunStore.Service, state: DurableEngineState.Service) => {
+  const rounds = (runId: string) => store.lineage(runId)
+
+  /** All other rounds and linked descendants, including earlier-round children. */
+  const descendants = (runId: string): Effect.Effect<ReadonlyArray<string>, RunStore.RunStoreError> =>
+    Effect.gen(function*() {
+      const seen = new Set<string>([runId])
+      const resolved = new Set<string>()
+      const ordered: Array<string> = []
+      const frontier = [runId]
+      for (let index = 0; index < frontier.length; index++) {
+        const parentId = frontier[index]!
+        const admit = (id: string) => {
+          if (seen.has(id)) return
+          seen.add(id)
+          ordered.push(id)
+          frontier.push(id)
+        }
+        if (!resolved.has(parentId)) {
+          resolved.add(parentId)
+          for (const round of yield* rounds(parentId)) {
+            resolved.add(round.runId)
+            admit(round.runId)
+          }
+        }
+        // Even a missing row can retain edges during external archival. Walk
+        // those edges rather than interpreting a missing parent as no children.
+        for (const edge of yield* state.runChildren(parentId)) admit(edge.childId)
+      }
+      return ordered
+    })
+
+  return { rounds, descendants }
+}

package/src/internal/RunListing.ts

@@ -0,0 +1,210 @@
+/**
+ * Bounded, indexed run pages.
+ *
+ * @since 1.0.0
+ */
+
+import * as Dialect from "@smthrs/database/Dialect"
+import { RunStatus } from "@smthrs/run-store/RunStore"
+import { Cause, Effect, Schema } from "effect"
+import * as SqlClient from "effect/unstable/sql/SqlClient"
+import * as Read from "./ExecutionSnapshotRead.ts"
+import { RunCatalogError } from "./RunListingError.ts"
+
+/**
+ * Supported equality and inclusive creation-range filters.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export const Filters = Schema.Struct({
+  status: Schema.optionalKey(RunStatus),
+  flowName: Schema.optionalKey(Read.Identifier),
+  parentRunId: Schema.optionalKey(Schema.NullOr(Read.Identifier)),
+  lineageId: Schema.optionalKey(Read.Identifier),
+  waitingReason: Schema.optionalKey(Schema.NullOr(Read.Identifier)),
+  createdAfterMs: Schema.optionalKey(Read.Natural),
+  createdBeforeMs: Schema.optionalKey(Read.Natural)
+})
+/**
+ * Supported filters; lineageId also selects its original root.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export type Filters = typeof Filters.Type
+/**
+ * Options for a bounded run page. Numeric offsets are not cursors.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export interface ListRunsOptions {
+  readonly filters?: Filters
+  readonly cursor?: string
+  readonly limit?: number
+}
+/**
+ * One transaction's page and watermark; no total count.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export interface RunPage extends Read.Position {
+  readonly runs: ReadonlyArray<Read.Observed>
+  readonly cursor: string | null
+}
+/**
+ * Bounded catalog operations.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export interface Service {
+  readonly listRuns: (options?: ListRunsOptions) => Effect.Effect<RunPage, RunCatalogError>
+}
+/**
+ * Maximum page size.
+ *
+ * @category constants
+ * @since 1.0.0
+ */
+export const maximumPageSize = 200
+const Cursor = Schema.fromJsonString(Schema.Struct({
+  version: Schema.Literal(1),
+  source: Read.Source,
+  filters: Schema.String,
+  createdAtMs: Read.Natural,
+  runId: Read.Identifier
+}))
+const Options = Schema.Struct({
+  filters: Schema.optionalKey(Filters),
+  // Four 1,024-unit filter strings can expand sevenfold inside the JSON tuple
+  // string, plus a sixfold escaped 1,024-unit run ID and bounded scalar fields.
+  // 64 Ki UTF-16 units therefore admits every cursor this port can produce.
+  cursor: Schema.optionalKey(Schema.String.check(Schema.isMaxLength(65536))),
+  limit: Schema.optionalKey(Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: maximumPageSize })))
+})
+
+/**
+ * Builds the bounded read against the engine database.
+ *
+ * @category constructors
+ * @since 1.0.0
+ */
+export const make = () =>
+  Effect.gen(function*() {
+    const sql = yield* SqlClient.SqlClient
+    const listRuns: Service["listRuns"] = (options = {}) =>
+      Effect.gen(function*() {
+        const admitted = yield* Schema.decodeUnknownEffect(Options)(options, { onExcessProperty: "error" }).pipe(
+          Effect.mapError((cause) =>
+            new RunCatalogError({ code: "invalid_options", message: "invalid run page options", cause })
+          )
+        )
+        const filters = admitted.filters ?? {}
+        const tuple = JSON.stringify([
+          filters.status ?? null,
+          filters.flowName ?? null,
+          filters.parentRunId === undefined ? [] : [filters.parentRunId],
+          filters.lineageId ?? null,
+          filters.waitingReason === undefined ? [] : [filters.waitingReason],
+          filters.createdAfterMs ?? null,
+          filters.createdBeforeMs ?? null
+        ])
+        const cursor = admitted.cursor === undefined ?
+          null :
+          yield* Schema.decodeUnknownEffect(Cursor)(admitted.cursor).pipe(
+            Effect.mapError((cause) =>
+              new RunCatalogError({ code: "invalid_cursor", message: "malformed run cursor", cause })
+            )
+          )
+        if (cursor !== null && cursor.filters !== tuple) {
+          return yield* Effect.fail(
+            new RunCatalogError({ code: "invalid_cursor", message: "run cursor belongs to another query" })
+          )
+        }
+        const limit = admitted.limit ?? 100
+        return yield* Read.transaction(
+          sql,
+          Effect.gen(function*() {
+            const at = yield* Read.position(sql)
+            if (cursor !== null && cursor.source !== at.source) {
+              return yield* Effect.fail(
+                new RunCatalogError({ code: "source_changed", message: "run cursor belongs to another engine source" })
+              )
+            }
+            const predicates: Array<string> = []
+            const values: Array<string | number | null> = []
+            let mask = 0
+            const equalities = [
+              ["status", filters.status],
+              ["execution_flow", filters.flowName],
+              ["execution_parent_id", filters.parentRunId],
+              ["execution_lineage", filters.lineageId],
+              ["waiting_reason", filters.waitingReason]
+            ] as const
+            for (const [index, [column, value]] of equalities.entries()) {
+              if (value === undefined) continue
+              mask |= 1 << index
+              predicates.push(`r.${column} IS NOT DISTINCT FROM ?`)
+              values.push(value)
+            }
+            if (filters.createdAfterMs !== undefined) {
+              predicates.push("r.created_at_ms >= ?")
+              values.push(filters.createdAfterMs)
+            }
+            if (filters.createdBeforeMs !== undefined) {
+              predicates.push("r.created_at_ms <= ?")
+              values.push(filters.createdBeforeMs)
+            }
+            if (cursor !== null) {
+              predicates.push("(r.created_at_ms, r.run_id) > (?, ?)")
+              values.push(cursor.createdAtMs, cursor.runId)
+            }
+            values.push(limit + 1)
+            const rows = yield* Dialect.query(
+              sql,
+              `SELECT ${Read.selectColumns} FROM flows_runs r ${
+                Dialect.isPostgres(sql) ? "" : `INDEXED BY flows_runs_listing_${mask}`
+              }
+        LEFT JOIN flows_run_changes c ON c.run_id = r.run_id
+        ${
+                predicates.length === 0
+                  ? ""
+                  : `WHERE ${predicates.join(" AND ")}`
+              }
+        ORDER BY r.created_at_ms, r.run_id LIMIT ?`,
+              values
+            )
+            const decoded = yield* Effect.forEach(rows, (row) => Read.observed(row, at))
+            const runs = decoded.slice(0, limit)
+            const last = runs[runs.length - 1]
+            return {
+              ...at,
+              runs,
+              cursor: decoded.length > limit ?
+                JSON.stringify({
+                  version: 1,
+                  source: at.source,
+                  filters: tuple,
+                  createdAtMs: last!.createdAtMs,
+                  runId: last!.runId
+                }) :
+                null
+            }
+          })
+        )
+      }).pipe(Effect.catchCause((cause) => {
+        if (Cause.hasInterruptsOnly(cause)) return Effect.failCause(cause as Cause.Cause<never>)
+        const original = Cause.squash(cause)
+        return Effect.fail(
+          original instanceof RunCatalogError ? original : new RunCatalogError({
+            code: "list_failed",
+            message: "the engine run page could not be read",
+            cause: original
+          })
+        )
+      }))
+    return { listRuns }
+  })

package/src/internal/RunListingError.ts

@@ -0,0 +1,37 @@
+/**
+ * Shared catalog failure vocabulary.
+ *
+ * @since 1.0.0
+ */
+
+import * as Schema from "effect/Schema"
+
+/**
+ * Catalog failure codes.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export const RunCatalogErrorCode = Schema.Literals([
+  "invalid_options",
+  "list_failed",
+  "invalid_cursor",
+  "source_changed"
+])
+/**
+ * Catalog failure codes.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type RunCatalogErrorCode = typeof RunCatalogErrorCode.Type
+/**
+ * Catalog read failure retaining its original cause.
+ *
+ * @category errors
+ * @since 0.1.0
+ */
+export class RunCatalogError extends Schema.TaggedError<RunCatalogError>()(
+  "@smthrs/engine-store/RunCatalogError",
+  { code: RunCatalogErrorCode, message: Schema.String, cause: Schema.optional(Schema.Unknown) }
+) {}

package/src/internal/RuntimeGraph.ts

@@ -0,0 +1,253 @@
+/**
+ * Rebuildable scheduler indexes. Only verified, append-only plan nodes enter
+ * here; durable plans, observations and attempts remain authoritative.
+ *
+ * @since 1.0.0
+ */
+
+import type { Plan, Scheduling } from "@smthrs/plan"
+import type { Outcome } from "../PlanScheduler.ts"
+
+/** Mutable coordinator state; dispatch fibers own only attempt-local progress.
+ * @since 1.0.0
+ * @category models
+ */
+export interface NodeState {
+  status: "pending" | "running" | "settled"
+  outcome: Outcome
+  attempts: number
+  rebases: number
+  waited: number
+  dispatchKey: string
+}
+
+/** Counts explicit graph work, not host time or VM-internal allocations.
+ * `allocations` counts new containers and records, including frontier copies
+ * and candidate records. Collection backing storage and sort internals are
+ * not observable here. Admission policy work is owned by Scheduling.
+ * @since 1.0.0
+ * @category diagnostics
+ */
+export interface Work {
+  nodes: number
+  edges: number
+  allocations: number
+  membership: number
+}
+
+interface Vertex {
+  readonly node: Plan.PlanNode
+  readonly index: number
+  readonly state: NodeState
+  readonly dependencies: Set<Vertex>
+  readonly dependents: Set<Vertex>
+  unresolved: number
+  blocked: number
+  propagated: Outcome | undefined
+}
+
+const work = (): Work => ({ nodes: 0, edges: 0, allocations: 0, membership: 0 })
+const blocks = (outcome: Outcome): boolean => outcome !== "built" && outcome !== "clean"
+
+/** Run-local adjacency, readiness, membership and work counters.
+ *
+ * A provisional outcome is staged while reconciliation reads the journal.
+ * Only `reconcile` publishes it to dependents. A later verdict can change an
+ * outcome's blocking contribution, but never decrement indegree twice.
+ *
+ * @since 1.0.0
+ * @category models
+ */
+export interface RuntimeGraph {
+  readonly states: ReadonlyMap<string, NodeState>
+  readonly remaining: number
+  readonly cost: {
+    readonly update: Work
+    readonly settlement: Work
+    readonly frontier: Work
+    readonly admission: Work
+    readonly reachability: Work
+  }
+  /** Adds only the newly admitted suffix, synchronously after its durable commit. */
+  readonly append: (nodes: ReadonlyArray<Plan.PlanNode>) => void
+  readonly stage: (nodeId: string, outcome: Outcome) => void
+  readonly reconcile: () => void
+  /** Declaration-ordered snapshots preserve passive-settlement wave ordering. */
+  readonly blocked: () => ReadonlyArray<Plan.PlanNode>
+  readonly ready: () => ReadonlyArray<Plan.PlanNode>
+  readonly candidates: (ready: ReadonlyArray<Plan.PlanNode>) => ReadonlyArray<Scheduling.Candidate>
+  readonly start: (nodeId: string) => void
+  /** Discovered edges affect readiness only, never declared file versions. */
+  readonly reorder: (nodeId: string, owners: ReadonlyArray<string>) => void
+  /** Visits declared ancestors once per query, including a not-yet-admitted suffix node. */
+  readonly visitAncestors: (node: Plan.PlanNode, visit: (node: Plan.PlanNode) => void) => void
+}
+
+/** Builds one private graph per run/resume. Recovery stages durable settlements
+ * before the first admission; ordinary attempt replay still verifies content.
+ * @since 1.0.0
+ * @category constructors
+ */
+export const make = (nodes: ReadonlyArray<Plan.PlanNode>): RuntimeGraph => {
+  const vertices = new Map<string, Vertex>()
+  const states = new Map<string, NodeState>()
+  const ready = new Set<Vertex>()
+  const blocked = new Set<Vertex>()
+  const staged = new Set<Vertex>()
+  const cost = { update: work(), settlement: work(), frontier: work(), admission: work(), reachability: work() }
+  // The graph object, cost records, maps, sets and method closures have a
+  // constant per-run footprint. Count the explicit containers/records.
+  cost.update.allocations = 12
+  let remaining = 0
+
+  const refresh = (vertex: Vertex): void => {
+    ready.delete(vertex)
+    blocked.delete(vertex)
+    if (vertex.state.status === "pending" && vertex.unresolved === 0) {
+      if (vertex.blocked > 0) blocked.add(vertex)
+      else ready.add(vertex)
+    }
+  }
+
+  const edge = (target: Vertex, dependency: Vertex, count: Work): void => {
+    count.edges++
+    count.membership++
+    if (target.dependencies.has(dependency)) return
+    target.dependencies.add(dependency)
+    dependency.dependents.add(target)
+    if (dependency.propagated === undefined) target.unresolved++
+    else if (blocks(dependency.propagated)) target.blocked++
+  }
+
+  const append = (suffix: ReadonlyArray<Plan.PlanNode>): void => {
+    // Register the whole suffix first: declaration order need not be a
+    // topological order. Verified generations never rewrite old vertices.
+    for (const node of suffix) {
+      const state: NodeState = {
+        status: "pending",
+        outcome: "skipped",
+        attempts: 0,
+        rebases: 0,
+        waited: 0,
+        dispatchKey: ""
+      }
+      const vertex: Vertex = {
+        node,
+        index: vertices.size,
+        state,
+        dependencies: new Set(),
+        dependents: new Set(),
+        unresolved: 0,
+        blocked: 0,
+        propagated: undefined
+      }
+      vertices.set(node.id, vertex)
+      states.set(node.id, state)
+      remaining++
+      cost.update.nodes++
+      cost.update.allocations += 4
+    }
+    for (const node of suffix) {
+      const vertex = vertices.get(node.id)!
+      cost.update.nodes++
+      cost.update.membership++
+      for (const dependency of node.dependsOn) {
+        cost.update.membership++
+        edge(vertex, vertices.get(dependency)!, cost.update)
+      }
+      refresh(vertex)
+    }
+  }
+
+  const snapshot = (frontier: ReadonlySet<Vertex>): ReadonlyArray<Plan.PlanNode> => {
+    cost.frontier.nodes += frontier.size
+    cost.frontier.allocations += 2
+    return [...frontier].sort((left, right) => left.index - right.index).map((vertex) => vertex.node)
+  }
+
+  append(nodes)
+  return {
+    states,
+    get remaining() {
+      return remaining
+    },
+    cost,
+    append,
+    stage: (nodeId, outcome) => {
+      cost.settlement.nodes++
+      cost.settlement.membership++
+      const vertex = vertices.get(nodeId)!
+      vertex.state.status = "settled"
+      vertex.state.outcome = outcome
+      staged.add(vertex)
+    },
+    reconcile: () => {
+      for (const vertex of staged) {
+        cost.settlement.nodes++
+        const previous = vertex.propagated
+        const outcome = vertex.state.outcome
+        if (previous === outcome) continue
+        if (previous === undefined) remaining--
+        const delta = Number(blocks(outcome)) - Number(previous !== undefined && blocks(previous))
+        vertex.propagated = outcome
+        refresh(vertex)
+        for (const dependent of vertex.dependents) {
+          cost.settlement.edges++
+          cost.settlement.nodes++
+          if (previous === undefined) dependent.unresolved--
+          dependent.blocked += delta
+          refresh(dependent)
+        }
+      }
+      staged.clear()
+    },
+    blocked: () => snapshot(blocked),
+    ready: () => snapshot(ready),
+    candidates: (frontier) => {
+      cost.admission.allocations += 1 + frontier.length
+      return frontier.map((node) => {
+        cost.admission.nodes++
+        cost.admission.membership++
+        const vertex = vertices.get(node.id)!
+        return { node, order: vertex.index, waited: vertex.state.waited }
+      })
+    },
+    start: (nodeId) => {
+      cost.admission.nodes++
+      cost.admission.membership++
+      const vertex = vertices.get(nodeId)!
+      vertex.state.status = "running"
+      ready.delete(vertex)
+    },
+    reorder: (nodeId, owners) => {
+      const dependency = vertices.get(nodeId)!
+      cost.update.membership++
+      for (const owner of owners) {
+        cost.update.nodes++
+        cost.update.membership++
+        const target = vertices.get(owner)
+        if (target?.state.status !== "pending") continue
+        edge(target, dependency, cost.update)
+        refresh(target)
+      }
+    },
+    visitAncestors: (node, visit) => {
+      const seen = new Set<string>([node.id])
+      const stack = [...node.dependsOn]
+      cost.reachability.allocations += 2
+      cost.reachability.edges += stack.length
+      while (stack.length > 0) {
+        const id = stack.pop()!
+        cost.reachability.membership++
+        if (seen.has(id)) continue
+        seen.add(id)
+        const predecessor = vertices.get(id)!.node
+        cost.reachability.membership++
+        cost.reachability.nodes++
+        visit(predecessor)
+        cost.reachability.edges += predecessor.dependsOn.length
+        stack.push(...predecessor.dependsOn)
+      }
+    }
+  }
+}