@smthrs/engine-store 0.0.0-stage → 1.0.0-rc.3

package/src/internal/EngineStateSchema.ts

@@ -0,0 +1,92 @@
+/**
+ * The schema objects `DurableEngineState` creates outside the journal
+ * migration, and the inventory of what porting them to another
+ * dialect costs.
+ *
+ * Each durable package owns its migration ladder. Spawn edges and their GC
+ * trigger moved into engine-store migration 0006. The remaining
+ * statements are engine-store-owned storage created idempotently at
+ * construction instead (issues #40/#41/#79/#81). That is a deliberate lane
+ * boundary, not an oversight — but it left the statements invisible to the
+ * pg-porting plan in the database roadmap, which
+ * scoped only the journal migration's SQLite-specific DDL. Declaring them here
+ * makes the inventory machine-readable: a test diffs the database's schema
+ * objects across `make` against this list, so no engine-owned statement can
+ * be added without appearing in the porting inventory (issue #92).
+ *
+ * @since 0.1.0
+ */
+
+import { isReadOnly } from "@smthrs/database/Dialect"
+import type { Service as WriterService } from "@smthrs/database/DurableWriter"
+import * as Effect from "effect/Effect"
+import type * as SqlClient from "effect/unstable/sql/SqlClient"
+
+/**
+ * SQL dialects a statement is known to be accepted by, verbatim.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export type Dialect = "sqlite" | "postgres"
+
+/**
+ * One engine-store-owned schema object.
+ *
+ * @since 0.1.0
+ * @category models
+ */
+export interface Statement {
+  /** The schema object's name, as it appears in the catalog. */
+  readonly name: string
+  /** Why the object exists, by issue. */
+  readonly reason: string
+  /**
+   * Dialects that accept this statement unchanged. A statement listing only
+   * `"sqlite"` must be rewritten before a Postgres/PGlite `Database` layer
+   * can construct this service — `make` pipes every one of them through
+   * `Effect.orDie`, so an unported statement is a construction-time defect,
+   * not a recoverable error.
+   */
+  readonly dialects: ReadonlyArray<Dialect>
+  readonly ddl: string
+}
+
+/**
+ * The porting inventory: every engine-store-owned statement, in creation order.
+ *
+ * @since 0.1.0
+ * @category constants
+ */
+export const statements: ReadonlyArray<Statement> = [
+  {
+    name: "flows_runs_stale_running_idx",
+    reason:
+      "Serves the stale-running sweep's per-tick predicate (issue #79): `status = 'running' AND heartbeat_at_ms < cutoff`, ordered by heartbeat. A partial index keeps it tiny — only live running rows appear.",
+    // Partial indexes are valid Postgres since 9.5, so this one ports as-is.
+    dialects: ["sqlite", "postgres"],
+    ddl: `
+      CREATE INDEX IF NOT EXISTS flows_runs_stale_running_idx
+      ON flows_runs (heartbeat_at_ms)
+      WHERE status = 'running'
+    `
+  }
+]
+
+/**
+ * Creates the engine-store-owned objects, idempotently, in inventory order.
+ *
+ * `flows_runs` must already exist — every query in this service assumes a
+ * migrated database, so `make` composes over one by construction. A read-only
+ * client installs nothing.
+ *
+ * @since 0.1.0
+ * @category constructors
+ */
+export const apply = (sql: SqlClient.SqlClient, writer: WriterService): Effect.Effect<void> =>
+  // A read-only observer reads the schema it finds and installs nothing.
+  isReadOnly(sql) ? Effect.void : Effect.forEach(
+    statements,
+    (statement) => writer.write(sql.unsafe(statement.ddl)).pipe(Effect.orDie),
+    { discard: true }
+  )

package/src/internal/ExecutionSnapshotRead.ts

@@ -0,0 +1,298 @@
+/**
+ * Shared transactional execution decoding.
+ *
+ * @since 1.0.0
+ */
+
+import * as Sha256 from "@smthrs/crypto/Sha256"
+import { OwnerId } from "@smthrs/journal/OwnerId"
+import { RunStatus, RunStoreError } from "@smthrs/run-store/RunStore"
+import { Cause, Effect, Exit, Option, Schema } from "effect"
+import type * as SqlClient from "effect/unstable/sql/SqlClient"
+import type { Batch } from "../ExecutionSnapshot.ts"
+import { RunState } from "../RunState.ts"
+
+/**
+ * Non-negative safe integer for timestamps and revisions.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export const Natural = Schema.Int.check(Schema.isBetween({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }))
+/**
+ * Bounded identifier with a durable text encoding.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export const Identifier = Schema.NonEmptyString.check(
+  Schema.isMaxLength(1024),
+  Schema.makeFilter((value) =>
+    !value.includes("\0") && !/[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/.test(value)
+  )
+)
+/**
+ * Engine database source identity.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export const Source = Schema.String.check(Schema.isPattern(/^[0-9a-f]{32}$/))
+/**
+ * Source identity and observed high-water mark.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export const Position = Schema.Struct({ source: Source, revision: Natural })
+/**
+ * Source identity and observed high-water mark.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export type Position = typeof Position.Type
+
+/**
+ * A deferred read transaction on the reserved engine connection. The normal
+ * writable client's transaction starts IMMEDIATE; a read needs only SQLite's
+ * snapshot and must allow a WAL writer to commit while it reads. PostgreSQL
+ * reads at REPEATABLE READ so every statement shares the snapshot the position
+ * was read from; READ COMMITTED would let a concurrent commit make a row look
+ * newer than its position. Reuse the SQL
+ * client's transaction service to join an existing transaction. No read body
+ * exposes a mutation callback. Connection reservation and interruption release
+ * remain scoped, including when BEGIN or the read fails.
+ * @since 1.0.0
+ * @private
+ */
+export const transaction = <A, E, R>(sql: SqlClient.SqlClient, effect: Effect.Effect<A, E, R>) =>
+  Effect.flatMap(
+    Effect.serviceOption(sql.transactionService),
+    (existing) =>
+      Option.isSome(existing) ? effect : Effect.scoped(Effect.gen(function*() {
+        const connection = yield* sql.reserve
+        return yield* Effect.acquireUseRelease(
+          connection.executeUnprepared(
+            sql.onDialectOrElse({ pg: () => "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY", orElse: () => "BEGIN" }),
+            [],
+            undefined
+          ),
+          () => effect.pipe(Effect.provideService(sql.transactionService, [connection, 0])),
+          (_, exit) =>
+            connection.executeUnprepared(Exit.isSuccess(exit) ? "COMMIT" : "ROLLBACK", [], undefined).pipe(Effect.orDie)
+        )
+      }))
+  )
+
+/**
+ * Preserves failure causes and interruption at the read boundary.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export const boundary = <A, E, R>(method: string, effect: Effect.Effect<A, E, R>): Effect.Effect<A, RunStoreError, R> =>
+  effect.pipe(Effect.catchCause((cause) => {
+    if (Cause.hasInterruptsOnly(cause)) return Effect.failCause(cause as Cause.Cause<never>)
+    const original = Cause.squash(cause)
+    return Effect.fail(
+      original instanceof RunStoreError ? original : new RunStoreError({
+        method,
+        code: "persistence_failed",
+        message: "engine execution observation failed",
+        cause: original
+      })
+    )
+  }))
+
+/**
+ * Decodes stored SQL values with the original schema error.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export const decode = <S extends Schema.Top>(schema: S, input: unknown) =>
+  Schema.decodeUnknownEffect(schema)(input).pipe(Effect.mapError((cause) =>
+    new RunStoreError({
+      method: "ExecutionSnapshot.decode",
+      code: "decode_failed",
+      message: "stored execution observation is malformed",
+      cause
+    })
+  ))
+
+/**
+ * Reads the engine database identity and revision.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export const position = (sql: SqlClient.SqlClient) =>
+  sql`SELECT source, revision FROM flows_run_source WHERE singleton = 1`.pipe(
+    Effect.flatMap((rows) => decode(Position, rows[0]))
+  )
+
+const Row = Schema.Struct({
+  run_id: Identifier,
+  status: RunStatus,
+  created_at_ms: Natural,
+  started_at_ms: Schema.NullOr(Natural),
+  finished_at_ms: Schema.NullOr(Natural),
+  owner_host_id: Schema.NullOr(Identifier),
+  owner_pid: Schema.NullOr(Natural),
+  owner_nonce: Schema.NullOr(Identifier),
+  heartbeat_at_ms: Schema.NullOr(Natural),
+  state_json: Schema.fromJsonString(RunState),
+  parent_run_id: Schema.NullOr(Identifier),
+  spawn_parent_id: Schema.NullOr(Identifier),
+  spawn_seq: Schema.NullOr(Natural),
+  execution_parent_id: Schema.NullOr(Identifier),
+  lineage_id: Schema.NullOr(Identifier),
+  round_ordinal: Schema.NullOr(Natural),
+  cancel_requested_at_ms: Schema.NullOr(Natural),
+  cancel_acknowledgement_json: Schema.NullOr(Schema.fromJsonString(Schema.Struct({
+    observedAtMs: Natural,
+    owner: OwnerId
+  }))),
+  // These opaque values use DurableEngineState's existing non-empty-string
+  // contract. They are not run IDs and must not acquire an identifier ceiling.
+  waiting_reason: Schema.NullOr(Schema.NonEmptyString),
+  waiting_wake_at_ms: Schema.NullOr(Natural),
+  waiting_token: Schema.NullOr(Schema.NonEmptyString),
+  waiting_request: Schema.optional(Schema.NullOr(Schema.fromJsonString(Schema.Json))),
+  revision: Natural,
+  deleted: Schema.Literal(0)
+})
+
+/**
+ * Structured waiting condition; nullable fields preserve older unknown values.
+ * The opaque wake token leaves the engine only as its SHA-256 `tokenDigest`,
+ * the same digest `ExecutionFacts` records: a reader may correlate a wait but
+ * never holds the address that resolves it.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export interface Waiting {
+  readonly kind: "timer" | "signal" | "approval" | "quota" | "human" | "other"
+  readonly reason: string
+  readonly wakeAtMs: number | null
+  readonly tokenDigest: Sha256.Digest | null
+  readonly request?: typeof Schema.Json.Type | undefined
+}
+
+const waitingKind = (reason: string): Waiting["kind"] => {
+  switch (reason) {
+    case "timer":
+    case "approval":
+    case "quota":
+    case "human":
+      return reason
+    case "event":
+    case "signal":
+      return "signal"
+    default:
+      return "other"
+  }
+}
+
+/**
+ * Page-scoped lifecycle fields and indexed earliest spawn edge.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export const selectColumns = `r.*, c.revision, c.deleted,
+  (SELECT parent_id FROM flows_run_parents WHERE child_id = r.run_id ORDER BY seq, parent_id LIMIT 1) AS spawn_parent_id,
+  (SELECT seq FROM flows_run_parents WHERE child_id = r.run_id ORDER BY seq, parent_id LIMIT 1) AS spawn_seq`
+
+/**
+ * Decodes a run row and validates observation invariants.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export const observed = (input: unknown, at: Position) =>
+  Effect.gen(function*() {
+    const row = yield* decode(Row, input)
+    const parentRunId = row.parent_run_id ?? row.spawn_parent_id
+    const ownerFields = [row.owner_host_id, row.owner_pid, row.owner_nonce, row.heartbeat_at_ms]
+    const validOwner = row.status === "running"
+      ? ownerFields.every((value) => value !== null)
+      : ownerFields.every((value) => value === null)
+    if (
+      !validOwner || row.revision === 0 || row.revision > at.revision ||
+      parentRunId !== row.execution_parent_id || parentRunId === row.run_id ||
+      (row.lineage_id === null) !== (row.round_ordinal === null) ||
+      (row.waiting_reason === null && (row.waiting_wake_at_ms !== null || row.waiting_token !== null ||
+        row.waiting_request !== undefined && row.waiting_request !== null)) ||
+      (row.cancel_acknowledgement_json !== null && row.cancel_requested_at_ms === null)
+    ) {
+      return yield* Effect.fail(
+        new RunStoreError({
+          method: "ExecutionSnapshot.decode",
+          code: "decode_failed",
+          message: "stored execution observation violates its invariants",
+          cause: new Error("inconsistent execution identity, wait, cancellation or revision")
+        })
+      )
+    }
+    return {
+      _tag: "Observed" as const,
+      runId: row.run_id,
+      source: at.source,
+      revision: row.revision,
+      status: row.status,
+      flowName: row.state_json.flowName,
+      createdAtMs: row.created_at_ms,
+      startedAtMs: row.started_at_ms,
+      finishedAtMs: row.finished_at_ms,
+      parentRunId,
+      lineageId: row.lineage_id ?? row.run_id,
+      roundOrdinal: row.round_ordinal ?? 0,
+      cancellation: {
+        requestedAtMs: row.cancel_requested_at_ms,
+        acknowledgement: row.cancel_acknowledgement_json
+      },
+      waiting: row.waiting_reason === null ? null : {
+        kind: waitingKind(row.waiting_reason),
+        reason: row.waiting_reason,
+        wakeAtMs: row.waiting_wake_at_ms,
+        tokenDigest: row.waiting_token === null ? null : Sha256.digestSync(row.waiting_token),
+        ...(row.waiting_request === undefined || row.waiting_request === null ? {} : { request: row.waiting_request })
+      }
+    }
+  })
+
+/**
+ * The result of a successful execution observation.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export type Observed = Effect.Success<ReturnType<typeof observed>>
+
+/**
+ * Validates source, identity, ordering and revision coherence at the batch boundary.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export const coherentBatch = (runIds: ReadonlyArray<string>, batch: Batch): Effect.Effect<Batch, RunStoreError> => {
+  const validPosition = Schema.is(Position)
+  if (
+    !validPosition(batch) || batch.snapshots.length !== runIds.length ||
+    batch.snapshots.some((row, index) =>
+      !validPosition(row) || row.runId !== runIds[index] || row.source !== batch.source ||
+      row.revision > batch.revision
+    )
+  ) {
+    return Effect.fail(new RunStoreError({
+      method: "ExecutionSnapshot.read",
+      code: "persistence_failed",
+      message: "stored execution batch is incoherent: source, identity, order or revision mismatch",
+      cause: undefined
+    }))
+  }
+  return Effect.succeed(batch)
+}

package/src/internal/ExecutionSnapshotRelations.ts

@@ -0,0 +1,121 @@
+/**
+ * Page-scoped durable child and round enumeration.
+ *
+ * @since 1.0.0
+ */
+
+import * as Dialect from "@smthrs/database/Dialect"
+import { RunStoreError } from "@smthrs/run-store/RunStore"
+import { Effect, Schema } from "effect"
+import type * as SqlClient from "effect/unstable/sql/SqlClient"
+import type { Batch, Snapshot } from "../ExecutionSnapshot.ts"
+import * as Read from "./ExecutionSnapshotRead.ts"
+
+/**
+ * Child edges and trampoline rounds are different relations.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export interface RelatedOptions {
+  readonly runId: string
+  readonly kind: "children" | "rounds"
+  readonly cursor?: string
+  readonly limit?: number
+}
+/**
+ * A requested anchor and its page, observed in one transaction.
+ *
+ * @category models
+ * @since 1.0.0
+ */
+export interface RelatedPage extends Batch {
+  readonly anchor: Snapshot
+  readonly cursor: string | null
+}
+const Kind = Schema.Literals(["children", "rounds"])
+const Options = Schema.Struct({
+  runId: Read.Identifier,
+  kind: Kind,
+  cursor: Schema.optionalKey(Schema.String.check(Schema.isMaxLength(32768))),
+  limit: Schema.optionalKey(Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 200 })))
+})
+const Cursor = Schema.fromJsonString(Schema.Struct({
+  version: Schema.Literal(1),
+  source: Read.Source,
+  runId: Read.Identifier,
+  kind: Kind,
+  ordinal: Read.Natural,
+  lastRunId: Read.Identifier
+}))
+const Relation = Schema.Struct({ run_id: Read.Identifier, ordinal: Read.Natural })
+
+/**
+ * Builds a bounded relation reader over the shared snapshot port.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export const make = (
+  sql: SqlClient.SqlClient,
+  read: (ids: ReadonlyArray<string>) => Effect.Effect<Batch, RunStoreError>
+) =>
+(options: RelatedOptions): Effect.Effect<RelatedPage, RunStoreError> =>
+  Read.boundary(
+    "ExecutionSnapshot.related",
+    Effect.gen(function*() {
+      const input = yield* Read.decode(Options, options)
+      const cursor = input.cursor === undefined ? null : yield* Read.decode(Cursor, input.cursor)
+      return yield* Read.transaction(
+        sql,
+        Effect.gen(function*() {
+          const at = yield* Read.position(sql)
+          if (
+            cursor !== null &&
+            (cursor.source !== at.source || cursor.runId !== input.runId || cursor.kind !== input.kind)
+          ) {
+            return yield* Effect.fail(
+              new RunStoreError({
+                method: "ExecutionSnapshot.related",
+                code: "invalid_run",
+                message: "relation cursor belongs to another query or source",
+                cause: cursor
+              })
+            )
+          }
+          const anchor = (yield* read([input.runId])).snapshots[0]!
+          const limit = input.limit ?? 100
+          const values: Array<string | number | null> = [input.runId]
+          const columns = input.kind === "children" ? ["seq", "child_id"] : ["COALESCE(round_ordinal, 0)", "run_id"]
+          const after = cursor === null ? "" : `AND (${columns.join(", ")}) > (?, ?)`
+          if (cursor !== null) values.push(cursor.ordinal, cursor.lastRunId)
+          values.push(limit + 1)
+          const query = input.kind === "children"
+            ? `SELECT child_id AS run_id, seq AS ordinal FROM flows_run_parents
+         WHERE parent_id = ? ${after} ORDER BY seq, child_id LIMIT ?`
+            : `SELECT run_id, COALESCE(round_ordinal, 0) AS ordinal FROM flows_runs
+         WHERE execution_lineage = (SELECT execution_lineage FROM flows_runs WHERE run_id = ?)
+         ${after} ORDER BY COALESCE(round_ordinal, 0), run_id LIMIT ?`
+          const rows = yield* Dialect.query(sql, query, values)
+          const decoded = yield* Effect.forEach(rows, (row) => Read.decode(Relation, row))
+          const selected = decoded.slice(0, limit)
+          const batch = yield* read(selected.map((row) => row.run_id))
+          const last = selected[selected.length - 1]
+          return {
+            ...batch,
+            anchor,
+            cursor: decoded.length > limit ?
+              JSON.stringify({
+                version: 1,
+                source: at.source,
+                runId: input.runId,
+                kind: input.kind,
+                ordinal: last!.ordinal,
+                lastRunId: last!.run_id
+              }) :
+              null
+          }
+        })
+      )
+    })
+  )