@smithers-orchestrator/db 0.16.0

package/src/storage/StorageServiceTypes.ts

@@ -0,0 +1,150 @@
+export type JsonRecord = Record<string, unknown>;
+export type OutputKey = Record<string, string | number | boolean | null>;
+
+export type Run = {
+  readonly runId: string;
+  readonly parentRunId?: string | null;
+  readonly workflowName?: string | null;
+  readonly workflowPath?: string | null;
+  readonly workflowHash?: string | null;
+  readonly status: string;
+  readonly createdAtMs?: number;
+  readonly startedAtMs?: number | null;
+  readonly finishedAtMs?: number | null;
+  readonly heartbeatAtMs?: number | null;
+  readonly runtimeOwnerId?: string | null;
+  readonly cancelRequestedAtMs?: number | null;
+  readonly hijackRequestedAtMs?: number | null;
+  readonly hijackTarget?: string | null;
+  readonly vcsType?: string | null;
+  readonly vcsRoot?: string | null;
+  readonly vcsRevision?: string | null;
+  readonly errorJson?: string | null;
+  readonly configJson?: string | null;
+};
+
+export type RunPatch = Partial<Omit<Run, "runId">>;
+
+export type RunAncestryRow = {
+  readonly runId: string;
+  readonly parentRunId: string | null;
+  readonly depth: number;
+};
+
+export type Attempt = {
+  readonly runId: string;
+  readonly nodeId: string;
+  readonly iteration: number;
+  readonly attempt: number;
+  readonly state: string;
+  readonly startedAtMs: number;
+  readonly finishedAtMs?: number | null;
+  readonly heartbeatAtMs?: number | null;
+  readonly heartbeatDataJson?: string | null;
+  readonly errorJson?: string | null;
+  readonly jjPointer?: string | null;
+  readonly responseText?: string | null;
+  readonly jjCwd?: string | null;
+  readonly cached?: boolean;
+  readonly metaJson?: string | null;
+};
+
+export type AttemptPatch = Partial<
+  Omit<Attempt, "runId" | "nodeId" | "iteration" | "attempt">
+>;
+
+export type FrameRow = {
+  readonly runId: string;
+  readonly frameNo: number;
+  readonly xmlJson?: string | null;
+  readonly graphJson?: string | null;
+  readonly xmlHash?: string | null;
+  readonly xmlEncoding?: string | null;
+  readonly createdAtMs?: number;
+  readonly [key: string]: unknown;
+};
+
+export type SignalInsertRow = {
+  readonly runId: string;
+  readonly signalName: string;
+  readonly correlationId: string | null;
+  readonly payloadJson: string;
+  readonly receivedAtMs: number;
+  readonly receivedBy?: string | null;
+};
+
+export type EventRow = {
+  readonly runId: string;
+  readonly seq: number;
+  readonly timestampMs: number;
+  readonly type: string;
+  readonly payloadJson: string;
+};
+
+export type EventInsertRow = Omit<EventRow, "seq">;
+
+export type RalphRow = {
+  readonly runId: string;
+  readonly ralphId: string;
+  readonly iteration: number;
+  readonly done?: boolean;
+  readonly stateJson?: string | null;
+  readonly updatedAtMs?: number;
+};
+
+export type SandboxRow = {
+  readonly runId: string;
+  readonly sandboxId: string;
+  readonly [key: string]: unknown;
+};
+
+export type ToolCallRow = {
+  readonly runId: string;
+  readonly nodeId: string;
+  readonly iteration: number;
+  readonly [key: string]: unknown;
+};
+
+export type CronRow = {
+  readonly cronId: string;
+  readonly pattern?: string;
+  readonly workflowPath?: string;
+  readonly enabled?: boolean;
+  readonly lastRunAtMs?: number | null;
+  readonly nextRunAtMs?: number | null;
+  readonly errorJson?: string | null;
+  readonly [key: string]: unknown;
+};
+
+export type ScorerResultRow = {
+  readonly runId: string;
+  readonly nodeId?: string;
+  readonly scorerId?: string;
+  readonly scoredAtMs?: number;
+  readonly [key: string]: unknown;
+};
+
+export type ClaimRunForResumeParams = {
+  readonly runId: string;
+  readonly expectedStatus?: string;
+  readonly expectedRuntimeOwnerId: string | null;
+  readonly expectedHeartbeatAtMs: number | null;
+  readonly staleBeforeMs: number;
+  readonly claimOwnerId: string;
+  readonly claimHeartbeatAtMs: number;
+  readonly requireStale?: boolean;
+};
+
+export type ReleaseRunResumeClaimParams = {
+  readonly runId: string;
+  readonly claimOwnerId: string;
+  readonly restoreRuntimeOwnerId: string | null;
+  readonly restoreHeartbeatAtMs: number | null;
+};
+
+export type UpdateClaimedRunParams = {
+  readonly runId: string;
+  readonly expectedRuntimeOwnerId: string;
+  readonly expectedHeartbeatAtMs: number | null;
+  readonly patch: RunPatch;
+};

package/src/unwrapZodType.js

@@ -0,0 +1,17 @@
+/**
+ * Unwraps Zod wrapper types (nullable, optional, default) to get the base type.
+ */
+export function unwrapZodType(t) {
+    if (!t)
+        return t;
+    if (t._zod?.def) {
+        const typeName = t._zod.def.type;
+        if (typeName === "nullable" ||
+            typeName === "optional" ||
+            typeName === "default") {
+            const inner = t._zod.def.innerType;
+            return inner ? unwrapZodType(inner) : t;
+        }
+    }
+    return t;
+}

package/src/utils/camelToSnake.js

@@ -0,0 +1,6 @@
+/**
+ * Converts a camelCase string to snake_case.
+ */
+export function camelToSnake(str) {
+    return str.replace(/([A-Z])/g, "_$1").toLowerCase();
+}

package/src/withSqliteWriteRetryEffect.js

@@ -0,0 +1,110 @@
+import { Duration, Effect, Metric, Schedule, ScheduleDecision, ScheduleIntervals, } from "effect";
+import { dbRetries } from "@smithers-orchestrator/observability/metrics";
+import { retryPolicyToSchedule } from "@smithers-orchestrator/scheduler/retryPolicyToSchedule";
+import { isRetryableSqliteWriteError } from "./isRetryableSqliteWriteError.js";
+/** @typedef {import("@smithers-orchestrator/errors/SmithersError").SmithersError} SmithersError */
+/** @typedef {import("./SqliteWriteRetryOptions.ts").SqliteWriteRetryOptions} SqliteWriteRetryOptions */
+
+// Raised from 6→10 and 2000→10000: concurrent Worktree tasks can produce
+// short bursts of SQLITE_IOERR_VNODE on macOS; more retries with a wider
+// window gives busy_timeout time to clear the VFS lock.
+const DEFAULT_MAX_ATTEMPTS = 10;
+const DEFAULT_BASE_DELAY_MS = 50;
+const DEFAULT_MAX_DELAY_MS = 10_000;
+/**
+ * @param {unknown} error
+ * @returns {string}
+ */
+function describeSqliteWriteError(error) {
+    const metadata = findSqliteErrorMetadata(error);
+    const code = metadata?.code ?? "";
+    const message = metadata?.message || String(error?.message ?? error ?? "unknown error");
+    return code ? `${code}: ${message}` : message;
+}
+/**
+ * @param {unknown} error
+ * @returns {SqliteErrorMetadata | null}
+ */
+function readSqliteErrorMetadata(error) {
+    if (!error || (typeof error !== "object" && !(error instanceof Error))) {
+        return null;
+    }
+    const code = typeof error?.code === "string" ? error.code : "";
+    const message = String(error?.message ?? "");
+    return { code, message };
+}
+/**
+ * @param {unknown} error
+ * @returns {SqliteErrorMetadata | null}
+ */
+function findSqliteErrorMetadata(error) {
+    const seen = new Set();
+    let current = error;
+    while (current && !seen.has(current)) {
+        seen.add(current);
+        const metadata = readSqliteErrorMetadata(current);
+        if (metadata) {
+            const message = metadata.message.toLowerCase();
+            if (metadata.code.startsWith("SQLITE_BUSY") ||
+                metadata.code.startsWith("SQLITE_IOERR") ||
+                message.includes("database is locked") ||
+                message.includes("database is busy") ||
+                message.includes("disk i/o error")) {
+                return metadata;
+            }
+        }
+        current = current?.cause;
+    }
+    return readSqliteErrorMetadata(error);
+}
+/**
+ * @param {number} maxAttempts
+ * @param {number} baseDelayMs
+ * @param {number} maxDelayMs
+ */
+function makeSqliteRetrySchedule(maxAttempts, baseDelayMs, maxDelayMs) {
+    const boundedBaseDelayMs = Math.max(1, Math.floor(baseDelayMs));
+    const boundedMaxDelayMs = Math.max(1, Math.floor(maxDelayMs));
+    return retryPolicyToSchedule({
+        backoff: "exponential",
+        initialDelayMs: boundedBaseDelayMs,
+    }).pipe(Schedule.modifyDelay((_, delay) => Duration.millis(Math.min(boundedMaxDelayMs, Duration.toMillis(delay)))), Schedule.jitteredWith({ min: 0.75, max: 1.25 }), Schedule.whileInput(isRetryableSqliteWriteError), Schedule.intersect(Schedule.recurs(Math.max(0, maxAttempts - 1))));
+}
+/**
+ * @template A
+ * @param {() => Effect.Effect<A, SmithersError>} operation
+ * @param {SqliteWriteRetryOptions} [opts]
+ * @returns {Effect.Effect<A, SmithersError>}
+ */
+export function withSqliteWriteRetryEffect(operation, opts = {}) {
+    const { label = "sqlite write", maxAttempts = DEFAULT_MAX_ATTEMPTS, baseDelayMs = DEFAULT_BASE_DELAY_MS, maxDelayMs = DEFAULT_MAX_DELAY_MS, sleep, } = opts;
+    const boundedMaxAttempts = Math.max(1, Math.floor(maxAttempts));
+    let retryAttempt = 0;
+    let lastRetryError;
+    const retrySchedule = Schedule.mapInput(makeSqliteRetrySchedule(boundedMaxAttempts, baseDelayMs, maxDelayMs), (error) => {
+        lastRetryError = error;
+        return error;
+    }).pipe(Schedule.onDecision((_, decision) => {
+        if (ScheduleDecision.isDone(decision) || !lastRetryError) {
+            return Effect.void;
+        }
+        retryAttempt += 1;
+        const delayMs = Math.max(1, Math.round(ScheduleIntervals.start(decision.intervals) - Date.now()));
+        return Effect.gen(function* () {
+            yield* Metric.increment(dbRetries);
+            yield* Effect.logWarning(`${label} failed with ${describeSqliteWriteError(lastRetryError)}; retrying in ${delayMs}ms (${retryAttempt}/${boundedMaxAttempts})`);
+            if (sleep) {
+                yield* Effect.promise(() => sleep(delayMs));
+            }
+        }).pipe(Effect.annotateLogs({
+            retryable: true,
+            retryAttempt,
+            retryMaxAttempts: boundedMaxAttempts,
+            retryDelayMs: delayMs,
+            retryLabel: label,
+        }));
+    }));
+    return Effect.suspend(operation).pipe(Effect.retry(sleep
+        ? Schedule.modifyDelay(retrySchedule, () => Duration.zero)
+        : retrySchedule), Effect.withLogSpan("sqlite-write-retry"));
+}

package/src/write-retry.js

@@ -0,0 +1,49 @@
+import { isRetryableSqliteWriteError } from "./isRetryableSqliteWriteError.js";
+/** @typedef {import("./SqliteWriteRetryOptions.ts").SqliteWriteRetryOptions} SqliteWriteRetryOptions */
+
+const DEFAULT_MAX_ATTEMPTS = 6;
+const DEFAULT_BASE_DELAY_MS = 50;
+const DEFAULT_MAX_DELAY_MS = 2_000;
+export { isRetryableSqliteWriteError } from "./isRetryableSqliteWriteError.js";
+export { withSqliteWriteRetryEffect } from "./withSqliteWriteRetryEffect.js";
+/**
+ * @param {number} ms
+ * @returns {Promise<void>}
+ */
+function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/**
+ * @param {number} retryAttempt
+ * @param {number} baseDelayMs
+ * @param {number} maxDelayMs
+ * @returns {number}
+ */
+function computeDelayMs(retryAttempt, baseDelayMs, maxDelayMs) {
+    const boundedBaseDelayMs = Math.max(1, Math.floor(baseDelayMs));
+    const boundedMaxDelayMs = Math.max(1, Math.floor(maxDelayMs));
+    const exponentialDelayMs = boundedBaseDelayMs * 2 ** Math.max(0, retryAttempt - 1);
+    return Math.min(boundedMaxDelayMs, exponentialDelayMs);
+}
+/**
+ * @template A
+ * @param {() => A | PromiseLike<A>} operation
+ * @param {SqliteWriteRetryOptions} [opts]
+ * @returns {Promise<A>}
+ */
+export async function withSqliteWriteRetry(operation, opts = {}) {
+    const { maxAttempts = DEFAULT_MAX_ATTEMPTS, baseDelayMs = DEFAULT_BASE_DELAY_MS, maxDelayMs = DEFAULT_MAX_DELAY_MS, sleep = delay, } = opts;
+    const boundedMaxAttempts = Math.max(1, Math.floor(maxAttempts));
+    for (let attempt = 1;; attempt += 1) {
+        try {
+            return await operation();
+        }
+        catch (error) {
+            if (attempt >= boundedMaxAttempts ||
+                !isRetryableSqliteWriteError(error)) {
+                throw error;
+            }
+            await sleep(computeDelayMs(attempt, baseDelayMs, maxDelayMs));
+        }
+    }
+}

package/src/zodToCreateTableSQL.js

@@ -0,0 +1,41 @@
+import { z } from "zod";
+import { unwrapZodType } from "./unwrapZodType.js";
+import { camelToSnake } from "./utils/camelToSnake.js";
+/**
+ * Determines the Zod base type name from a (possibly unwrapped) Zod type.
+ */
+function getZodBaseTypeName(zodType) {
+    return zodType._zod?.def?.type ?? "unknown";
+}
+/**
+ * Generates a CREATE TABLE IF NOT EXISTS SQL statement from a Zod schema.
+ * Used for runtime table creation without Drizzle migrations.
+ */
+export function zodToCreateTableSQL(tableName, schema, opts) {
+    const colDefs = opts?.isInput
+        ? [`run_id TEXT NOT NULL PRIMARY KEY`]
+        : [
+            `run_id TEXT NOT NULL`,
+            `node_id TEXT NOT NULL`,
+            `iteration INTEGER NOT NULL DEFAULT 0`,
+        ];
+    const shape = schema.shape;
+    for (const [key] of Object.entries(shape)) {
+        const colName = camelToSnake(key);
+        const baseType = unwrapZodType(shape[key]);
+        const baseTypeName = getZodBaseTypeName(baseType);
+        if (baseTypeName === "number" ||
+            baseTypeName === "int" ||
+            baseTypeName === "float" ||
+            baseTypeName === "boolean") {
+            colDefs.push(`"${colName}" INTEGER`);
+        }
+        else {
+            colDefs.push(`"${colName}" TEXT`);
+        }
+    }
+    if (!opts?.isInput) {
+        colDefs.push(`PRIMARY KEY (run_id, node_id, iteration)`);
+    }
+    return `CREATE TABLE IF NOT EXISTS "${tableName}" (${colDefs.join(", ")})`;
+}

package/src/zodToTable.js

@@ -0,0 +1,60 @@
+import { sqliteTable, text, integer, primaryKey, } from "drizzle-orm/sqlite-core";
+import { z } from "zod";
+import { unwrapZodType } from "./unwrapZodType.js";
+import { camelToSnake } from "./utils/camelToSnake.js";
+/**
+ * Determines the Zod base type name from a (possibly unwrapped) Zod type.
+ */
+function getZodBaseTypeName(zodType) {
+    return zodType._zod?.def?.type ?? "unknown";
+}
+/**
+ * Generates a Drizzle sqliteTable from a Zod object schema.
+ *
+ * Each Zod field is mapped to a SQLite column:
+ * - z.string() / z.enum() -> text column
+ * - z.number() -> integer column
+ * - z.boolean() -> integer column with boolean mode
+ * - z.array() / z.object() / complex -> text column with json mode
+ *
+ * All tables include standard smithers key columns:
+ * runId, nodeId, iteration with a composite primary key.
+ */
+export function zodToTable(tableName, schema, opts) {
+    const columns = opts?.isInput
+        ? { runId: text("run_id").primaryKey() }
+        : {
+            runId: text("run_id").notNull(),
+            nodeId: text("node_id").notNull(),
+            iteration: integer("iteration").notNull().default(0),
+        };
+    const shape = schema.shape;
+    for (const [key, zodType] of Object.entries(shape)) {
+        const colName = camelToSnake(key);
+        const baseType = unwrapZodType(zodType);
+        const baseTypeName = getZodBaseTypeName(baseType);
+        if (baseTypeName === "number" ||
+            baseTypeName === "int" ||
+            baseTypeName === "float") {
+            columns[key] = integer(colName);
+        }
+        else if (baseTypeName === "boolean") {
+            columns[key] = integer(colName, { mode: "boolean" });
+        }
+        else if (baseTypeName === "string" ||
+            baseTypeName === "enum" ||
+            baseTypeName === "literal") {
+            columns[key] = text(colName);
+        }
+        else {
+            // arrays, objects, unions, and anything complex -> JSON text
+            columns[key] = text(colName, { mode: "json" });
+        }
+    }
+    if (opts?.isInput) {
+        return sqliteTable(tableName, columns);
+    }
+    return sqliteTable(tableName, columns, (t) => [
+        primaryKey({ columns: [t.runId, t.nodeId, t.iteration] }),
+    ]);
+}