@smithers-orchestrator/db 0.21.0 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@smithers-orchestrator/db",
3
- "version": "0.21.0",
3
+ "version": "0.23.0",
4
4
  "description": "SQLite and Drizzle persistence adapter for Smithers workflows",
5
5
  "type": "module",
6
6
  "sideEffects": false,
@@ -26,13 +26,16 @@
26
26
  "drizzle-zod": "^0.8.3",
27
27
  "effect": "^3.21.1",
28
28
  "zod": "^4.3.6",
29
- "@smithers-orchestrator/errors": "0.21.0",
30
- "@smithers-orchestrator/graph": "0.21.0",
31
- "@smithers-orchestrator/scheduler": "0.21.0",
32
- "@smithers-orchestrator/observability": "0.21.0"
29
+ "@smithers-orchestrator/errors": "0.23.0",
30
+ "@smithers-orchestrator/observability": "0.23.0",
31
+ "@smithers-orchestrator/graph": "0.23.0",
32
+ "@smithers-orchestrator/scheduler": "0.23.0"
33
33
  },
34
34
  "devDependencies": {
35
+ "@electric-sql/pglite": "0.5.1",
36
+ "@electric-sql/pglite-socket": "0.2.1",
35
37
  "@types/bun": "latest",
38
+ "pg": "^8.13.1",
36
39
  "typescript": "~5.9.3"
37
40
  },
38
41
  "scripts": {
package/src/adapter.js CHANGED
@@ -10,10 +10,11 @@
10
10
  /** @typedef {import("./adapter/StaleRunRecord.ts").StaleRunRecord} StaleRunRecord */
11
11
  // @smithers-type-exports-end
12
12
 
13
- import { getTableName, sql } from "drizzle-orm";
13
+ import { getTableName } from "drizzle-orm";
14
14
  import { Effect, Exit, FiberId, Metric } from "effect";
15
15
  import { toSmithersError } from "@smithers-orchestrator/errors/toSmithersError";
16
16
  import { getSqlMessageStorage } from "./sql-message-storage.js";
17
+ import { POSTGRES, beginTransactionSql } from "./dialect.js";
17
18
  import { alertsAcknowledgedTotal, alertsActive, alertsFiredTotal, dbQueryDuration, dbTransactionDuration, dbTransactionRollbacks, } from "@smithers-orchestrator/observability/metrics";
18
19
  import { assertOptionalStringMaxLength, assertPositiveFiniteNumber, } from "./input-bounds.js";
19
20
  import { FRAME_KEYFRAME_INTERVAL, applyFrameDeltaJson, encodeFrameDelta, normalizeFrameEncoding, serializeFrameDelta, } from "./frame-codec.js";
@@ -409,6 +410,23 @@ function getSqliteTransactionState(client) {
409
410
  }
410
411
  return state;
411
412
  }
413
+ /**
414
+ * Resolve a Drizzle output table's on-disk name. The name lives on the
415
+ * `Symbol(drizzle:Name)` slot, which `getTableName` reads; `table["_"].name` is
416
+ * `undefined` for these tables, so the raw-SQL Postgres path must not depend on
417
+ * it. Mirrors how the Drizzle `db.insert(table)` path resolves the name on
418
+ * SQLite.
419
+ * @param {unknown} table
420
+ * @returns {string}
421
+ */
422
+ function resolveOutputTableName(table) {
423
+ try {
424
+ return getTableName(/** @type {Table} */ (table));
425
+ }
426
+ catch {
427
+ return "output";
428
+ }
429
+ }
412
430
  /**
413
431
  * @param {unknown} db
414
432
  * @returns {{ run: (sql: string) => unknown; query: (sql: string) => { run: (...args: unknown[]) => unknown; get: (...args: unknown[]) => Record<string, unknown> | null | undefined; all: () => Array<Record<string, unknown>> }; exec: (sql: string) => unknown; $client?: unknown }}
@@ -505,6 +523,9 @@ export class SmithersDb {
505
523
  }),
506
524
  });
507
525
  return yield* self.read(`raw query ${validatedQuery.slice(0, 20)}`, () => {
526
+ if (self.internalStorage.dialect === POSTGRES) {
527
+ return self.internalStorage.queryAllRaw(validatedQuery);
528
+ }
508
529
  const client = self.db.session.client;
509
530
  const stmt = client.query(validatedQuery);
510
531
  return Promise.resolve(stmt.all());
@@ -648,7 +669,18 @@ export class SmithersDb {
648
669
  const transactionState = getSqliteTransactionState(resolveSqliteClientKey(self.db));
649
670
  const start = performance.now();
650
671
  return yield* Effect.gen(function* () {
651
- const client = yield* self.getSqliteTransactionClient();
672
+ const isPostgres = self.internalStorage.dialect === POSTGRES;
673
+ const client = isPostgres ? null : yield* self.getSqliteTransactionClient();
674
+ /**
675
+ * Run a transaction-control statement on the active connection,
676
+ * dialect-appropriately: synchronous bun:sqlite client.run for
677
+ * SQLite, async @effect/sql execute (same connection) for Postgres.
678
+ * @param {string} sql
679
+ * @returns {Promise<unknown>}
680
+ */
681
+ const runControl = (sql) => isPostgres
682
+ ? self.internalStorage.execute(sql)
683
+ : Promise.resolve(client.run(sql));
652
684
  /**
653
685
  * @param {"operation" | "commit"} phase
654
686
  * @param {unknown} error
@@ -660,18 +692,11 @@ export class SmithersDb {
660
692
  phase,
661
693
  error: String(error),
662
694
  }));
663
- yield* Effect.sync(() => {
664
- try {
665
- client.run("ROLLBACK");
666
- }
667
- catch {
668
- // ignore rollback failures
669
- }
670
- });
695
+ yield* Effect.promise(() => runControl("ROLLBACK").then(() => undefined, () => undefined));
671
696
  });
672
- yield* Effect.try({
673
- try: () => {
674
- client.run("BEGIN IMMEDIATE");
697
+ yield* Effect.tryPromise({
698
+ try: async () => {
699
+ await runControl(beginTransactionSql(self.internalStorage.dialect));
675
700
  transactionState.depth += 1;
676
701
  transactionState.ownerThread = currentFiberThread;
677
702
  self.transactionDepth = transactionState.depth;
@@ -688,10 +713,8 @@ export class SmithersDb {
688
713
  yield* rollback("operation", operationExit.cause);
689
714
  return yield* Effect.failCause(operationExit.cause);
690
715
  }
691
- const commitExit = yield* Effect.exit(Effect.try({
692
- try: () => {
693
- client.run("COMMIT");
694
- },
716
+ const commitExit = yield* Effect.exit(Effect.tryPromise({
717
+ try: () => runControl("COMMIT"),
695
718
  catch: (cause) => toSmithersError(cause, "commit sqlite transaction", {
696
719
  code: "DB_WRITE_FAILED",
697
720
  details: { writeGroup, phase: "commit" },
@@ -884,9 +907,34 @@ export class SmithersDb {
884
907
  */
885
908
  claimRunForResume(params) {
886
909
  return this.write(`claim stale run ${params.runId}`, () => {
887
- const client = this.db.session.client;
888
910
  const expectedStatus = params.expectedStatus ?? "running";
889
911
  const requireStale = params.requireStale ?? expectedStatus === "running";
912
+ if (this.internalStorage.dialect === POSTGRES) {
913
+ // Null-safe heartbeat compare without wrapping the bigint param in a
914
+ // numeric COALESCE(?, -1): the int4 `-1` literal would force the
915
+ // ms-timestamp param to int4 and overflow. Compare against the
916
+ // bigint column directly so Postgres infers bigint.
917
+ return this.internalStorage
918
+ .queryAllRaw(`UPDATE _smithers_runs
919
+ SET runtime_owner_id = ?, heartbeat_at_ms = ?
920
+ WHERE run_id = ?
921
+ AND status = ?
922
+ AND COALESCE(runtime_owner_id, '') = COALESCE(?, '')
923
+ AND (heartbeat_at_ms IS NOT DISTINCT FROM ?)
924
+ AND (? = 0 OR heartbeat_at_ms IS NULL OR heartbeat_at_ms < ?)
925
+ RETURNING run_id`, [
926
+ params.claimOwnerId,
927
+ params.claimHeartbeatAtMs,
928
+ params.runId,
929
+ expectedStatus,
930
+ params.expectedRuntimeOwnerId,
931
+ params.expectedHeartbeatAtMs,
932
+ requireStale ? 1 : 0,
933
+ params.staleBeforeMs,
934
+ ])
935
+ .then((rows) => rows.length > 0);
936
+ }
937
+ const client = this.db.session.client;
890
938
  client
891
939
  .query(`UPDATE _smithers_runs
892
940
  SET runtime_owner_id = ?, heartbeat_at_ms = ?
@@ -924,19 +972,33 @@ export class SmithersDb {
924
972
  updateClaimedRun(params) {
925
973
  validateRunPatch(params.patch);
926
974
  return this.write(`update claimed run ${params.runId}`, () => {
927
- const client = this.db.session.client;
928
975
  const patchEntries = Object.entries(params.patch);
929
976
  if (patchEntries.length === 0) {
930
977
  return Promise.resolve(true);
931
978
  }
932
979
  const assignments = patchEntries.map(([key]) => `${camelToSnake(key)} = ?`);
980
+ const setArgs = patchEntries.map(([, value]) => value);
981
+ if (this.internalStorage.dialect === POSTGRES) {
982
+ // Null-safe heartbeat compare (see claimRunForResume): IS NOT
983
+ // DISTINCT FROM keeps the bigint param from being coerced to int4
984
+ // by an int4 `-1` sentinel.
985
+ return this.internalStorage
986
+ .queryAllRaw(`UPDATE _smithers_runs
987
+ SET ${assignments.join(", ")}
988
+ WHERE run_id = ?
989
+ AND runtime_owner_id = ?
990
+ AND (heartbeat_at_ms IS NOT DISTINCT FROM ?)
991
+ RETURNING run_id`, [...setArgs, params.runId, params.expectedRuntimeOwnerId, params.expectedHeartbeatAtMs])
992
+ .then((rows) => rows.length > 0);
993
+ }
994
+ const client = this.db.session.client;
933
995
  client
934
996
  .query(`UPDATE _smithers_runs
935
997
  SET ${assignments.join(", ")}
936
998
  WHERE run_id = ?
937
999
  AND runtime_owner_id = ?
938
1000
  AND COALESCE(heartbeat_at_ms, -1) = COALESCE(?, -1)`)
939
- .run(...patchEntries.map(([, value]) => value), params.runId, params.expectedRuntimeOwnerId, params.expectedHeartbeatAtMs);
1001
+ .run(...setArgs, params.runId, params.expectedRuntimeOwnerId, params.expectedHeartbeatAtMs);
940
1002
  return this.internalStorage
941
1003
  .queryOne("SELECT changes() AS count")
942
1004
  .then((row) => Number(row?.count ?? 0) > 0);
@@ -1005,14 +1067,22 @@ export class SmithersDb {
1005
1067
  const target = cols.iteration
1006
1068
  ? [cols.runId, cols.nodeId, cols.iteration]
1007
1069
  : [cols.runId, cols.nodeId];
1008
- const tableName = table?.["_"]?.name ?? "output";
1009
- return this.write(`upsert output ${tableName}`, () => this.db
1010
- .insert(table)
1011
- .values(values)
1012
- .onConflictDoUpdate({
1013
- target: target,
1014
- set: values,
1015
- }));
1070
+ const tableName = resolveOutputTableName(table);
1071
+ const conflictColumns = cols.iteration
1072
+ ? ["runId", "nodeId", "iteration"]
1073
+ : ["runId", "nodeId"];
1074
+ return this.write(`upsert output ${tableName}`, () => {
1075
+ if (this.internalStorage.dialect === POSTGRES) {
1076
+ return this.internalStorage.upsert(tableName, values, conflictColumns);
1077
+ }
1078
+ return this.db
1079
+ .insert(table)
1080
+ .values(values)
1081
+ .onConflictDoUpdate({
1082
+ target: target,
1083
+ set: values,
1084
+ });
1085
+ });
1016
1086
  }
1017
1087
  /**
1018
1088
  * @param {Table} table
@@ -1030,6 +1100,14 @@ export class SmithersDb {
1030
1100
  */
1031
1101
  deleteOutputRow(tableName, key) {
1032
1102
  return this.write(`delete output ${tableName}`, () => {
1103
+ if (this.internalStorage.dialect === POSTGRES) {
1104
+ // PostgreSQL output tables are created from the Zod schema with
1105
+ // snake_case run_id/node_id/iteration columns, so no PRAGMA-based
1106
+ // column discovery is needed.
1107
+ const escapedPg = tableName.replaceAll(`"`, `""`);
1108
+ return this.internalStorage.execute(`DELETE FROM "${escapedPg}"
1109
+ WHERE run_id = ? AND node_id = ? AND iteration = ?`, [key.runId, key.nodeId, key.iteration ?? 0]);
1110
+ }
1033
1111
  const client = this.db.session.client;
1034
1112
  let resolvedTableName = tableName;
1035
1113
  let escapedTableName = resolvedTableName.replaceAll(`"`, `""`);
@@ -1111,9 +1189,16 @@ export class SmithersDb {
1111
1189
  */
1112
1190
  getRawNodeOutput(tableName, runId, nodeId) {
1113
1191
  return runnableEffect(this.read(`get raw node output ${tableName}`, () => {
1114
- const query = sql.raw(`SELECT * FROM "${tableName}" WHERE run_id = '${runId}' AND node_id = '${nodeId}' ORDER BY iteration DESC LIMIT 1`);
1115
- const res = this.db.get(query);
1116
- return Promise.resolve(res ?? null);
1192
+ const escaped = tableName.replaceAll(`"`, `""`);
1193
+ if (this.internalStorage.dialect === POSTGRES) {
1194
+ return this.internalStorage
1195
+ .queryOneRaw(`SELECT * FROM "${escaped}" WHERE run_id = ? AND node_id = ? ORDER BY iteration DESC LIMIT 1`, [runId, nodeId])
1196
+ .then((row) => row ?? null);
1197
+ }
1198
+ const client = this.db.session.client;
1199
+ const stmt = client.query(`SELECT * FROM "${escaped}" WHERE run_id = ? AND node_id = ? ORDER BY iteration DESC LIMIT 1`);
1200
+ const row = stmt.get(runId, nodeId);
1201
+ return Promise.resolve(row ?? null);
1117
1202
  }).pipe(Effect.catchAll(() => Effect.succeed(null))));
1118
1203
  }
1119
1204
  /**
@@ -1126,6 +1211,11 @@ export class SmithersDb {
1126
1211
  getRawNodeOutputForIteration(tableName, runId, nodeId, iteration) {
1127
1212
  return runnableEffect(this.read(`get raw node output ${tableName} iteration ${iteration}`, () => {
1128
1213
  const escaped = tableName.replaceAll(`"`, `""`);
1214
+ if (this.internalStorage.dialect === POSTGRES) {
1215
+ return this.internalStorage
1216
+ .queryOneRaw(`SELECT * FROM "${escaped}" WHERE run_id = ? AND node_id = ? AND iteration = ? LIMIT 1`, [runId, nodeId, iteration])
1217
+ .then((row) => row ?? null);
1218
+ }
1129
1219
  const client = this.db.session.client;
1130
1220
  const stmt = client.query(`SELECT * FROM "${escaped}" WHERE run_id = ? AND node_id = ? AND iteration = ? LIMIT 1`);
1131
1221
  const row = stmt.get(runId, nodeId, iteration);
@@ -1808,6 +1898,75 @@ export class SmithersDb {
1808
1898
  ORDER BY attempt ASC, seq ASC`, [runId, nodeId, iteration]));
1809
1899
  }
1810
1900
  /**
1901
+ * Record a distinct working-copy state (deduped by jj commit id). Upsert so a
1902
+ * re-snapshot of the same tree refreshes the operation handle.
1903
+ * @param {Record<string, unknown>} row
1904
+ * @returns {RunnableEffect<void, SmithersError>}
1905
+ */
1906
+ upsertWorkspaceState(row) {
1907
+ return this.write(`upsert workspace state ${row.jjCommitId}`, () => this.internalStorage.upsert("_smithers_workspace_states", row, ["runId", "jjCwd", "jjCommitId"]));
1908
+ }
1909
+ /**
1910
+ * @param {string} runId
1911
+ * @returns {RunnableEffect<Array<Record<string, unknown>>, SmithersError>}
1912
+ */
1913
+ listWorkspaceStates(runId) {
1914
+ return this.read(`list workspace states ${runId}`, () => this.internalStorage.queryAll(`SELECT *
1915
+ FROM _smithers_workspace_states
1916
+ WHERE run_id = ?
1917
+ ORDER BY created_at_ms ASC`, [runId]));
1918
+ }
1919
+ /**
1920
+ * Record a snapshot checkpoint event. One row per tool/watch boundary; never
1921
+ * deduped, so a Tier 1 boundary always has a seq to bind a resume to.
1922
+ * @param {Record<string, unknown>} row
1923
+ * @returns {RunnableEffect<void, SmithersError>}
1924
+ */
1925
+ insertWorkspaceCheckpoint(row) {
1926
+ return this.write(`insert workspace checkpoint ${row.nodeId}#${row.seq}`, () => this.internalStorage.insertIgnore("_smithers_workspace_checkpoints", row));
1927
+ }
1928
+ /**
1929
+ * @param {string} runId
1930
+ * @returns {RunnableEffect<Array<Record<string, unknown>>, SmithersError>}
1931
+ */
1932
+ listWorkspaceCheckpoints(runId) {
1933
+ return this.read(`list workspace checkpoints ${runId}`, () => this.internalStorage.queryAll(`SELECT *
1934
+ FROM _smithers_workspace_checkpoints
1935
+ WHERE run_id = ?
1936
+ ORDER BY seq ASC`, [runId]));
1937
+ }
1938
+ /**
1939
+ * Prune old workspace states for a run, keeping the most recent maxKeep by
1940
+ * created_at_ms. The latest rows are always kept. Row-value NOT IN avoids the
1941
+ * string-concat collision a naive key would have; portable to SQLite + Postgres.
1942
+ * @param {string} runId
1943
+ * @param {number} [maxKeep]
1944
+ * @returns {RunnableEffect<void, SmithersError>}
1945
+ */
1946
+ pruneWorkspaceStates(runId, maxKeep = 50) {
1947
+ const keep = Math.max(1, maxKeep);
1948
+ return this.write(`prune workspace states ${runId}`, () => this.internalStorage.deleteWhere("_smithers_workspace_states", `run_id = ? AND (jj_cwd, jj_commit_id) NOT IN (
1949
+ SELECT jj_cwd, jj_commit_id FROM _smithers_workspace_states
1950
+ WHERE run_id = ? ORDER BY created_at_ms DESC LIMIT ?)`, [runId, runId, keep]));
1951
+ }
1952
+ /**
1953
+ * Prune old workspace checkpoints, keeping the most recent maxKeepPerScope per
1954
+ * (node_id, iteration, attempt) via a window function. The latest checkpoint per
1955
+ * scope is always kept, so resume-restore targets survive.
1956
+ * @param {string} runId
1957
+ * @param {number} [maxKeepPerScope]
1958
+ * @returns {RunnableEffect<void, SmithersError>}
1959
+ */
1960
+ pruneWorkspaceCheckpoints(runId, maxKeepPerScope = 100) {
1961
+ const keep = Math.max(1, maxKeepPerScope);
1962
+ return this.write(`prune workspace checkpoints ${runId}`, () => this.internalStorage.deleteWhere("_smithers_workspace_checkpoints", `run_id = ? AND (node_id, iteration, attempt, seq) NOT IN (
1963
+ SELECT node_id, iteration, attempt, seq FROM (
1964
+ SELECT node_id, iteration, attempt, seq,
1965
+ ROW_NUMBER() OVER (PARTITION BY node_id, iteration, attempt ORDER BY seq DESC) AS rn
1966
+ FROM _smithers_workspace_checkpoints WHERE run_id = ?
1967
+ ) sub WHERE rn <= ?)`, [runId, runId, keep]));
1968
+ }
1969
+ /**
1811
1970
  * @param {Record<string, unknown>} row
1812
1971
  * @returns {RunnableEffect<void, SmithersError>}
1813
1972
  */
package/src/dialect.js ADDED
@@ -0,0 +1,229 @@
1
+ /**
2
+ * SQL dialect seam for the Smithers persistence layer.
3
+ *
4
+ * Smithers' storage was authored against SQLite (bun:sqlite). This module is the
5
+ * single source of truth for the handful of places where SQLite and PostgreSQL
6
+ * SQL diverge, so the same hand-written SQL the adapter emits can run unchanged
7
+ * against either backend:
8
+ *
9
+ * - placeholder style: SQLite `?` vs PostgreSQL `$1, $2, …`
10
+ * - DDL types: SQLite's permissive `INTEGER`/`REAL`/`BLOB` vs PostgreSQL's
11
+ * stricter `BIGINT`/`DOUBLE PRECISION`/`BYTEA` (millisecond timestamps stored
12
+ * in `INTEGER` overflow PostgreSQL's 32-bit `integer`, hence `BIGINT`)
13
+ * - autoincrement: `INTEGER PRIMARY KEY AUTOINCREMENT` vs `BIGSERIAL PRIMARY KEY`
14
+ * - schema introspection: `PRAGMA table_info` vs `information_schema.columns`
15
+ * - transaction start: `BEGIN IMMEDIATE` (SQLite write-lock) vs `BEGIN`
16
+ *
17
+ * Identifier quoting (`"x"`), `ON CONFLICT … DO UPDATE SET … = excluded.…`, and
18
+ * `CREATE TABLE/INDEX IF NOT EXISTS` are already standard across both dialects,
19
+ * so they need no translation.
20
+ *
21
+ * @typedef {"sqlite" | "postgres"} Dialect
22
+ */
23
+
24
+ export const SQLITE = /** @type {const} */ ("sqlite");
25
+ export const POSTGRES = /** @type {const} */ ("postgres");
26
+
27
+ /**
28
+ * @param {unknown} value
29
+ * @returns {value is Dialect}
30
+ */
31
+ export function isDialect(value) {
32
+ return value === SQLITE || value === POSTGRES;
33
+ }
34
+
35
+ /**
36
+ * @param {string} identifier
37
+ * @returns {string}
38
+ */
39
+ export function quoteIdentifier(identifier) {
40
+ return `"${String(identifier).replaceAll(`"`, `""`)}"`;
41
+ }
42
+
43
+ /**
44
+ * Rewrite positional `?` placeholders to PostgreSQL's `$1, $2, …`. SQLite keeps
45
+ * `?`. A `?` inside a string literal, quoted identifier, or comment is left
46
+ * untouched so only real placeholders are renumbered.
47
+ *
48
+ * @param {Dialect} dialect
49
+ * @param {string} sql
50
+ * @returns {string}
51
+ */
52
+ export function translatePlaceholders(dialect, sql) {
53
+ if (dialect !== POSTGRES) {
54
+ return sql;
55
+ }
56
+ let out = "";
57
+ let index = 1;
58
+ let inSingle = false;
59
+ let inDouble = false;
60
+ let inLineComment = false;
61
+ let inBlockComment = false;
62
+ for (let i = 0; i < sql.length; i++) {
63
+ const ch = sql[i];
64
+ const next = sql[i + 1];
65
+ if (inLineComment) {
66
+ out += ch;
67
+ if (ch === "\n") inLineComment = false;
68
+ continue;
69
+ }
70
+ if (inBlockComment) {
71
+ out += ch;
72
+ if (ch === "*" && next === "/") {
73
+ out += next;
74
+ i++;
75
+ inBlockComment = false;
76
+ }
77
+ continue;
78
+ }
79
+ if (inSingle) {
80
+ out += ch;
81
+ if (ch === "'") {
82
+ if (next === "'") {
83
+ out += next;
84
+ i++;
85
+ } else {
86
+ inSingle = false;
87
+ }
88
+ }
89
+ continue;
90
+ }
91
+ if (inDouble) {
92
+ out += ch;
93
+ if (ch === `"`) {
94
+ if (next === `"`) {
95
+ out += next;
96
+ i++;
97
+ } else {
98
+ inDouble = false;
99
+ }
100
+ }
101
+ continue;
102
+ }
103
+ if (ch === "'") {
104
+ inSingle = true;
105
+ out += ch;
106
+ continue;
107
+ }
108
+ if (ch === `"`) {
109
+ inDouble = true;
110
+ out += ch;
111
+ continue;
112
+ }
113
+ if (ch === "-" && next === "-") {
114
+ inLineComment = true;
115
+ out += ch;
116
+ continue;
117
+ }
118
+ if (ch === "/" && next === "*") {
119
+ inBlockComment = true;
120
+ out += ch;
121
+ continue;
122
+ }
123
+ if (ch === "?") {
124
+ out += `$${index++}`;
125
+ continue;
126
+ }
127
+ out += ch;
128
+ }
129
+ return out;
130
+ }
131
+
132
+ /**
133
+ * Map a single SQLite column type keyword (`INTEGER`, `REAL`, `BLOB`, `TEXT`) to
134
+ * the dialect equivalent. Used by the Zod→DDL generator, which only ever emits
135
+ * `INTEGER` or `TEXT`.
136
+ *
137
+ * @param {Dialect} dialect
138
+ * @param {string} sqliteType
139
+ * @returns {string}
140
+ */
141
+ export function columnType(dialect, sqliteType) {
142
+ if (dialect !== POSTGRES) {
143
+ return sqliteType;
144
+ }
145
+ switch (sqliteType.toUpperCase()) {
146
+ case "INTEGER":
147
+ return "BIGINT";
148
+ case "REAL":
149
+ return "DOUBLE PRECISION";
150
+ case "BLOB":
151
+ return "BYTEA";
152
+ default:
153
+ return sqliteType;
154
+ }
155
+ }
156
+
157
+ /**
158
+ * Translate a complete `CREATE TABLE`/`CREATE INDEX` statement written in SQLite
159
+ * DDL to the target dialect. Only the controlled, internal Smithers DDL passes
160
+ * through here (no user-supplied text), so keyword-level regex substitution is
161
+ * safe. Order matters: the autoincrement primary key is rewritten before the
162
+ * blanket `INTEGER → BIGINT` so it is not double-translated.
163
+ *
164
+ * @param {Dialect} dialect
165
+ * @param {string} ddl
166
+ * @returns {string}
167
+ */
168
+ export function translateDdl(dialect, ddl) {
169
+ if (dialect !== POSTGRES) {
170
+ return ddl;
171
+ }
172
+ return ddl
173
+ .replace(/\bINTEGER\s+PRIMARY\s+KEY\s+AUTOINCREMENT\b/gi, "BIGSERIAL PRIMARY KEY")
174
+ .replace(/\s+AUTOINCREMENT\b/gi, "")
175
+ .replace(/\bBLOB\b/gi, "BYTEA")
176
+ .replace(/\bREAL\b/gi, "DOUBLE PRECISION")
177
+ .replace(/\bINTEGER\b/gi, "BIGINT");
178
+ }
179
+
180
+ /**
181
+ * SQL that lists the column names of a table, normalized to rows shaped
182
+ * `{ name: string }`.
183
+ *
184
+ * @param {Dialect} dialect
185
+ * @param {string} table
186
+ * @returns {{ sql: string; params: ReadonlyArray<string> }}
187
+ */
188
+ export function tableColumnsSql(dialect, table) {
189
+ if (dialect === POSTGRES) {
190
+ return {
191
+ sql: "SELECT column_name AS name FROM information_schema.columns WHERE table_name = ? AND table_schema = current_schema()",
192
+ params: [table],
193
+ };
194
+ }
195
+ return {
196
+ sql: `PRAGMA table_info(${quoteIdentifier(table)})`,
197
+ params: [],
198
+ };
199
+ }
200
+
201
+ /**
202
+ * The statement that begins a write transaction. SQLite takes the write lock
203
+ * eagerly with `BEGIN IMMEDIATE`; PostgreSQL uses a plain `BEGIN`.
204
+ *
205
+ * @param {Dialect} dialect
206
+ * @returns {string}
207
+ */
208
+ export function beginTransactionSql(dialect) {
209
+ return dialect === POSTGRES ? "BEGIN" : "BEGIN IMMEDIATE";
210
+ }
211
+
212
+ /**
213
+ * A scalar SQL expression extracting a top-level string field from a JSON column
214
+ * stored as TEXT. SQLite uses `json_extract(col, '$.key')`; PostgreSQL casts to
215
+ * `json` and uses the `->>` operator. Only simple top-level paths (`$.key`) are
216
+ * used in Smithers' queries.
217
+ *
218
+ * @param {Dialect} dialect
219
+ * @param {string} columnSql Already-quoted/qualified column expression.
220
+ * @param {string} jsonPath SQLite-style path, e.g. `$.nodeId`.
221
+ * @returns {string}
222
+ */
223
+ export function jsonExtractText(dialect, columnSql, jsonPath) {
224
+ if (dialect === POSTGRES) {
225
+ const key = jsonPath.replace(/^\$\./, "").replace(/'/g, "''");
226
+ return `(${columnSql}::json->>'${key}')`;
227
+ }
228
+ return `json_extract(${columnSql}, '${jsonPath.replace(/'/g, "''")}')`;
229
+ }
package/src/ensure.js CHANGED
@@ -14,5 +14,12 @@ export function ensureSmithersTablesEffect(db) {
14
14
  * @param {_BunSQLiteDatabase<Record<string, unknown>>} db
15
15
  */
16
16
  export function ensureSmithersTables(db) {
17
+ // Postgres schema initialization is asynchronous and cannot be driven by
18
+ // runSync. The Postgres/PGlite entry points ensure the schema (awaited)
19
+ // before the engine starts, so this synchronous SQLite helper is a no-op
20
+ // for a Postgres connection descriptor.
21
+ if (db && typeof db === "object" && /** @type {any} */ (db).dialect === "postgres") {
22
+ return;
23
+ }
17
24
  Effect.runSync(ensureSmithersTablesEffect(db));
18
25
  }
@@ -20,3 +20,5 @@ export { smithersMemoryMessages } from "./smithersMemoryMessages.js";
20
20
  export { smithersVectors } from "./smithersVectors.js";
21
21
  export { smithersCron } from "./smithersCron.js";
22
22
  export { smithersSchemaMigrations } from "./smithersSchemaMigrations.js";
23
+ export { smithersWorkspaceStates } from "./smithersWorkspaceStates.js";
24
+ export { smithersWorkspaceCheckpoints } from "./smithersWorkspaceCheckpoints.js";
@@ -0,0 +1,17 @@
1
+ import { integer, primaryKey, sqliteTable, text } from "drizzle-orm/sqlite-core";
2
+ export const smithersWorkspaceCheckpoints = sqliteTable("_smithers_workspace_checkpoints", {
3
+ runId: text("run_id").notNull(),
4
+ nodeId: text("node_id").notNull(),
5
+ iteration: integer("iteration").notNull().default(0),
6
+ attempt: integer("attempt").notNull(),
7
+ seq: integer("seq").notNull(),
8
+ jjCwd: text("jj_cwd").notNull(),
9
+ jjCommitId: text("jj_commit_id").notNull(),
10
+ source: text("source").notNull(),
11
+ tier: integer("tier").notNull(),
12
+ label: text("label"),
13
+ toolUseId: text("tool_use_id"),
14
+ createdAtMs: integer("created_at_ms").notNull(),
15
+ }, (t) => ({
16
+ pk: primaryKey({ columns: [t.runId, t.nodeId, t.iteration, t.attempt, t.seq] }),
17
+ }));
@@ -0,0 +1,11 @@
1
+ import { integer, primaryKey, sqliteTable, text } from "drizzle-orm/sqlite-core";
2
+ export const smithersWorkspaceStates = sqliteTable("_smithers_workspace_states", {
3
+ runId: text("run_id").notNull(),
4
+ jjCwd: text("jj_cwd").notNull(),
5
+ jjCommitId: text("jj_commit_id").notNull(),
6
+ jjOperationId: text("jj_operation_id").notNull(),
7
+ jjChangeId: text("jj_change_id"),
8
+ createdAtMs: integer("created_at_ms").notNull(),
9
+ }, (t) => ({
10
+ pk: primaryKey({ columns: [t.runId, t.jjCwd, t.jjCommitId] }),
11
+ }));
@@ -268,3 +268,5 @@ export const smithersSchemaMigrations = sqliteTable("_smithers_schema_migrations
268
268
  .default(false),
269
269
  detailsJson: text("details_json"),
270
270
  });
271
+ export { smithersWorkspaceStates } from "./internal-schema/smithersWorkspaceStates.js";
272
+ export { smithersWorkspaceCheckpoints } from "./internal-schema/smithersWorkspaceCheckpoints.js";
@@ -13,16 +13,18 @@ import { SmithersError } from "@smithers-orchestrator/errors/SmithersError";
13
13
  * @returns {Effect.Effect<Record<string, unknown> | undefined, SmithersError>}
14
14
  */
15
15
  export function loadInput(db, inputTable, runId) {
16
- const cols = getTableColumns(inputTable);
17
- const runIdCol = cols.runId;
18
- if (!runIdCol) {
19
- throw new SmithersError("DB_MISSING_COLUMNS", "schema.input must include runId column");
20
- }
21
- return Effect.tryPromise({
22
- try: () => db.select().from(inputTable).where(eq(runIdCol, runId)).limit(1),
23
- catch: (cause) => toSmithersError(cause, "load input", {
24
- code: "DB_QUERY_FAILED",
25
- details: { runId },
26
- }),
27
- }).pipe(Effect.map((rows) => rows[0]), Effect.annotateLogs({ runId }), Effect.withLogSpan("db:load-input"));
16
+ return Effect.suspend(() => {
17
+ const cols = getTableColumns(inputTable);
18
+ const runIdCol = cols.runId;
19
+ if (!runIdCol) {
20
+ return Effect.fail(new SmithersError("DB_MISSING_COLUMNS", "schema.input must include runId column"));
21
+ }
22
+ return Effect.tryPromise({
23
+ try: () => db.select().from(inputTable).where(eq(runIdCol, runId)).limit(1),
24
+ catch: (cause) => toSmithersError(cause, "load input", {
25
+ code: "DB_QUERY_FAILED",
26
+ details: { runId },
27
+ }),
28
+ }).pipe(Effect.map((rows) => rows[0]));
29
+ }).pipe(Effect.annotateLogs({ runId }), Effect.withLogSpan("db:load-input"));
28
30
  }
package/src/output.js CHANGED
@@ -1,10 +1,11 @@
1
- import { and, eq } from "drizzle-orm";
1
+ import { and, eq, getTableName } from "drizzle-orm";
2
2
  import { getTableColumns } from "drizzle-orm/utils";
3
3
  import { createInsertSchema, createSelectSchema } from "drizzle-zod";
4
4
  import { Effect } from "effect";
5
5
  import { z } from "zod";
6
6
  import { SmithersError } from "@smithers-orchestrator/errors/SmithersError";
7
7
  import { toSmithersError } from "@smithers-orchestrator/errors/toSmithersError";
8
+ import { getJsonColumnKeys, isPostgresDb, pgRowToDrizzle } from "./snapshot.js";
8
9
  import { withSqliteWriteRetryEffect } from "./write-retry.js";
9
10
  /** @typedef {import("drizzle-orm").AnyColumn} AnyColumn */
10
11
  /** @typedef {import("./output/OutputKey.ts").OutputKey} _OutputKey */
@@ -76,9 +77,25 @@ export function buildKeyWhere(table, key) {
76
77
  * @returns {Effect.Effect<T | undefined, SmithersError>}
77
78
  */
78
79
  export function selectOutputRowEffect(db, table, key) {
79
- const where = buildKeyWhere(table, key);
80
+ const cols = getKeyColumns(table);
81
+ const hasIteration = Boolean(cols.iteration);
82
+ const jsonKeys = getJsonColumnKeys(table);
80
83
  return Effect.tryPromise({
81
- try: () => db.select().from(table).where(where).limit(1),
84
+ try: () => {
85
+ if (isPostgresDb(db)) {
86
+ const tableName = getTableName(table).replaceAll(`"`, `""`);
87
+ const clauses = ["run_id = $1", "node_id = $2"];
88
+ const values = [key.runId, key.nodeId];
89
+ if (hasIteration) {
90
+ clauses.push("iteration = $3");
91
+ values.push(key.iteration ?? 0);
92
+ }
93
+ return db.connection
94
+ .query({ text: `SELECT * FROM "${tableName}" WHERE ${clauses.join(" AND ")} LIMIT 1`, values })
95
+ .then((result) => (result.rows[0] ? [pgRowToDrizzle(result.rows[0], jsonKeys)] : []));
96
+ }
97
+ return db.select().from(table).where(buildKeyWhere(table, key)).limit(1);
98
+ },
82
99
  catch: (cause) => toSmithersError(cause, `select output ${table["_"]?.name ?? "output"}`, {
83
100
  code: "DB_QUERY_FAILED",
84
101
  details: { outputTable: table["_"]?.name ?? "output" },
@@ -1,5 +1,6 @@
1
1
  import { eq } from "drizzle-orm";
2
2
  import { drizzle } from "drizzle-orm/bun-sqlite";
3
+ import { POSTGRES, translateDdl } from "./dialect.js";
3
4
  import { smithersSchemaMigrations } from "./internal-schema/smithersSchemaMigrations.js";
4
5
 
5
6
  const MIGRATION_TABLE_SQL = `CREATE TABLE IF NOT EXISTS _smithers_schema_migrations (
@@ -452,6 +453,26 @@ function buildMigrations(context) {
452
453
  return { statementCount: context.createIndexStatements.length + EXTRA_INDEX_STATEMENTS.length };
453
454
  },
454
455
  },
456
+ {
457
+ id: "0015_add_workspace_states",
458
+ name: "Add workspace snapshot states table",
459
+ checksum: "packages/db/migrations/0015_add_workspace_states.sql",
460
+ isApplied: (sqlite) => tableExists(sqlite, "_smithers_workspace_states"),
461
+ up: (sqlite) => {
462
+ sqlite.run(createTableStatementFor("_smithers_workspace_states", context.createTableStatements));
463
+ return { table: "_smithers_workspace_states" };
464
+ },
465
+ },
466
+ {
467
+ id: "0016_add_workspace_checkpoints",
468
+ name: "Add workspace snapshot checkpoints table",
469
+ checksum: "packages/db/migrations/0016_add_workspace_checkpoints.sql",
470
+ isApplied: (sqlite) => tableExists(sqlite, "_smithers_workspace_checkpoints"),
471
+ up: (sqlite) => {
472
+ sqlite.run(createTableStatementFor("_smithers_workspace_checkpoints", context.createTableStatements));
473
+ return { table: "_smithers_workspace_checkpoints" };
474
+ },
475
+ },
455
476
  ];
456
477
  }
457
478
 
@@ -479,3 +500,30 @@ export function runSmithersSchemaMigrations(sqlite, context) {
479
500
  applied.add(migration.id);
480
501
  }
481
502
  }
503
+
504
+ /**
505
+ * PostgreSQL schema initialization. A fresh PostgreSQL database starts at the
506
+ * current schema, so it skips the SQLite-only legacy column/foreign-key
507
+ * migrations entirely and simply creates every current table and index
508
+ * idempotently, with DDL translated to PostgreSQL types. Tables are created in
509
+ * declaration order, which keeps foreign-key references valid (referenced
510
+ * tables precede their dependents).
511
+ *
512
+ * @param {{ query: (config: { text: string }) => Promise<unknown> }} pgConn
513
+ * @param {{
514
+ * createTableStatements: readonly string[];
515
+ * createIndexStatements: readonly string[];
516
+ * }} context
517
+ * @returns {Promise<void>}
518
+ */
519
+ export async function runSmithersSchemaInitPostgres(pgConn, context) {
520
+ const statements = [
521
+ MIGRATION_TABLE_SQL,
522
+ ...context.createTableStatements,
523
+ ...context.createIndexStatements,
524
+ ...EXTRA_INDEX_STATEMENTS,
525
+ ];
526
+ for (const statement of statements) {
527
+ await pgConn.query({ text: translateDdl(POSTGRES, statement) });
528
+ }
529
+ }
package/src/snapshot.js CHANGED
@@ -27,6 +27,29 @@ function getBooleanColumnKeys(table) {
27
27
  return [];
28
28
  }
29
29
  }
30
+ /**
31
+ * Keys of every json-mode column on the table. zodToTable maps any
32
+ * array/object/union/complex Zod field to a Drizzle `text(col,{mode:'json'})`
33
+ * column, which Drizzle's bun:sqlite reader auto-decodes on read. The Postgres
34
+ * path stores those as TEXT, so we must JSON.parse them to match.
35
+ * @param {_Table} table
36
+ * @returns {string[]}
37
+ */
38
+ export function getJsonColumnKeys(table) {
39
+ try {
40
+ const cols = getTableColumns(table);
41
+ const keys = [];
42
+ for (const [key, col] of Object.entries(cols)) {
43
+ const c = /** @type {Record<string, unknown> & { config?: { mode?: string } }} */ (/** @type {unknown} */ (col));
44
+ if (c?.columnType === "SQLiteTextJson" || c?.config?.mode === "json" || c?.mode === "json" || c?.dataType === "json") {
45
+ keys.push(key);
46
+ }
47
+ }
48
+ return keys;
49
+ } catch {
50
+ return [];
51
+ }
52
+ }
30
53
  /**
31
54
  * @param {ReadonlyArray<Record<string, unknown>>} rows
32
55
  * @param {readonly string[]} boolKeys
@@ -52,19 +75,73 @@ function coerceBooleanColumns(rows, boolKeys) {
52
75
  * @param {string} runId
53
76
  * @returns {Effect.Effect<Record<string, unknown> | undefined, SmithersError>}
54
77
  */
55
- export function loadInputEffect(db, inputTable, runId) {
56
- const cols = getTableColumns(inputTable);
57
- const runIdCol = cols.runId;
58
- if (!runIdCol) {
59
- throw new SmithersError("DB_MISSING_COLUMNS", "schema.input must include runId column");
78
+ /**
79
+ * @param {unknown} db
80
+ * @returns {boolean}
81
+ */
82
+ export function isPostgresDb(db) {
83
+ return Boolean(db && typeof db === "object" && /** @type {any} */ (db).dialect === "postgres" && /** @type {any} */ (db).connection);
84
+ }
85
+ /**
86
+ * Map a raw node-postgres row (snake_case columns, JSON stored as TEXT) into the
87
+ * shape Drizzle's bun:sqlite reader returns (camelCase keys, json-mode columns
88
+ * decoded), so input/output consumers stay dialect-agnostic. `jsonKeys` is the
89
+ * set of camelCase json-mode column keys (from getJsonColumnKeys); every TEXT
90
+ * value for those columns is JSON.parsed to match Drizzle's mode:'json' read.
91
+ * The literal `payload` column is always decoded so callers that omit `jsonKeys`
92
+ * (single-value outputs) keep working.
93
+ * @param {Record<string, unknown>} row
94
+ * @param {readonly string[]} [jsonKeys]
95
+ * @returns {Record<string, unknown>}
96
+ */
97
+ export function pgRowToDrizzle(row, jsonKeys) {
98
+ /** @type {Record<string, unknown>} */
99
+ const out = {};
100
+ const jsonSet = new Set(jsonKeys ?? []);
101
+ for (const [columnName, value] of Object.entries(row)) {
102
+ const camel = columnName.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase());
103
+ if ((camel === "payload" || jsonSet.has(camel)) && typeof value === "string") {
104
+ try {
105
+ out[camel] = JSON.parse(value);
106
+ }
107
+ catch {
108
+ out[camel] = value;
109
+ }
110
+ }
111
+ else {
112
+ out[camel] = value;
113
+ }
60
114
  }
61
- return Effect.tryPromise({
62
- try: () => db.select().from(inputTable).where(eq(runIdCol, runId)).limit(1),
63
- catch: (cause) => toSmithersError(cause, "load input", {
64
- code: "DB_QUERY_FAILED",
65
- details: { runId },
66
- }),
67
- }).pipe(Effect.map((rows) => rows[0]), Effect.annotateLogs({ runId }), Effect.withLogSpan("db:load-input"));
115
+ return out;
116
+ }
117
+ export function loadInputEffect(db, inputTable, runId) {
118
+ return Effect.suspend(() => {
119
+ const cols = getTableColumns(inputTable);
120
+ const runIdCol = cols.runId;
121
+ if (!runIdCol) {
122
+ return Effect.fail(new SmithersError("DB_MISSING_COLUMNS", "schema.input must include runId column"));
123
+ }
124
+ if (isPostgresDb(db)) {
125
+ const tableName = getTableName(inputTable).replaceAll(`"`, `""`);
126
+ const jsonKeys = getJsonColumnKeys(inputTable);
127
+ return Effect.tryPromise({
128
+ try: () => db.connection
129
+ .query({ text: `SELECT * FROM "${tableName}" WHERE run_id = $1 LIMIT 1`, values: [runId] })
130
+ .then((result) => (result.rows[0] ? pgRowToDrizzle(result.rows[0], jsonKeys) : undefined)),
131
+ catch: (cause) => toSmithersError(cause, "load input", {
132
+ code: "DB_QUERY_FAILED",
133
+ details: { runId },
134
+ }),
135
+ });
136
+ }
137
+ return Effect.tryPromise({
138
+ try: () => db.select().from(inputTable).where(eq(runIdCol, runId)).limit(1),
139
+ catch: (cause) => toSmithersError(cause, "load input", {
140
+ code: "DB_QUERY_FAILED",
141
+ details: { runId },
142
+ }),
143
+ }).pipe(Effect.map((rows) => rows[0]));
144
+ }).pipe(Effect.annotateLogs({ runId }), Effect.withLogSpan("db:load-input"));
68
145
  }
69
146
  /**
70
147
  * @param {BunSQLiteDatabase<Record<string, unknown>>} db
@@ -102,8 +179,13 @@ export function loadOutputsEffect(db, schema, runId) {
102
179
  }).pipe(Effect.option);
103
180
  if (Option.isNone(tableNameOpt)) continue;
104
181
  const tableName = tableNameOpt.value;
182
+ const jsonKeys = getJsonColumnKeys(/** @type {_Table} */ (table));
105
183
  const rawRows = yield* Effect.tryPromise({
106
- try: () => db.select().from(/** @type {_Table} */ (table)).where(eq(runIdCol, runId)),
184
+ try: () => isPostgresDb(db)
185
+ ? db.connection
186
+ .query({ text: `SELECT * FROM "${tableName.replaceAll(`"`, `""`)}" WHERE run_id = $1`, values: [runId] })
187
+ .then((result) => result.rows.map((r) => pgRowToDrizzle(r, jsonKeys)))
188
+ : db.select().from(/** @type {_Table} */ (table)).where(eq(runIdCol, runId)),
107
189
  catch: (cause) => toSmithersError(cause, `load outputs ${tableName}`, { code: "DB_QUERY_FAILED", details: { runId, tableName } }),
108
190
  });
109
191
  const boolKeys = getBooleanColumnKeys(/** @type {_Table} */ (table));
@@ -123,3 +205,43 @@ export function loadOutputsEffect(db, schema, runId) {
123
205
  export function loadOutputs(db, schema, runId) {
124
206
  return Effect.runPromise(loadOutputsEffect(db, schema, runId));
125
207
  }
208
+ /**
209
+ * Read every row of a single output table for a run, returning Drizzle-shaped
210
+ * rows (camelCase keys, boolean columns coerced to JS booleans). Dialect-aware:
211
+ * Drizzle for bun:sqlite, a raw `$n` query for the Postgres descriptor.
212
+ * @param {unknown} db
213
+ * @param {_Table} table
214
+ * @param {string} [runId]
215
+ * @returns {Effect.Effect<Array<Record<string, unknown>>, SmithersError>}
216
+ */
217
+ export function loadRunOutputRowsEffect(db, table, runId) {
218
+ return Effect.gen(function* () {
219
+ const cols = getTableColumns(table);
220
+ const runIdCol = cols.runId;
221
+ const tableName = getTableName(table);
222
+ const boolKeys = getBooleanColumnKeys(table);
223
+ const jsonKeys = getJsonColumnKeys(table);
224
+ const rawRows = yield* Effect.tryPromise({
225
+ try: () => {
226
+ if (isPostgresDb(db)) {
227
+ const escaped = tableName.replaceAll(`"`, `""`);
228
+ const text = runId && runIdCol
229
+ ? `SELECT * FROM "${escaped}" WHERE run_id = $1`
230
+ : `SELECT * FROM "${escaped}"`;
231
+ const values = runId && runIdCol ? [runId] : [];
232
+ return db.connection
233
+ .query({ text, values })
234
+ .then((result) => result.rows.map((r) => pgRowToDrizzle(r, jsonKeys)));
235
+ }
236
+ return runId && runIdCol
237
+ ? db.select().from(table).where(eq(runIdCol, runId))
238
+ : db.select().from(table);
239
+ },
240
+ catch: (cause) => toSmithersError(cause, `load run output ${tableName}`, {
241
+ code: "DB_QUERY_FAILED",
242
+ details: { runId, tableName },
243
+ }),
244
+ });
245
+ return coerceBooleanColumns(rawRows, boolKeys);
246
+ }).pipe(Effect.annotateLogs({ runId: runId ?? "" }), Effect.withLogSpan("db:load-run-output"));
247
+ }
@@ -3,8 +3,17 @@ import * as SqlClient from "@effect/sql/SqlClient";
3
3
  import { SqlError } from "@effect/sql/SqlError";
4
4
  import * as Statement from "@effect/sql/Statement";
5
5
  import { Database } from "bun:sqlite";
6
- import { Context, Effect, Layer, ManagedRuntime, Scope } from "effect";
7
- import { runSmithersSchemaMigrations } from "./schema-migrations.js";
6
+ import { Context, Effect, Layer, ManagedRuntime, Scope, Stream } from "effect";
7
+ import {
8
+ POSTGRES,
9
+ SQLITE,
10
+ jsonExtractText,
11
+ translatePlaceholders,
12
+ } from "./dialect.js";
13
+ import {
14
+ runSmithersSchemaMigrations,
15
+ runSmithersSchemaInitPostgres,
16
+ } from "./schema-migrations.js";
8
17
  import { camelToSnake } from "./utils/camelToSnake.js";
9
18
  /** @typedef {import("drizzle-orm/bun-sqlite").BunSQLiteDatabase} BunSQLiteDatabase */
10
19
  /** @typedef {import("./SqlMessageStorageEventHistoryQuery.ts").SqlMessageStorageEventHistoryQuery} SqlMessageStorageEventHistoryQuery */
@@ -205,6 +214,30 @@ const CREATE_TABLE_STATEMENTS = [
205
214
  status TEXT NOT NULL,
206
215
  error_json TEXT,
207
216
  PRIMARY KEY (run_id, node_id, iteration, attempt, seq)
217
+ )`,
218
+ `CREATE TABLE IF NOT EXISTS _smithers_workspace_states (
219
+ run_id TEXT NOT NULL,
220
+ jj_cwd TEXT NOT NULL,
221
+ jj_commit_id TEXT NOT NULL,
222
+ jj_operation_id TEXT NOT NULL,
223
+ jj_change_id TEXT,
224
+ created_at_ms INTEGER NOT NULL,
225
+ PRIMARY KEY (run_id, jj_cwd, jj_commit_id)
226
+ )`,
227
+ `CREATE TABLE IF NOT EXISTS _smithers_workspace_checkpoints (
228
+ run_id TEXT NOT NULL,
229
+ node_id TEXT NOT NULL,
230
+ iteration INTEGER NOT NULL DEFAULT 0,
231
+ attempt INTEGER NOT NULL,
232
+ seq INTEGER NOT NULL,
233
+ jj_cwd TEXT NOT NULL,
234
+ jj_commit_id TEXT NOT NULL,
235
+ source TEXT NOT NULL,
236
+ tier INTEGER NOT NULL,
237
+ label TEXT,
238
+ tool_use_id TEXT,
239
+ created_at_ms INTEGER NOT NULL,
240
+ PRIMARY KEY (run_id, node_id, iteration, attempt, seq)
208
241
  )`,
209
242
  `CREATE TABLE IF NOT EXISTS _smithers_events (
210
243
  run_id TEXT NOT NULL,
@@ -393,14 +426,17 @@ function applyBooleanColumns(row, booleanColumns) {
393
426
  * @param {Record<string, unknown>} row
394
427
  * @param {{ orIgnore?: boolean; conflictColumns?: readonly string[]; updateColumns?: readonly string[]; }} [options]
395
428
  */
396
- function buildInsertSql(table, row, options) {
429
+ function buildInsertSql(table, row, options, dialect = SQLITE) {
397
430
  const entries = Object.entries(row).filter(([, value]) => value !== undefined);
398
431
  const columns = entries.map(([key]) => camelToSnake(key));
399
432
  const params = entries.map(([, value]) => encodeParam(value));
400
433
  const tableSql = quoteIdentifier(table);
401
434
  const columnSql = columns.map(quoteIdentifier).join(", ");
402
435
  const placeholderSql = columns.map(() => "?").join(", ");
403
- let statement = `INSERT${options?.orIgnore ? " OR IGNORE" : ""} INTO ${tableSql} (${columnSql}) ` +
436
+ // SQLite spells the ignore-on-conflict shorthand `INSERT OR IGNORE`;
437
+ // PostgreSQL has no such prefix and instead appends `ON CONFLICT DO NOTHING`.
438
+ const orIgnorePrefix = options?.orIgnore && dialect !== POSTGRES ? " OR IGNORE" : "";
439
+ let statement = `INSERT${orIgnorePrefix} INTO ${tableSql} (${columnSql}) ` +
404
440
  `VALUES (${placeholderSql})`;
405
441
  if (options?.conflictColumns && options.conflictColumns.length > 0) {
406
442
  const conflictSql = options.conflictColumns.map(camelToSnake).map(quoteIdentifier).join(", ");
@@ -417,6 +453,9 @@ function buildInsertSql(table, row, options) {
417
453
  statement += ` ON CONFLICT (${conflictSql}) DO UPDATE SET ${updateSql}`;
418
454
  }
419
455
  }
456
+ else if (options?.orIgnore && dialect === POSTGRES) {
457
+ statement += ` ON CONFLICT DO NOTHING`;
458
+ }
420
459
  return { statement, params };
421
460
  }
422
461
  /**
@@ -494,7 +533,7 @@ function createConnection(sqlite) {
494
533
  }
495
534
  }),
496
535
  executeUnprepared: (statement, params, transformRows) => execute(statement, params, transformRows),
497
- executeStream: () => Effect.dieMessage("executeStream not implemented"),
536
+ executeStream: (statement, params, transformRows) => Stream.fromIterableEffect(execute(statement, params, transformRows)),
498
537
  };
499
538
  }
500
539
  /**
@@ -524,18 +563,118 @@ function makeSqlClientEffect(sqlite) {
524
563
  function makeSqlClientLayer(sqlite) {
525
564
  return Layer.scoped(SqlClient.SqlClient, makeSqlClientEffect(sqlite));
526
565
  }
566
+ /**
567
+ * @param {SqliteParam} value
568
+ * @returns {unknown}
569
+ */
570
+ function toPostgresParam(value) {
571
+ // node-postgres maps Buffer → bytea; a bare Uint8Array does not round-trip.
572
+ if (value instanceof Uint8Array || Buffer.isBuffer(value)) {
573
+ return Buffer.isBuffer(value) ? value : Buffer.from(value);
574
+ }
575
+ // Smithers mirrors SQLite's storage model on Postgres: JSON lives in TEXT
576
+ // columns. SQLite/Drizzle stringify objects automatically; node-postgres does
577
+ // not, so do it here to keep a single encoding contract across dialects.
578
+ if (value !== null && typeof value === "object" && !(value instanceof Date)) {
579
+ return JSON.stringify(value);
580
+ }
581
+ return value;
582
+ }
583
+ /**
584
+ * A `@effect/sql` connection backed by a single node-postgres connection (any
585
+ * object exposing `query({ text, values, rowMode })` — a `pg.Client`, or a
586
+ * PGlite socket connection). Smithers writes SQL with `?` placeholders; this
587
+ * connection rewrites them to PostgreSQL's `$n` on the way out, mirroring the
588
+ * SQLite connection so the rest of the adapter is dialect-agnostic.
589
+ * @param {{ query: (config: { text: string; values?: ReadonlyArray<unknown>; rowMode?: "array" }) => Promise<{ rows?: ReadonlyArray<any> }> }} pgConn
590
+ * @returns {Connection}
591
+ */
592
+ function createPostgresConnection(pgConn) {
593
+ const run = (statement, params, transformRows) => Effect.tryPromise({
594
+ try: async () => {
595
+ const text = translatePlaceholders(POSTGRES, statement);
596
+ const result = await pgConn.query({ text, values: params.map(toPostgresParam) });
597
+ const rows = result.rows ?? [];
598
+ return transformRows ? transformRows(rows) : rows;
599
+ },
600
+ catch: (cause) => new SqlError({ cause, message: "Failed to execute Postgres statement" }),
601
+ });
602
+ return {
603
+ execute: (statement, params, transformRows) => run(statement, params, transformRows),
604
+ executeRaw: (statement, params) => run(statement, params, undefined),
605
+ executeValues: (statement, params) => Effect.tryPromise({
606
+ try: async () => {
607
+ const text = translatePlaceholders(POSTGRES, statement);
608
+ const result = await pgConn.query({
609
+ text,
610
+ values: params.map(toPostgresParam),
611
+ rowMode: "array",
612
+ });
613
+ return result.rows ?? [];
614
+ },
615
+ catch: (cause) => new SqlError({ cause, message: "Failed to execute Postgres values statement" }),
616
+ }),
617
+ executeUnprepared: (statement, params, transformRows) => run(statement, params, transformRows),
618
+ executeStream: (statement, params, transformRows) => Stream.fromIterableEffect(run(statement, params, transformRows)),
619
+ };
620
+ }
621
+ /**
622
+ * @param {object} pgConn
623
+ * @returns {Effect.Effect<SqlClient.SqlClient, never>}
624
+ */
625
+ function makePostgresSqlClientEffect(pgConn) {
626
+ // The compiler is only exercised by the `sql``` tagged template, which this
627
+ // storage never uses — every query is a pre-built string run through the raw
628
+ // connection, where the `?`→`$n` rewrite happens. So the SQLite compiler is
629
+ // an inert placeholder here.
630
+ const compiler = Statement.makeCompilerSqlite(camelToSnake);
631
+ const connection = createPostgresConnection(pgConn);
632
+ return Effect.gen(function* () {
633
+ const semaphore = yield* Effect.makeSemaphore(1);
634
+ const acquirer = semaphore.withPermits(1)(Effect.succeed(connection));
635
+ const transactionAcquirer = Effect.uninterruptibleMask((restore) => Effect.as(Effect.zipRight(restore(semaphore.take(1)), Effect.tap(Effect.scope, (scope) => Scope.addFinalizer(scope, semaphore.release(1)))), connection));
636
+ const reactivity = yield* Reactivity.make;
637
+ return yield* SqlClient.make({
638
+ acquirer,
639
+ compiler,
640
+ transactionAcquirer,
641
+ spanAttributes: [[ATTR_DB_SYSTEM_NAME, "postgresql"]],
642
+ transformRows: transformRowKeys,
643
+ }).pipe(Effect.provideService(Reactivity.Reactivity, reactivity));
644
+ });
645
+ }
646
+ /**
647
+ * @param {object} pgConn
648
+ */
649
+ function makePostgresSqlClientLayer(pgConn) {
650
+ return Layer.scoped(SqlClient.SqlClient, makePostgresSqlClientEffect(pgConn));
651
+ }
527
652
  export class SqlMessageStorage {
528
653
  sqlite;
654
+ /** @type {import("./dialect.js").Dialect} */
655
+ dialect;
656
+ /** @type {object | null} */
657
+ pgConn;
529
658
  // TODO(Phase 8): Keep this per-DB runtime until the unified runtime can
530
659
  // inject a scoped SqlClient without rebuilding the per-connection semaphore.
531
660
  runtime;
532
661
  tableColumnsCache = new Map();
533
662
  /**
534
- * @param {BunSQLiteDatabase<any> | Database} db
663
+ * @param {BunSQLiteDatabase<any> | Database | { dialect: "postgres"; connection: object }} db
535
664
  */
536
665
  constructor(db) {
537
- this.sqlite = resolveSqliteDatabase(db);
538
- this.runtime = ManagedRuntime.make(makeSqlClientLayer(this.sqlite));
666
+ if (db && typeof db === "object" && /** @type {any} */ (db).dialect === POSTGRES) {
667
+ this.dialect = POSTGRES;
668
+ this.pgConn = /** @type {any} */ (db).connection;
669
+ this.sqlite = null;
670
+ this.runtime = ManagedRuntime.make(makePostgresSqlClientLayer(this.pgConn));
671
+ }
672
+ else {
673
+ this.dialect = SQLITE;
674
+ this.sqlite = resolveSqliteDatabase(db);
675
+ this.pgConn = null;
676
+ this.runtime = ManagedRuntime.make(makeSqlClientLayer(this.sqlite));
677
+ }
539
678
  }
540
679
  /**
541
680
  * @param {string} table
@@ -546,6 +685,13 @@ export class SqlMessageStorage {
546
685
  if (cached) {
547
686
  return cached;
548
687
  }
688
+ if (this.dialect === POSTGRES) {
689
+ // A fresh PostgreSQL schema has no historical column drift to defend
690
+ // against, and PRAGMA is unavailable. Returning null tells
691
+ // filterKnownColumns to skip filtering, so a genuinely missing column
692
+ // surfaces as a loud insert error rather than a silently dropped field.
693
+ return null;
694
+ }
549
695
  const rows = this.sqlite
550
696
  .query(`PRAGMA table_info(${quoteIdentifier(table)})`)
551
697
  .all();
@@ -562,7 +708,7 @@ export class SqlMessageStorage {
562
708
  */
563
709
  filterKnownColumns(table, row) {
564
710
  const knownColumns = this.getTableColumns(table);
565
- return Object.fromEntries(Object.entries(row).filter(([key, value]) => value !== undefined && knownColumns.has(key)));
711
+ return Object.fromEntries(Object.entries(row).filter(([key, value]) => value !== undefined && (knownColumns === null || knownColumns.has(key))));
566
712
  }
567
713
  /**
568
714
  * @template A, E
@@ -584,6 +730,16 @@ export class SqlMessageStorage {
584
730
  * @returns {Effect.Effect<void, never>}
585
731
  */
586
732
  ensureSchemaEffect() {
733
+ if (this.dialect === POSTGRES) {
734
+ const pgConn = this.pgConn;
735
+ return Effect.tryPromise({
736
+ try: () => runSmithersSchemaInitPostgres(pgConn, {
737
+ createTableStatements: CREATE_TABLE_STATEMENTS,
738
+ createIndexStatements: CREATE_INDEX_STATEMENTS,
739
+ }),
740
+ catch: (cause) => new SqlError({ cause, message: "Failed to initialize Postgres schema" }),
741
+ });
742
+ }
587
743
  const sqlite = this.sqlite;
588
744
  return Effect.sync(() => {
589
745
  runSmithersSchemaMigrations(sqlite, {
@@ -622,6 +778,28 @@ export class SqlMessageStorage {
622
778
  return rows[0];
623
779
  }
624
780
  /**
781
+ * Like {@link queryAll} but returns rows with their on-disk column names (no
782
+ * snake→camel transform). Used for "raw" output-table reads where callers
783
+ * expect the storage column names verbatim.
784
+ * @template T
785
+ * @param {string} statement
786
+ * @param {ReadonlyArray<SqliteParam>} [params]
787
+ * @returns {Promise<Array<T>>}
788
+ */
789
+ queryAllRaw(statement, params = []) {
790
+ return this.withConnection((connection) => connection.execute(statement, params.map(encodeParam), undefined));
791
+ }
792
+ /**
793
+ * @template T
794
+ * @param {string} statement
795
+ * @param {ReadonlyArray<SqliteParam>} [params]
796
+ * @returns {Promise<T | undefined>}
797
+ */
798
+ async queryOneRaw(statement, params = []) {
799
+ const rows = await this.queryAllRaw(statement, params);
800
+ return rows[0];
801
+ }
802
+ /**
625
803
  * @param {string} statement
626
804
  * @param {ReadonlyArray<SqliteParam>} [params]
627
805
  * @returns {Promise<void>}
@@ -636,7 +814,7 @@ export class SqlMessageStorage {
636
814
  */
637
815
  insertIgnore(table, row) {
638
816
  const filteredRow = this.filterKnownColumns(table, row);
639
- const { statement, params } = buildInsertSql(table, filteredRow, { orIgnore: true });
817
+ const { statement, params } = buildInsertSql(table, filteredRow, { orIgnore: true }, this.dialect);
640
818
  return this.execute(statement, params);
641
819
  }
642
820
  /**
@@ -651,7 +829,7 @@ export class SqlMessageStorage {
651
829
  const { statement, params } = buildInsertSql(table, filteredRow, {
652
830
  conflictColumns,
653
831
  updateColumns,
654
- });
832
+ }, this.dialect);
655
833
  return this.execute(statement, params);
656
834
  }
657
835
  /**
@@ -694,7 +872,7 @@ export class SqlMessageStorage {
694
872
  params.push(...query.types);
695
873
  }
696
874
  if (query.nodeId) {
697
- clauses.push("json_extract(payload_json, '$.nodeId') = ?");
875
+ clauses.push(`${jsonExtractText(this.dialect, "payload_json", "$.nodeId")} = ?`);
698
876
  params.push(query.nodeId);
699
877
  }
700
878
  return {
@@ -1,4 +1,5 @@
1
1
  import { unwrapZodType } from "./unwrapZodType.js";
2
+ import { columnType, SQLITE } from "./dialect.js";
2
3
  import { camelToSnake } from "./utils/camelToSnake.js";
3
4
  /**
4
5
  * Determines the Zod base type name from a (possibly unwrapped) Zod type.
@@ -34,18 +35,22 @@ export function zodSchemaColumns(schema) {
34
35
  }
35
36
  /**
36
37
  * Generates a CREATE TABLE IF NOT EXISTS SQL statement from a Zod schema.
37
- * Used for runtime table creation without Drizzle migrations.
38
+ * Used for runtime table creation without Drizzle migrations. Pass
39
+ * `opts.dialect` to emit PostgreSQL-compatible column types; the default
40
+ * (`sqlite`) is byte-identical to the historical output.
38
41
  */
39
42
  export function zodToCreateTableSQL(tableName, schema, opts) {
43
+ const dialect = opts?.dialect ?? SQLITE;
44
+ const integer = columnType(dialect, "INTEGER");
40
45
  const colDefs = opts?.isInput
41
46
  ? [`run_id TEXT NOT NULL PRIMARY KEY`]
42
47
  : [
43
48
  `run_id TEXT NOT NULL`,
44
49
  `node_id TEXT NOT NULL`,
45
- `iteration INTEGER NOT NULL DEFAULT 0`,
50
+ `iteration ${integer} NOT NULL DEFAULT 0`,
46
51
  ];
47
52
  for (const { name, sqliteType } of zodSchemaColumns(schema)) {
48
- colDefs.push(`"${name}" ${sqliteType}`);
53
+ colDefs.push(`"${name}" ${columnType(dialect, sqliteType)}`);
49
54
  }
50
55
  if (!opts?.isInput) {
51
56
  colDefs.push(`PRIMARY KEY (run_id, node_id, iteration)`);