@smithers-orchestrator/db 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -5
- package/src/adapter.js +190 -31
- package/src/dialect.js +229 -0
- package/src/ensure.js +7 -0
- package/src/internal-schema/index.js +2 -0
- package/src/internal-schema/smithersWorkspaceCheckpoints.js +17 -0
- package/src/internal-schema/smithersWorkspaceStates.js +11 -0
- package/src/internal-schema.js +2 -0
- package/src/loadInputEffect.js +14 -12
- package/src/output.js +20 -3
- package/src/schema-migrations.js +48 -0
- package/src/snapshot.js +135 -13
- package/src/sql-message-storage.js +190 -12
- package/src/zodToCreateTableSQL.js +8 -3
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@smithers-orchestrator/db",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.23.0",
|
|
4
4
|
"description": "SQLite and Drizzle persistence adapter for Smithers workflows",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"sideEffects": false,
|
|
@@ -26,13 +26,16 @@
|
|
|
26
26
|
"drizzle-zod": "^0.8.3",
|
|
27
27
|
"effect": "^3.21.1",
|
|
28
28
|
"zod": "^4.3.6",
|
|
29
|
-
"@smithers-orchestrator/errors": "0.
|
|
30
|
-
"@smithers-orchestrator/
|
|
31
|
-
"@smithers-orchestrator/
|
|
32
|
-
"@smithers-orchestrator/
|
|
29
|
+
"@smithers-orchestrator/errors": "0.23.0",
|
|
30
|
+
"@smithers-orchestrator/observability": "0.23.0",
|
|
31
|
+
"@smithers-orchestrator/graph": "0.23.0",
|
|
32
|
+
"@smithers-orchestrator/scheduler": "0.23.0"
|
|
33
33
|
},
|
|
34
34
|
"devDependencies": {
|
|
35
|
+
"@electric-sql/pglite": "0.5.1",
|
|
36
|
+
"@electric-sql/pglite-socket": "0.2.1",
|
|
35
37
|
"@types/bun": "latest",
|
|
38
|
+
"pg": "^8.13.1",
|
|
36
39
|
"typescript": "~5.9.3"
|
|
37
40
|
},
|
|
38
41
|
"scripts": {
|
package/src/adapter.js
CHANGED
|
@@ -10,10 +10,11 @@
|
|
|
10
10
|
/** @typedef {import("./adapter/StaleRunRecord.ts").StaleRunRecord} StaleRunRecord */
|
|
11
11
|
// @smithers-type-exports-end
|
|
12
12
|
|
|
13
|
-
import { getTableName
|
|
13
|
+
import { getTableName } from "drizzle-orm";
|
|
14
14
|
import { Effect, Exit, FiberId, Metric } from "effect";
|
|
15
15
|
import { toSmithersError } from "@smithers-orchestrator/errors/toSmithersError";
|
|
16
16
|
import { getSqlMessageStorage } from "./sql-message-storage.js";
|
|
17
|
+
import { POSTGRES, beginTransactionSql } from "./dialect.js";
|
|
17
18
|
import { alertsAcknowledgedTotal, alertsActive, alertsFiredTotal, dbQueryDuration, dbTransactionDuration, dbTransactionRollbacks, } from "@smithers-orchestrator/observability/metrics";
|
|
18
19
|
import { assertOptionalStringMaxLength, assertPositiveFiniteNumber, } from "./input-bounds.js";
|
|
19
20
|
import { FRAME_KEYFRAME_INTERVAL, applyFrameDeltaJson, encodeFrameDelta, normalizeFrameEncoding, serializeFrameDelta, } from "./frame-codec.js";
|
|
@@ -409,6 +410,23 @@ function getSqliteTransactionState(client) {
|
|
|
409
410
|
}
|
|
410
411
|
return state;
|
|
411
412
|
}
|
|
413
|
+
/**
|
|
414
|
+
* Resolve a Drizzle output table's on-disk name. The name lives on the
|
|
415
|
+
* `Symbol(drizzle:Name)` slot, which `getTableName` reads; `table["_"].name` is
|
|
416
|
+
* `undefined` for these tables, so the raw-SQL Postgres path must not depend on
|
|
417
|
+
* it. Mirrors how the Drizzle `db.insert(table)` path resolves the name on
|
|
418
|
+
* SQLite.
|
|
419
|
+
* @param {unknown} table
|
|
420
|
+
* @returns {string}
|
|
421
|
+
*/
|
|
422
|
+
function resolveOutputTableName(table) {
|
|
423
|
+
try {
|
|
424
|
+
return getTableName(/** @type {Table} */ (table));
|
|
425
|
+
}
|
|
426
|
+
catch {
|
|
427
|
+
return "output";
|
|
428
|
+
}
|
|
429
|
+
}
|
|
412
430
|
/**
|
|
413
431
|
* @param {unknown} db
|
|
414
432
|
* @returns {{ run: (sql: string) => unknown; query: (sql: string) => { run: (...args: unknown[]) => unknown; get: (...args: unknown[]) => Record<string, unknown> | null | undefined; all: () => Array<Record<string, unknown>> }; exec: (sql: string) => unknown; $client?: unknown }}
|
|
@@ -505,6 +523,9 @@ export class SmithersDb {
|
|
|
505
523
|
}),
|
|
506
524
|
});
|
|
507
525
|
return yield* self.read(`raw query ${validatedQuery.slice(0, 20)}`, () => {
|
|
526
|
+
if (self.internalStorage.dialect === POSTGRES) {
|
|
527
|
+
return self.internalStorage.queryAllRaw(validatedQuery);
|
|
528
|
+
}
|
|
508
529
|
const client = self.db.session.client;
|
|
509
530
|
const stmt = client.query(validatedQuery);
|
|
510
531
|
return Promise.resolve(stmt.all());
|
|
@@ -648,7 +669,18 @@ export class SmithersDb {
|
|
|
648
669
|
const transactionState = getSqliteTransactionState(resolveSqliteClientKey(self.db));
|
|
649
670
|
const start = performance.now();
|
|
650
671
|
return yield* Effect.gen(function* () {
|
|
651
|
-
const
|
|
672
|
+
const isPostgres = self.internalStorage.dialect === POSTGRES;
|
|
673
|
+
const client = isPostgres ? null : yield* self.getSqliteTransactionClient();
|
|
674
|
+
/**
|
|
675
|
+
* Run a transaction-control statement on the active connection,
|
|
676
|
+
* dialect-appropriately: synchronous bun:sqlite client.run for
|
|
677
|
+
* SQLite, async @effect/sql execute (same connection) for Postgres.
|
|
678
|
+
* @param {string} sql
|
|
679
|
+
* @returns {Promise<unknown>}
|
|
680
|
+
*/
|
|
681
|
+
const runControl = (sql) => isPostgres
|
|
682
|
+
? self.internalStorage.execute(sql)
|
|
683
|
+
: Promise.resolve(client.run(sql));
|
|
652
684
|
/**
|
|
653
685
|
* @param {"operation" | "commit"} phase
|
|
654
686
|
* @param {unknown} error
|
|
@@ -660,18 +692,11 @@ export class SmithersDb {
|
|
|
660
692
|
phase,
|
|
661
693
|
error: String(error),
|
|
662
694
|
}));
|
|
663
|
-
yield* Effect.
|
|
664
|
-
try {
|
|
665
|
-
client.run("ROLLBACK");
|
|
666
|
-
}
|
|
667
|
-
catch {
|
|
668
|
-
// ignore rollback failures
|
|
669
|
-
}
|
|
670
|
-
});
|
|
695
|
+
yield* Effect.promise(() => runControl("ROLLBACK").then(() => undefined, () => undefined));
|
|
671
696
|
});
|
|
672
|
-
yield* Effect.
|
|
673
|
-
try: () => {
|
|
674
|
-
|
|
697
|
+
yield* Effect.tryPromise({
|
|
698
|
+
try: async () => {
|
|
699
|
+
await runControl(beginTransactionSql(self.internalStorage.dialect));
|
|
675
700
|
transactionState.depth += 1;
|
|
676
701
|
transactionState.ownerThread = currentFiberThread;
|
|
677
702
|
self.transactionDepth = transactionState.depth;
|
|
@@ -688,10 +713,8 @@ export class SmithersDb {
|
|
|
688
713
|
yield* rollback("operation", operationExit.cause);
|
|
689
714
|
return yield* Effect.failCause(operationExit.cause);
|
|
690
715
|
}
|
|
691
|
-
const commitExit = yield* Effect.exit(Effect.
|
|
692
|
-
try: () =>
|
|
693
|
-
client.run("COMMIT");
|
|
694
|
-
},
|
|
716
|
+
const commitExit = yield* Effect.exit(Effect.tryPromise({
|
|
717
|
+
try: () => runControl("COMMIT"),
|
|
695
718
|
catch: (cause) => toSmithersError(cause, "commit sqlite transaction", {
|
|
696
719
|
code: "DB_WRITE_FAILED",
|
|
697
720
|
details: { writeGroup, phase: "commit" },
|
|
@@ -884,9 +907,34 @@ export class SmithersDb {
|
|
|
884
907
|
*/
|
|
885
908
|
claimRunForResume(params) {
|
|
886
909
|
return this.write(`claim stale run ${params.runId}`, () => {
|
|
887
|
-
const client = this.db.session.client;
|
|
888
910
|
const expectedStatus = params.expectedStatus ?? "running";
|
|
889
911
|
const requireStale = params.requireStale ?? expectedStatus === "running";
|
|
912
|
+
if (this.internalStorage.dialect === POSTGRES) {
|
|
913
|
+
// Null-safe heartbeat compare without wrapping the bigint param in a
|
|
914
|
+
// numeric COALESCE(?, -1): the int4 `-1` literal would force the
|
|
915
|
+
// ms-timestamp param to int4 and overflow. Compare against the
|
|
916
|
+
// bigint column directly so Postgres infers bigint.
|
|
917
|
+
return this.internalStorage
|
|
918
|
+
.queryAllRaw(`UPDATE _smithers_runs
|
|
919
|
+
SET runtime_owner_id = ?, heartbeat_at_ms = ?
|
|
920
|
+
WHERE run_id = ?
|
|
921
|
+
AND status = ?
|
|
922
|
+
AND COALESCE(runtime_owner_id, '') = COALESCE(?, '')
|
|
923
|
+
AND (heartbeat_at_ms IS NOT DISTINCT FROM ?)
|
|
924
|
+
AND (? = 0 OR heartbeat_at_ms IS NULL OR heartbeat_at_ms < ?)
|
|
925
|
+
RETURNING run_id`, [
|
|
926
|
+
params.claimOwnerId,
|
|
927
|
+
params.claimHeartbeatAtMs,
|
|
928
|
+
params.runId,
|
|
929
|
+
expectedStatus,
|
|
930
|
+
params.expectedRuntimeOwnerId,
|
|
931
|
+
params.expectedHeartbeatAtMs,
|
|
932
|
+
requireStale ? 1 : 0,
|
|
933
|
+
params.staleBeforeMs,
|
|
934
|
+
])
|
|
935
|
+
.then((rows) => rows.length > 0);
|
|
936
|
+
}
|
|
937
|
+
const client = this.db.session.client;
|
|
890
938
|
client
|
|
891
939
|
.query(`UPDATE _smithers_runs
|
|
892
940
|
SET runtime_owner_id = ?, heartbeat_at_ms = ?
|
|
@@ -924,19 +972,33 @@ export class SmithersDb {
|
|
|
924
972
|
updateClaimedRun(params) {
|
|
925
973
|
validateRunPatch(params.patch);
|
|
926
974
|
return this.write(`update claimed run ${params.runId}`, () => {
|
|
927
|
-
const client = this.db.session.client;
|
|
928
975
|
const patchEntries = Object.entries(params.patch);
|
|
929
976
|
if (patchEntries.length === 0) {
|
|
930
977
|
return Promise.resolve(true);
|
|
931
978
|
}
|
|
932
979
|
const assignments = patchEntries.map(([key]) => `${camelToSnake(key)} = ?`);
|
|
980
|
+
const setArgs = patchEntries.map(([, value]) => value);
|
|
981
|
+
if (this.internalStorage.dialect === POSTGRES) {
|
|
982
|
+
// Null-safe heartbeat compare (see claimRunForResume): IS NOT
|
|
983
|
+
// DISTINCT FROM keeps the bigint param from being coerced to int4
|
|
984
|
+
// by an int4 `-1` sentinel.
|
|
985
|
+
return this.internalStorage
|
|
986
|
+
.queryAllRaw(`UPDATE _smithers_runs
|
|
987
|
+
SET ${assignments.join(", ")}
|
|
988
|
+
WHERE run_id = ?
|
|
989
|
+
AND runtime_owner_id = ?
|
|
990
|
+
AND (heartbeat_at_ms IS NOT DISTINCT FROM ?)
|
|
991
|
+
RETURNING run_id`, [...setArgs, params.runId, params.expectedRuntimeOwnerId, params.expectedHeartbeatAtMs])
|
|
992
|
+
.then((rows) => rows.length > 0);
|
|
993
|
+
}
|
|
994
|
+
const client = this.db.session.client;
|
|
933
995
|
client
|
|
934
996
|
.query(`UPDATE _smithers_runs
|
|
935
997
|
SET ${assignments.join(", ")}
|
|
936
998
|
WHERE run_id = ?
|
|
937
999
|
AND runtime_owner_id = ?
|
|
938
1000
|
AND COALESCE(heartbeat_at_ms, -1) = COALESCE(?, -1)`)
|
|
939
|
-
.run(...
|
|
1001
|
+
.run(...setArgs, params.runId, params.expectedRuntimeOwnerId, params.expectedHeartbeatAtMs);
|
|
940
1002
|
return this.internalStorage
|
|
941
1003
|
.queryOne("SELECT changes() AS count")
|
|
942
1004
|
.then((row) => Number(row?.count ?? 0) > 0);
|
|
@@ -1005,14 +1067,22 @@ export class SmithersDb {
|
|
|
1005
1067
|
const target = cols.iteration
|
|
1006
1068
|
? [cols.runId, cols.nodeId, cols.iteration]
|
|
1007
1069
|
: [cols.runId, cols.nodeId];
|
|
1008
|
-
const tableName = table
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1070
|
+
const tableName = resolveOutputTableName(table);
|
|
1071
|
+
const conflictColumns = cols.iteration
|
|
1072
|
+
? ["runId", "nodeId", "iteration"]
|
|
1073
|
+
: ["runId", "nodeId"];
|
|
1074
|
+
return this.write(`upsert output ${tableName}`, () => {
|
|
1075
|
+
if (this.internalStorage.dialect === POSTGRES) {
|
|
1076
|
+
return this.internalStorage.upsert(tableName, values, conflictColumns);
|
|
1077
|
+
}
|
|
1078
|
+
return this.db
|
|
1079
|
+
.insert(table)
|
|
1080
|
+
.values(values)
|
|
1081
|
+
.onConflictDoUpdate({
|
|
1082
|
+
target: target,
|
|
1083
|
+
set: values,
|
|
1084
|
+
});
|
|
1085
|
+
});
|
|
1016
1086
|
}
|
|
1017
1087
|
/**
|
|
1018
1088
|
* @param {Table} table
|
|
@@ -1030,6 +1100,14 @@ export class SmithersDb {
|
|
|
1030
1100
|
*/
|
|
1031
1101
|
deleteOutputRow(tableName, key) {
|
|
1032
1102
|
return this.write(`delete output ${tableName}`, () => {
|
|
1103
|
+
if (this.internalStorage.dialect === POSTGRES) {
|
|
1104
|
+
// PostgreSQL output tables are created from the Zod schema with
|
|
1105
|
+
// snake_case run_id/node_id/iteration columns, so no PRAGMA-based
|
|
1106
|
+
// column discovery is needed.
|
|
1107
|
+
const escapedPg = tableName.replaceAll(`"`, `""`);
|
|
1108
|
+
return this.internalStorage.execute(`DELETE FROM "${escapedPg}"
|
|
1109
|
+
WHERE run_id = ? AND node_id = ? AND iteration = ?`, [key.runId, key.nodeId, key.iteration ?? 0]);
|
|
1110
|
+
}
|
|
1033
1111
|
const client = this.db.session.client;
|
|
1034
1112
|
let resolvedTableName = tableName;
|
|
1035
1113
|
let escapedTableName = resolvedTableName.replaceAll(`"`, `""`);
|
|
@@ -1111,9 +1189,16 @@ export class SmithersDb {
|
|
|
1111
1189
|
*/
|
|
1112
1190
|
getRawNodeOutput(tableName, runId, nodeId) {
|
|
1113
1191
|
return runnableEffect(this.read(`get raw node output ${tableName}`, () => {
|
|
1114
|
-
const
|
|
1115
|
-
|
|
1116
|
-
|
|
1192
|
+
const escaped = tableName.replaceAll(`"`, `""`);
|
|
1193
|
+
if (this.internalStorage.dialect === POSTGRES) {
|
|
1194
|
+
return this.internalStorage
|
|
1195
|
+
.queryOneRaw(`SELECT * FROM "${escaped}" WHERE run_id = ? AND node_id = ? ORDER BY iteration DESC LIMIT 1`, [runId, nodeId])
|
|
1196
|
+
.then((row) => row ?? null);
|
|
1197
|
+
}
|
|
1198
|
+
const client = this.db.session.client;
|
|
1199
|
+
const stmt = client.query(`SELECT * FROM "${escaped}" WHERE run_id = ? AND node_id = ? ORDER BY iteration DESC LIMIT 1`);
|
|
1200
|
+
const row = stmt.get(runId, nodeId);
|
|
1201
|
+
return Promise.resolve(row ?? null);
|
|
1117
1202
|
}).pipe(Effect.catchAll(() => Effect.succeed(null))));
|
|
1118
1203
|
}
|
|
1119
1204
|
/**
|
|
@@ -1126,6 +1211,11 @@ export class SmithersDb {
|
|
|
1126
1211
|
getRawNodeOutputForIteration(tableName, runId, nodeId, iteration) {
|
|
1127
1212
|
return runnableEffect(this.read(`get raw node output ${tableName} iteration ${iteration}`, () => {
|
|
1128
1213
|
const escaped = tableName.replaceAll(`"`, `""`);
|
|
1214
|
+
if (this.internalStorage.dialect === POSTGRES) {
|
|
1215
|
+
return this.internalStorage
|
|
1216
|
+
.queryOneRaw(`SELECT * FROM "${escaped}" WHERE run_id = ? AND node_id = ? AND iteration = ? LIMIT 1`, [runId, nodeId, iteration])
|
|
1217
|
+
.then((row) => row ?? null);
|
|
1218
|
+
}
|
|
1129
1219
|
const client = this.db.session.client;
|
|
1130
1220
|
const stmt = client.query(`SELECT * FROM "${escaped}" WHERE run_id = ? AND node_id = ? AND iteration = ? LIMIT 1`);
|
|
1131
1221
|
const row = stmt.get(runId, nodeId, iteration);
|
|
@@ -1808,6 +1898,75 @@ export class SmithersDb {
|
|
|
1808
1898
|
ORDER BY attempt ASC, seq ASC`, [runId, nodeId, iteration]));
|
|
1809
1899
|
}
|
|
1810
1900
|
/**
|
|
1901
|
+
* Record a distinct working-copy state (deduped by jj commit id). Upsert so a
|
|
1902
|
+
* re-snapshot of the same tree refreshes the operation handle.
|
|
1903
|
+
* @param {Record<string, unknown>} row
|
|
1904
|
+
* @returns {RunnableEffect<void, SmithersError>}
|
|
1905
|
+
*/
|
|
1906
|
+
upsertWorkspaceState(row) {
|
|
1907
|
+
return this.write(`upsert workspace state ${row.jjCommitId}`, () => this.internalStorage.upsert("_smithers_workspace_states", row, ["runId", "jjCwd", "jjCommitId"]));
|
|
1908
|
+
}
|
|
1909
|
+
/**
|
|
1910
|
+
* @param {string} runId
|
|
1911
|
+
* @returns {RunnableEffect<Array<Record<string, unknown>>, SmithersError>}
|
|
1912
|
+
*/
|
|
1913
|
+
listWorkspaceStates(runId) {
|
|
1914
|
+
return this.read(`list workspace states ${runId}`, () => this.internalStorage.queryAll(`SELECT *
|
|
1915
|
+
FROM _smithers_workspace_states
|
|
1916
|
+
WHERE run_id = ?
|
|
1917
|
+
ORDER BY created_at_ms ASC`, [runId]));
|
|
1918
|
+
}
|
|
1919
|
+
/**
|
|
1920
|
+
* Record a snapshot checkpoint event. One row per tool/watch boundary; never
|
|
1921
|
+
* deduped, so a Tier 1 boundary always has a seq to bind a resume to.
|
|
1922
|
+
* @param {Record<string, unknown>} row
|
|
1923
|
+
* @returns {RunnableEffect<void, SmithersError>}
|
|
1924
|
+
*/
|
|
1925
|
+
insertWorkspaceCheckpoint(row) {
|
|
1926
|
+
return this.write(`insert workspace checkpoint ${row.nodeId}#${row.seq}`, () => this.internalStorage.insertIgnore("_smithers_workspace_checkpoints", row));
|
|
1927
|
+
}
|
|
1928
|
+
/**
|
|
1929
|
+
* @param {string} runId
|
|
1930
|
+
* @returns {RunnableEffect<Array<Record<string, unknown>>, SmithersError>}
|
|
1931
|
+
*/
|
|
1932
|
+
listWorkspaceCheckpoints(runId) {
|
|
1933
|
+
return this.read(`list workspace checkpoints ${runId}`, () => this.internalStorage.queryAll(`SELECT *
|
|
1934
|
+
FROM _smithers_workspace_checkpoints
|
|
1935
|
+
WHERE run_id = ?
|
|
1936
|
+
ORDER BY seq ASC`, [runId]));
|
|
1937
|
+
}
|
|
1938
|
+
/**
|
|
1939
|
+
* Prune old workspace states for a run, keeping the most recent maxKeep by
|
|
1940
|
+
* created_at_ms. The latest rows are always kept. Row-value NOT IN avoids the
|
|
1941
|
+
* string-concat collision a naive key would have; portable to SQLite + Postgres.
|
|
1942
|
+
* @param {string} runId
|
|
1943
|
+
* @param {number} [maxKeep]
|
|
1944
|
+
* @returns {RunnableEffect<void, SmithersError>}
|
|
1945
|
+
*/
|
|
1946
|
+
pruneWorkspaceStates(runId, maxKeep = 50) {
|
|
1947
|
+
const keep = Math.max(1, maxKeep);
|
|
1948
|
+
return this.write(`prune workspace states ${runId}`, () => this.internalStorage.deleteWhere("_smithers_workspace_states", `run_id = ? AND (jj_cwd, jj_commit_id) NOT IN (
|
|
1949
|
+
SELECT jj_cwd, jj_commit_id FROM _smithers_workspace_states
|
|
1950
|
+
WHERE run_id = ? ORDER BY created_at_ms DESC LIMIT ?)`, [runId, runId, keep]));
|
|
1951
|
+
}
|
|
1952
|
+
/**
|
|
1953
|
+
* Prune old workspace checkpoints, keeping the most recent maxKeepPerScope per
|
|
1954
|
+
* (node_id, iteration, attempt) via a window function. The latest checkpoint per
|
|
1955
|
+
* scope is always kept, so resume-restore targets survive.
|
|
1956
|
+
* @param {string} runId
|
|
1957
|
+
* @param {number} [maxKeepPerScope]
|
|
1958
|
+
* @returns {RunnableEffect<void, SmithersError>}
|
|
1959
|
+
*/
|
|
1960
|
+
pruneWorkspaceCheckpoints(runId, maxKeepPerScope = 100) {
|
|
1961
|
+
const keep = Math.max(1, maxKeepPerScope);
|
|
1962
|
+
return this.write(`prune workspace checkpoints ${runId}`, () => this.internalStorage.deleteWhere("_smithers_workspace_checkpoints", `run_id = ? AND (node_id, iteration, attempt, seq) NOT IN (
|
|
1963
|
+
SELECT node_id, iteration, attempt, seq FROM (
|
|
1964
|
+
SELECT node_id, iteration, attempt, seq,
|
|
1965
|
+
ROW_NUMBER() OVER (PARTITION BY node_id, iteration, attempt ORDER BY seq DESC) AS rn
|
|
1966
|
+
FROM _smithers_workspace_checkpoints WHERE run_id = ?
|
|
1967
|
+
) sub WHERE rn <= ?)`, [runId, runId, keep]));
|
|
1968
|
+
}
|
|
1969
|
+
/**
|
|
1811
1970
|
* @param {Record<string, unknown>} row
|
|
1812
1971
|
* @returns {RunnableEffect<void, SmithersError>}
|
|
1813
1972
|
*/
|
package/src/dialect.js
ADDED
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SQL dialect seam for the Smithers persistence layer.
|
|
3
|
+
*
|
|
4
|
+
* Smithers' storage was authored against SQLite (bun:sqlite). This module is the
|
|
5
|
+
* single source of truth for the handful of places where SQLite and PostgreSQL
|
|
6
|
+
* SQL diverge, so the same hand-written SQL the adapter emits can run unchanged
|
|
7
|
+
* against either backend:
|
|
8
|
+
*
|
|
9
|
+
* - placeholder style: SQLite `?` vs PostgreSQL `$1, $2, …`
|
|
10
|
+
* - DDL types: SQLite's permissive `INTEGER`/`REAL`/`BLOB` vs PostgreSQL's
|
|
11
|
+
* stricter `BIGINT`/`DOUBLE PRECISION`/`BYTEA` (millisecond timestamps stored
|
|
12
|
+
* in `INTEGER` overflow PostgreSQL's 32-bit `integer`, hence `BIGINT`)
|
|
13
|
+
* - autoincrement: `INTEGER PRIMARY KEY AUTOINCREMENT` vs `BIGSERIAL PRIMARY KEY`
|
|
14
|
+
* - schema introspection: `PRAGMA table_info` vs `information_schema.columns`
|
|
15
|
+
* - transaction start: `BEGIN IMMEDIATE` (SQLite write-lock) vs `BEGIN`
|
|
16
|
+
*
|
|
17
|
+
* Identifier quoting (`"x"`), `ON CONFLICT … DO UPDATE SET … = excluded.…`, and
|
|
18
|
+
* `CREATE TABLE/INDEX IF NOT EXISTS` are already standard across both dialects,
|
|
19
|
+
* so they need no translation.
|
|
20
|
+
*
|
|
21
|
+
* @typedef {"sqlite" | "postgres"} Dialect
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
export const SQLITE = /** @type {const} */ ("sqlite");
|
|
25
|
+
export const POSTGRES = /** @type {const} */ ("postgres");
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* @param {unknown} value
|
|
29
|
+
* @returns {value is Dialect}
|
|
30
|
+
*/
|
|
31
|
+
export function isDialect(value) {
|
|
32
|
+
return value === SQLITE || value === POSTGRES;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* @param {string} identifier
|
|
37
|
+
* @returns {string}
|
|
38
|
+
*/
|
|
39
|
+
export function quoteIdentifier(identifier) {
|
|
40
|
+
return `"${String(identifier).replaceAll(`"`, `""`)}"`;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Rewrite positional `?` placeholders to PostgreSQL's `$1, $2, …`. SQLite keeps
|
|
45
|
+
* `?`. A `?` inside a string literal, quoted identifier, or comment is left
|
|
46
|
+
* untouched so only real placeholders are renumbered.
|
|
47
|
+
*
|
|
48
|
+
* @param {Dialect} dialect
|
|
49
|
+
* @param {string} sql
|
|
50
|
+
* @returns {string}
|
|
51
|
+
*/
|
|
52
|
+
export function translatePlaceholders(dialect, sql) {
|
|
53
|
+
if (dialect !== POSTGRES) {
|
|
54
|
+
return sql;
|
|
55
|
+
}
|
|
56
|
+
let out = "";
|
|
57
|
+
let index = 1;
|
|
58
|
+
let inSingle = false;
|
|
59
|
+
let inDouble = false;
|
|
60
|
+
let inLineComment = false;
|
|
61
|
+
let inBlockComment = false;
|
|
62
|
+
for (let i = 0; i < sql.length; i++) {
|
|
63
|
+
const ch = sql[i];
|
|
64
|
+
const next = sql[i + 1];
|
|
65
|
+
if (inLineComment) {
|
|
66
|
+
out += ch;
|
|
67
|
+
if (ch === "\n") inLineComment = false;
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
if (inBlockComment) {
|
|
71
|
+
out += ch;
|
|
72
|
+
if (ch === "*" && next === "/") {
|
|
73
|
+
out += next;
|
|
74
|
+
i++;
|
|
75
|
+
inBlockComment = false;
|
|
76
|
+
}
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
if (inSingle) {
|
|
80
|
+
out += ch;
|
|
81
|
+
if (ch === "'") {
|
|
82
|
+
if (next === "'") {
|
|
83
|
+
out += next;
|
|
84
|
+
i++;
|
|
85
|
+
} else {
|
|
86
|
+
inSingle = false;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
if (inDouble) {
|
|
92
|
+
out += ch;
|
|
93
|
+
if (ch === `"`) {
|
|
94
|
+
if (next === `"`) {
|
|
95
|
+
out += next;
|
|
96
|
+
i++;
|
|
97
|
+
} else {
|
|
98
|
+
inDouble = false;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
if (ch === "'") {
|
|
104
|
+
inSingle = true;
|
|
105
|
+
out += ch;
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
if (ch === `"`) {
|
|
109
|
+
inDouble = true;
|
|
110
|
+
out += ch;
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
113
|
+
if (ch === "-" && next === "-") {
|
|
114
|
+
inLineComment = true;
|
|
115
|
+
out += ch;
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
if (ch === "/" && next === "*") {
|
|
119
|
+
inBlockComment = true;
|
|
120
|
+
out += ch;
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
if (ch === "?") {
|
|
124
|
+
out += `$${index++}`;
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
out += ch;
|
|
128
|
+
}
|
|
129
|
+
return out;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Map a single SQLite column type keyword (`INTEGER`, `REAL`, `BLOB`, `TEXT`) to
|
|
134
|
+
* the dialect equivalent. Used by the Zod→DDL generator, which only ever emits
|
|
135
|
+
* `INTEGER` or `TEXT`.
|
|
136
|
+
*
|
|
137
|
+
* @param {Dialect} dialect
|
|
138
|
+
* @param {string} sqliteType
|
|
139
|
+
* @returns {string}
|
|
140
|
+
*/
|
|
141
|
+
export function columnType(dialect, sqliteType) {
|
|
142
|
+
if (dialect !== POSTGRES) {
|
|
143
|
+
return sqliteType;
|
|
144
|
+
}
|
|
145
|
+
switch (sqliteType.toUpperCase()) {
|
|
146
|
+
case "INTEGER":
|
|
147
|
+
return "BIGINT";
|
|
148
|
+
case "REAL":
|
|
149
|
+
return "DOUBLE PRECISION";
|
|
150
|
+
case "BLOB":
|
|
151
|
+
return "BYTEA";
|
|
152
|
+
default:
|
|
153
|
+
return sqliteType;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Translate a complete `CREATE TABLE`/`CREATE INDEX` statement written in SQLite
|
|
159
|
+
* DDL to the target dialect. Only the controlled, internal Smithers DDL passes
|
|
160
|
+
* through here (no user-supplied text), so keyword-level regex substitution is
|
|
161
|
+
* safe. Order matters: the autoincrement primary key is rewritten before the
|
|
162
|
+
* blanket `INTEGER → BIGINT` so it is not double-translated.
|
|
163
|
+
*
|
|
164
|
+
* @param {Dialect} dialect
|
|
165
|
+
* @param {string} ddl
|
|
166
|
+
* @returns {string}
|
|
167
|
+
*/
|
|
168
|
+
export function translateDdl(dialect, ddl) {
|
|
169
|
+
if (dialect !== POSTGRES) {
|
|
170
|
+
return ddl;
|
|
171
|
+
}
|
|
172
|
+
return ddl
|
|
173
|
+
.replace(/\bINTEGER\s+PRIMARY\s+KEY\s+AUTOINCREMENT\b/gi, "BIGSERIAL PRIMARY KEY")
|
|
174
|
+
.replace(/\s+AUTOINCREMENT\b/gi, "")
|
|
175
|
+
.replace(/\bBLOB\b/gi, "BYTEA")
|
|
176
|
+
.replace(/\bREAL\b/gi, "DOUBLE PRECISION")
|
|
177
|
+
.replace(/\bINTEGER\b/gi, "BIGINT");
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* SQL that lists the column names of a table, normalized to rows shaped
|
|
182
|
+
* `{ name: string }`.
|
|
183
|
+
*
|
|
184
|
+
* @param {Dialect} dialect
|
|
185
|
+
* @param {string} table
|
|
186
|
+
* @returns {{ sql: string; params: ReadonlyArray<string> }}
|
|
187
|
+
*/
|
|
188
|
+
export function tableColumnsSql(dialect, table) {
|
|
189
|
+
if (dialect === POSTGRES) {
|
|
190
|
+
return {
|
|
191
|
+
sql: "SELECT column_name AS name FROM information_schema.columns WHERE table_name = ? AND table_schema = current_schema()",
|
|
192
|
+
params: [table],
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
return {
|
|
196
|
+
sql: `PRAGMA table_info(${quoteIdentifier(table)})`,
|
|
197
|
+
params: [],
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* The statement that begins a write transaction. SQLite takes the write lock
|
|
203
|
+
* eagerly with `BEGIN IMMEDIATE`; PostgreSQL uses a plain `BEGIN`.
|
|
204
|
+
*
|
|
205
|
+
* @param {Dialect} dialect
|
|
206
|
+
* @returns {string}
|
|
207
|
+
*/
|
|
208
|
+
export function beginTransactionSql(dialect) {
|
|
209
|
+
return dialect === POSTGRES ? "BEGIN" : "BEGIN IMMEDIATE";
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* A scalar SQL expression extracting a top-level string field from a JSON column
|
|
214
|
+
* stored as TEXT. SQLite uses `json_extract(col, '$.key')`; PostgreSQL casts to
|
|
215
|
+
* `json` and uses the `->>` operator. Only simple top-level paths (`$.key`) are
|
|
216
|
+
* used in Smithers' queries.
|
|
217
|
+
*
|
|
218
|
+
* @param {Dialect} dialect
|
|
219
|
+
* @param {string} columnSql Already-quoted/qualified column expression.
|
|
220
|
+
* @param {string} jsonPath SQLite-style path, e.g. `$.nodeId`.
|
|
221
|
+
* @returns {string}
|
|
222
|
+
*/
|
|
223
|
+
export function jsonExtractText(dialect, columnSql, jsonPath) {
|
|
224
|
+
if (dialect === POSTGRES) {
|
|
225
|
+
const key = jsonPath.replace(/^\$\./, "").replace(/'/g, "''");
|
|
226
|
+
return `(${columnSql}::json->>'${key}')`;
|
|
227
|
+
}
|
|
228
|
+
return `json_extract(${columnSql}, '${jsonPath.replace(/'/g, "''")}')`;
|
|
229
|
+
}
|
package/src/ensure.js
CHANGED
|
@@ -14,5 +14,12 @@ export function ensureSmithersTablesEffect(db) {
|
|
|
14
14
|
* @param {_BunSQLiteDatabase<Record<string, unknown>>} db
|
|
15
15
|
*/
|
|
16
16
|
export function ensureSmithersTables(db) {
|
|
17
|
+
// Postgres schema initialization is asynchronous and cannot be driven by
|
|
18
|
+
// runSync. The Postgres/PGlite entry points ensure the schema (awaited)
|
|
19
|
+
// before the engine starts, so this synchronous SQLite helper is a no-op
|
|
20
|
+
// for a Postgres connection descriptor.
|
|
21
|
+
if (db && typeof db === "object" && /** @type {any} */ (db).dialect === "postgres") {
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
17
24
|
Effect.runSync(ensureSmithersTablesEffect(db));
|
|
18
25
|
}
|
|
@@ -20,3 +20,5 @@ export { smithersMemoryMessages } from "./smithersMemoryMessages.js";
|
|
|
20
20
|
export { smithersVectors } from "./smithersVectors.js";
|
|
21
21
|
export { smithersCron } from "./smithersCron.js";
|
|
22
22
|
export { smithersSchemaMigrations } from "./smithersSchemaMigrations.js";
|
|
23
|
+
export { smithersWorkspaceStates } from "./smithersWorkspaceStates.js";
|
|
24
|
+
export { smithersWorkspaceCheckpoints } from "./smithersWorkspaceCheckpoints.js";
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { integer, primaryKey, sqliteTable, text } from "drizzle-orm/sqlite-core";
|
|
2
|
+
export const smithersWorkspaceCheckpoints = sqliteTable("_smithers_workspace_checkpoints", {
|
|
3
|
+
runId: text("run_id").notNull(),
|
|
4
|
+
nodeId: text("node_id").notNull(),
|
|
5
|
+
iteration: integer("iteration").notNull().default(0),
|
|
6
|
+
attempt: integer("attempt").notNull(),
|
|
7
|
+
seq: integer("seq").notNull(),
|
|
8
|
+
jjCwd: text("jj_cwd").notNull(),
|
|
9
|
+
jjCommitId: text("jj_commit_id").notNull(),
|
|
10
|
+
source: text("source").notNull(),
|
|
11
|
+
tier: integer("tier").notNull(),
|
|
12
|
+
label: text("label"),
|
|
13
|
+
toolUseId: text("tool_use_id"),
|
|
14
|
+
createdAtMs: integer("created_at_ms").notNull(),
|
|
15
|
+
}, (t) => ({
|
|
16
|
+
pk: primaryKey({ columns: [t.runId, t.nodeId, t.iteration, t.attempt, t.seq] }),
|
|
17
|
+
}));
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { integer, primaryKey, sqliteTable, text } from "drizzle-orm/sqlite-core";
|
|
2
|
+
export const smithersWorkspaceStates = sqliteTable("_smithers_workspace_states", {
|
|
3
|
+
runId: text("run_id").notNull(),
|
|
4
|
+
jjCwd: text("jj_cwd").notNull(),
|
|
5
|
+
jjCommitId: text("jj_commit_id").notNull(),
|
|
6
|
+
jjOperationId: text("jj_operation_id").notNull(),
|
|
7
|
+
jjChangeId: text("jj_change_id"),
|
|
8
|
+
createdAtMs: integer("created_at_ms").notNull(),
|
|
9
|
+
}, (t) => ({
|
|
10
|
+
pk: primaryKey({ columns: [t.runId, t.jjCwd, t.jjCommitId] }),
|
|
11
|
+
}));
|
package/src/internal-schema.js
CHANGED
|
@@ -268,3 +268,5 @@ export const smithersSchemaMigrations = sqliteTable("_smithers_schema_migrations
|
|
|
268
268
|
.default(false),
|
|
269
269
|
detailsJson: text("details_json"),
|
|
270
270
|
});
|
|
271
|
+
export { smithersWorkspaceStates } from "./internal-schema/smithersWorkspaceStates.js";
|
|
272
|
+
export { smithersWorkspaceCheckpoints } from "./internal-schema/smithersWorkspaceCheckpoints.js";
|
package/src/loadInputEffect.js
CHANGED
|
@@ -13,16 +13,18 @@ import { SmithersError } from "@smithers-orchestrator/errors/SmithersError";
|
|
|
13
13
|
* @returns {Effect.Effect<Record<string, unknown> | undefined, SmithersError>}
|
|
14
14
|
*/
|
|
15
15
|
export function loadInput(db, inputTable, runId) {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
16
|
+
return Effect.suspend(() => {
|
|
17
|
+
const cols = getTableColumns(inputTable);
|
|
18
|
+
const runIdCol = cols.runId;
|
|
19
|
+
if (!runIdCol) {
|
|
20
|
+
return Effect.fail(new SmithersError("DB_MISSING_COLUMNS", "schema.input must include runId column"));
|
|
21
|
+
}
|
|
22
|
+
return Effect.tryPromise({
|
|
23
|
+
try: () => db.select().from(inputTable).where(eq(runIdCol, runId)).limit(1),
|
|
24
|
+
catch: (cause) => toSmithersError(cause, "load input", {
|
|
25
|
+
code: "DB_QUERY_FAILED",
|
|
26
|
+
details: { runId },
|
|
27
|
+
}),
|
|
28
|
+
}).pipe(Effect.map((rows) => rows[0]));
|
|
29
|
+
}).pipe(Effect.annotateLogs({ runId }), Effect.withLogSpan("db:load-input"));
|
|
28
30
|
}
|
package/src/output.js
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { and, eq } from "drizzle-orm";
|
|
1
|
+
import { and, eq, getTableName } from "drizzle-orm";
|
|
2
2
|
import { getTableColumns } from "drizzle-orm/utils";
|
|
3
3
|
import { createInsertSchema, createSelectSchema } from "drizzle-zod";
|
|
4
4
|
import { Effect } from "effect";
|
|
5
5
|
import { z } from "zod";
|
|
6
6
|
import { SmithersError } from "@smithers-orchestrator/errors/SmithersError";
|
|
7
7
|
import { toSmithersError } from "@smithers-orchestrator/errors/toSmithersError";
|
|
8
|
+
import { getJsonColumnKeys, isPostgresDb, pgRowToDrizzle } from "./snapshot.js";
|
|
8
9
|
import { withSqliteWriteRetryEffect } from "./write-retry.js";
|
|
9
10
|
/** @typedef {import("drizzle-orm").AnyColumn} AnyColumn */
|
|
10
11
|
/** @typedef {import("./output/OutputKey.ts").OutputKey} _OutputKey */
|
|
@@ -76,9 +77,25 @@ export function buildKeyWhere(table, key) {
|
|
|
76
77
|
* @returns {Effect.Effect<T | undefined, SmithersError>}
|
|
77
78
|
*/
|
|
78
79
|
export function selectOutputRowEffect(db, table, key) {
|
|
79
|
-
const
|
|
80
|
+
const cols = getKeyColumns(table);
|
|
81
|
+
const hasIteration = Boolean(cols.iteration);
|
|
82
|
+
const jsonKeys = getJsonColumnKeys(table);
|
|
80
83
|
return Effect.tryPromise({
|
|
81
|
-
try: () =>
|
|
84
|
+
try: () => {
|
|
85
|
+
if (isPostgresDb(db)) {
|
|
86
|
+
const tableName = getTableName(table).replaceAll(`"`, `""`);
|
|
87
|
+
const clauses = ["run_id = $1", "node_id = $2"];
|
|
88
|
+
const values = [key.runId, key.nodeId];
|
|
89
|
+
if (hasIteration) {
|
|
90
|
+
clauses.push("iteration = $3");
|
|
91
|
+
values.push(key.iteration ?? 0);
|
|
92
|
+
}
|
|
93
|
+
return db.connection
|
|
94
|
+
.query({ text: `SELECT * FROM "${tableName}" WHERE ${clauses.join(" AND ")} LIMIT 1`, values })
|
|
95
|
+
.then((result) => (result.rows[0] ? [pgRowToDrizzle(result.rows[0], jsonKeys)] : []));
|
|
96
|
+
}
|
|
97
|
+
return db.select().from(table).where(buildKeyWhere(table, key)).limit(1);
|
|
98
|
+
},
|
|
82
99
|
catch: (cause) => toSmithersError(cause, `select output ${table["_"]?.name ?? "output"}`, {
|
|
83
100
|
code: "DB_QUERY_FAILED",
|
|
84
101
|
details: { outputTable: table["_"]?.name ?? "output" },
|
package/src/schema-migrations.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { eq } from "drizzle-orm";
|
|
2
2
|
import { drizzle } from "drizzle-orm/bun-sqlite";
|
|
3
|
+
import { POSTGRES, translateDdl } from "./dialect.js";
|
|
3
4
|
import { smithersSchemaMigrations } from "./internal-schema/smithersSchemaMigrations.js";
|
|
4
5
|
|
|
5
6
|
const MIGRATION_TABLE_SQL = `CREATE TABLE IF NOT EXISTS _smithers_schema_migrations (
|
|
@@ -452,6 +453,26 @@ function buildMigrations(context) {
|
|
|
452
453
|
return { statementCount: context.createIndexStatements.length + EXTRA_INDEX_STATEMENTS.length };
|
|
453
454
|
},
|
|
454
455
|
},
|
|
456
|
+
{
|
|
457
|
+
id: "0015_add_workspace_states",
|
|
458
|
+
name: "Add workspace snapshot states table",
|
|
459
|
+
checksum: "packages/db/migrations/0015_add_workspace_states.sql",
|
|
460
|
+
isApplied: (sqlite) => tableExists(sqlite, "_smithers_workspace_states"),
|
|
461
|
+
up: (sqlite) => {
|
|
462
|
+
sqlite.run(createTableStatementFor("_smithers_workspace_states", context.createTableStatements));
|
|
463
|
+
return { table: "_smithers_workspace_states" };
|
|
464
|
+
},
|
|
465
|
+
},
|
|
466
|
+
{
|
|
467
|
+
id: "0016_add_workspace_checkpoints",
|
|
468
|
+
name: "Add workspace snapshot checkpoints table",
|
|
469
|
+
checksum: "packages/db/migrations/0016_add_workspace_checkpoints.sql",
|
|
470
|
+
isApplied: (sqlite) => tableExists(sqlite, "_smithers_workspace_checkpoints"),
|
|
471
|
+
up: (sqlite) => {
|
|
472
|
+
sqlite.run(createTableStatementFor("_smithers_workspace_checkpoints", context.createTableStatements));
|
|
473
|
+
return { table: "_smithers_workspace_checkpoints" };
|
|
474
|
+
},
|
|
475
|
+
},
|
|
455
476
|
];
|
|
456
477
|
}
|
|
457
478
|
|
|
@@ -479,3 +500,30 @@ export function runSmithersSchemaMigrations(sqlite, context) {
|
|
|
479
500
|
applied.add(migration.id);
|
|
480
501
|
}
|
|
481
502
|
}
|
|
503
|
+
|
|
504
|
+
/**
|
|
505
|
+
* PostgreSQL schema initialization. A fresh PostgreSQL database starts at the
|
|
506
|
+
* current schema, so it skips the SQLite-only legacy column/foreign-key
|
|
507
|
+
* migrations entirely and simply creates every current table and index
|
|
508
|
+
* idempotently, with DDL translated to PostgreSQL types. Tables are created in
|
|
509
|
+
* declaration order, which keeps foreign-key references valid (referenced
|
|
510
|
+
* tables precede their dependents).
|
|
511
|
+
*
|
|
512
|
+
* @param {{ query: (config: { text: string }) => Promise<unknown> }} pgConn
|
|
513
|
+
* @param {{
|
|
514
|
+
* createTableStatements: readonly string[];
|
|
515
|
+
* createIndexStatements: readonly string[];
|
|
516
|
+
* }} context
|
|
517
|
+
* @returns {Promise<void>}
|
|
518
|
+
*/
|
|
519
|
+
export async function runSmithersSchemaInitPostgres(pgConn, context) {
|
|
520
|
+
const statements = [
|
|
521
|
+
MIGRATION_TABLE_SQL,
|
|
522
|
+
...context.createTableStatements,
|
|
523
|
+
...context.createIndexStatements,
|
|
524
|
+
...EXTRA_INDEX_STATEMENTS,
|
|
525
|
+
];
|
|
526
|
+
for (const statement of statements) {
|
|
527
|
+
await pgConn.query({ text: translateDdl(POSTGRES, statement) });
|
|
528
|
+
}
|
|
529
|
+
}
|
package/src/snapshot.js
CHANGED
|
@@ -27,6 +27,29 @@ function getBooleanColumnKeys(table) {
|
|
|
27
27
|
return [];
|
|
28
28
|
}
|
|
29
29
|
}
|
|
30
|
+
/**
|
|
31
|
+
* Keys of every json-mode column on the table. zodToTable maps any
|
|
32
|
+
* array/object/union/complex Zod field to a Drizzle `text(col,{mode:'json'})`
|
|
33
|
+
* column, which Drizzle's bun:sqlite reader auto-decodes on read. The Postgres
|
|
34
|
+
* path stores those as TEXT, so we must JSON.parse them to match.
|
|
35
|
+
* @param {_Table} table
|
|
36
|
+
* @returns {string[]}
|
|
37
|
+
*/
|
|
38
|
+
export function getJsonColumnKeys(table) {
|
|
39
|
+
try {
|
|
40
|
+
const cols = getTableColumns(table);
|
|
41
|
+
const keys = [];
|
|
42
|
+
for (const [key, col] of Object.entries(cols)) {
|
|
43
|
+
const c = /** @type {Record<string, unknown> & { config?: { mode?: string } }} */ (/** @type {unknown} */ (col));
|
|
44
|
+
if (c?.columnType === "SQLiteTextJson" || c?.config?.mode === "json" || c?.mode === "json" || c?.dataType === "json") {
|
|
45
|
+
keys.push(key);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
return keys;
|
|
49
|
+
} catch {
|
|
50
|
+
return [];
|
|
51
|
+
}
|
|
52
|
+
}
|
|
30
53
|
/**
|
|
31
54
|
* @param {ReadonlyArray<Record<string, unknown>>} rows
|
|
32
55
|
* @param {readonly string[]} boolKeys
|
|
@@ -52,19 +75,73 @@ function coerceBooleanColumns(rows, boolKeys) {
|
|
|
52
75
|
* @param {string} runId
|
|
53
76
|
* @returns {Effect.Effect<Record<string, unknown> | undefined, SmithersError>}
|
|
54
77
|
*/
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
78
|
+
/**
|
|
79
|
+
* @param {unknown} db
|
|
80
|
+
* @returns {boolean}
|
|
81
|
+
*/
|
|
82
|
+
export function isPostgresDb(db) {
|
|
83
|
+
return Boolean(db && typeof db === "object" && /** @type {any} */ (db).dialect === "postgres" && /** @type {any} */ (db).connection);
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Map a raw node-postgres row (snake_case columns, JSON stored as TEXT) into the
|
|
87
|
+
* shape Drizzle's bun:sqlite reader returns (camelCase keys, json-mode columns
|
|
88
|
+
* decoded), so input/output consumers stay dialect-agnostic. `jsonKeys` is the
|
|
89
|
+
* set of camelCase json-mode column keys (from getJsonColumnKeys); every TEXT
|
|
90
|
+
* value for those columns is JSON.parsed to match Drizzle's mode:'json' read.
|
|
91
|
+
* The literal `payload` column is always decoded so callers that omit `jsonKeys`
|
|
92
|
+
* (single-value outputs) keep working.
|
|
93
|
+
* @param {Record<string, unknown>} row
|
|
94
|
+
* @param {readonly string[]} [jsonKeys]
|
|
95
|
+
* @returns {Record<string, unknown>}
|
|
96
|
+
*/
|
|
97
|
+
export function pgRowToDrizzle(row, jsonKeys) {
|
|
98
|
+
/** @type {Record<string, unknown>} */
|
|
99
|
+
const out = {};
|
|
100
|
+
const jsonSet = new Set(jsonKeys ?? []);
|
|
101
|
+
for (const [columnName, value] of Object.entries(row)) {
|
|
102
|
+
const camel = columnName.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase());
|
|
103
|
+
if ((camel === "payload" || jsonSet.has(camel)) && typeof value === "string") {
|
|
104
|
+
try {
|
|
105
|
+
out[camel] = JSON.parse(value);
|
|
106
|
+
}
|
|
107
|
+
catch {
|
|
108
|
+
out[camel] = value;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
else {
|
|
112
|
+
out[camel] = value;
|
|
113
|
+
}
|
|
60
114
|
}
|
|
61
|
-
return
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
115
|
+
return out;
|
|
116
|
+
}
|
|
117
|
+
export function loadInputEffect(db, inputTable, runId) {
|
|
118
|
+
return Effect.suspend(() => {
|
|
119
|
+
const cols = getTableColumns(inputTable);
|
|
120
|
+
const runIdCol = cols.runId;
|
|
121
|
+
if (!runIdCol) {
|
|
122
|
+
return Effect.fail(new SmithersError("DB_MISSING_COLUMNS", "schema.input must include runId column"));
|
|
123
|
+
}
|
|
124
|
+
if (isPostgresDb(db)) {
|
|
125
|
+
const tableName = getTableName(inputTable).replaceAll(`"`, `""`);
|
|
126
|
+
const jsonKeys = getJsonColumnKeys(inputTable);
|
|
127
|
+
return Effect.tryPromise({
|
|
128
|
+
try: () => db.connection
|
|
129
|
+
.query({ text: `SELECT * FROM "${tableName}" WHERE run_id = $1 LIMIT 1`, values: [runId] })
|
|
130
|
+
.then((result) => (result.rows[0] ? pgRowToDrizzle(result.rows[0], jsonKeys) : undefined)),
|
|
131
|
+
catch: (cause) => toSmithersError(cause, "load input", {
|
|
132
|
+
code: "DB_QUERY_FAILED",
|
|
133
|
+
details: { runId },
|
|
134
|
+
}),
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
return Effect.tryPromise({
|
|
138
|
+
try: () => db.select().from(inputTable).where(eq(runIdCol, runId)).limit(1),
|
|
139
|
+
catch: (cause) => toSmithersError(cause, "load input", {
|
|
140
|
+
code: "DB_QUERY_FAILED",
|
|
141
|
+
details: { runId },
|
|
142
|
+
}),
|
|
143
|
+
}).pipe(Effect.map((rows) => rows[0]));
|
|
144
|
+
}).pipe(Effect.annotateLogs({ runId }), Effect.withLogSpan("db:load-input"));
|
|
68
145
|
}
|
|
69
146
|
/**
|
|
70
147
|
* @param {BunSQLiteDatabase<Record<string, unknown>>} db
|
|
@@ -102,8 +179,13 @@ export function loadOutputsEffect(db, schema, runId) {
|
|
|
102
179
|
}).pipe(Effect.option);
|
|
103
180
|
if (Option.isNone(tableNameOpt)) continue;
|
|
104
181
|
const tableName = tableNameOpt.value;
|
|
182
|
+
const jsonKeys = getJsonColumnKeys(/** @type {_Table} */ (table));
|
|
105
183
|
const rawRows = yield* Effect.tryPromise({
|
|
106
|
-
try: () => db
|
|
184
|
+
try: () => isPostgresDb(db)
|
|
185
|
+
? db.connection
|
|
186
|
+
.query({ text: `SELECT * FROM "${tableName.replaceAll(`"`, `""`)}" WHERE run_id = $1`, values: [runId] })
|
|
187
|
+
.then((result) => result.rows.map((r) => pgRowToDrizzle(r, jsonKeys)))
|
|
188
|
+
: db.select().from(/** @type {_Table} */ (table)).where(eq(runIdCol, runId)),
|
|
107
189
|
catch: (cause) => toSmithersError(cause, `load outputs ${tableName}`, { code: "DB_QUERY_FAILED", details: { runId, tableName } }),
|
|
108
190
|
});
|
|
109
191
|
const boolKeys = getBooleanColumnKeys(/** @type {_Table} */ (table));
|
|
@@ -123,3 +205,43 @@ export function loadOutputsEffect(db, schema, runId) {
|
|
|
123
205
|
export function loadOutputs(db, schema, runId) {
|
|
124
206
|
return Effect.runPromise(loadOutputsEffect(db, schema, runId));
|
|
125
207
|
}
|
|
208
|
+
/**
|
|
209
|
+
* Read every row of a single output table for a run, returning Drizzle-shaped
|
|
210
|
+
* rows (camelCase keys, boolean columns coerced to JS booleans). Dialect-aware:
|
|
211
|
+
* Drizzle for bun:sqlite, a raw `$n` query for the Postgres descriptor.
|
|
212
|
+
* @param {unknown} db
|
|
213
|
+
* @param {_Table} table
|
|
214
|
+
* @param {string} [runId]
|
|
215
|
+
* @returns {Effect.Effect<Array<Record<string, unknown>>, SmithersError>}
|
|
216
|
+
*/
|
|
217
|
+
export function loadRunOutputRowsEffect(db, table, runId) {
|
|
218
|
+
return Effect.gen(function* () {
|
|
219
|
+
const cols = getTableColumns(table);
|
|
220
|
+
const runIdCol = cols.runId;
|
|
221
|
+
const tableName = getTableName(table);
|
|
222
|
+
const boolKeys = getBooleanColumnKeys(table);
|
|
223
|
+
const jsonKeys = getJsonColumnKeys(table);
|
|
224
|
+
const rawRows = yield* Effect.tryPromise({
|
|
225
|
+
try: () => {
|
|
226
|
+
if (isPostgresDb(db)) {
|
|
227
|
+
const escaped = tableName.replaceAll(`"`, `""`);
|
|
228
|
+
const text = runId && runIdCol
|
|
229
|
+
? `SELECT * FROM "${escaped}" WHERE run_id = $1`
|
|
230
|
+
: `SELECT * FROM "${escaped}"`;
|
|
231
|
+
const values = runId && runIdCol ? [runId] : [];
|
|
232
|
+
return db.connection
|
|
233
|
+
.query({ text, values })
|
|
234
|
+
.then((result) => result.rows.map((r) => pgRowToDrizzle(r, jsonKeys)));
|
|
235
|
+
}
|
|
236
|
+
return runId && runIdCol
|
|
237
|
+
? db.select().from(table).where(eq(runIdCol, runId))
|
|
238
|
+
: db.select().from(table);
|
|
239
|
+
},
|
|
240
|
+
catch: (cause) => toSmithersError(cause, `load run output ${tableName}`, {
|
|
241
|
+
code: "DB_QUERY_FAILED",
|
|
242
|
+
details: { runId, tableName },
|
|
243
|
+
}),
|
|
244
|
+
});
|
|
245
|
+
return coerceBooleanColumns(rawRows, boolKeys);
|
|
246
|
+
}).pipe(Effect.annotateLogs({ runId: runId ?? "" }), Effect.withLogSpan("db:load-run-output"));
|
|
247
|
+
}
|
|
@@ -3,8 +3,17 @@ import * as SqlClient from "@effect/sql/SqlClient";
|
|
|
3
3
|
import { SqlError } from "@effect/sql/SqlError";
|
|
4
4
|
import * as Statement from "@effect/sql/Statement";
|
|
5
5
|
import { Database } from "bun:sqlite";
|
|
6
|
-
import { Context, Effect, Layer, ManagedRuntime, Scope } from "effect";
|
|
7
|
-
import {
|
|
6
|
+
import { Context, Effect, Layer, ManagedRuntime, Scope, Stream } from "effect";
|
|
7
|
+
import {
|
|
8
|
+
POSTGRES,
|
|
9
|
+
SQLITE,
|
|
10
|
+
jsonExtractText,
|
|
11
|
+
translatePlaceholders,
|
|
12
|
+
} from "./dialect.js";
|
|
13
|
+
import {
|
|
14
|
+
runSmithersSchemaMigrations,
|
|
15
|
+
runSmithersSchemaInitPostgres,
|
|
16
|
+
} from "./schema-migrations.js";
|
|
8
17
|
import { camelToSnake } from "./utils/camelToSnake.js";
|
|
9
18
|
/** @typedef {import("drizzle-orm/bun-sqlite").BunSQLiteDatabase} BunSQLiteDatabase */
|
|
10
19
|
/** @typedef {import("./SqlMessageStorageEventHistoryQuery.ts").SqlMessageStorageEventHistoryQuery} SqlMessageStorageEventHistoryQuery */
|
|
@@ -205,6 +214,30 @@ const CREATE_TABLE_STATEMENTS = [
|
|
|
205
214
|
status TEXT NOT NULL,
|
|
206
215
|
error_json TEXT,
|
|
207
216
|
PRIMARY KEY (run_id, node_id, iteration, attempt, seq)
|
|
217
|
+
)`,
|
|
218
|
+
`CREATE TABLE IF NOT EXISTS _smithers_workspace_states (
|
|
219
|
+
run_id TEXT NOT NULL,
|
|
220
|
+
jj_cwd TEXT NOT NULL,
|
|
221
|
+
jj_commit_id TEXT NOT NULL,
|
|
222
|
+
jj_operation_id TEXT NOT NULL,
|
|
223
|
+
jj_change_id TEXT,
|
|
224
|
+
created_at_ms INTEGER NOT NULL,
|
|
225
|
+
PRIMARY KEY (run_id, jj_cwd, jj_commit_id)
|
|
226
|
+
)`,
|
|
227
|
+
`CREATE TABLE IF NOT EXISTS _smithers_workspace_checkpoints (
|
|
228
|
+
run_id TEXT NOT NULL,
|
|
229
|
+
node_id TEXT NOT NULL,
|
|
230
|
+
iteration INTEGER NOT NULL DEFAULT 0,
|
|
231
|
+
attempt INTEGER NOT NULL,
|
|
232
|
+
seq INTEGER NOT NULL,
|
|
233
|
+
jj_cwd TEXT NOT NULL,
|
|
234
|
+
jj_commit_id TEXT NOT NULL,
|
|
235
|
+
source TEXT NOT NULL,
|
|
236
|
+
tier INTEGER NOT NULL,
|
|
237
|
+
label TEXT,
|
|
238
|
+
tool_use_id TEXT,
|
|
239
|
+
created_at_ms INTEGER NOT NULL,
|
|
240
|
+
PRIMARY KEY (run_id, node_id, iteration, attempt, seq)
|
|
208
241
|
)`,
|
|
209
242
|
`CREATE TABLE IF NOT EXISTS _smithers_events (
|
|
210
243
|
run_id TEXT NOT NULL,
|
|
@@ -393,14 +426,17 @@ function applyBooleanColumns(row, booleanColumns) {
|
|
|
393
426
|
* @param {Record<string, unknown>} row
|
|
394
427
|
* @param {{ orIgnore?: boolean; conflictColumns?: readonly string[]; updateColumns?: readonly string[]; }} [options]
|
|
395
428
|
*/
|
|
396
|
-
function buildInsertSql(table, row, options) {
|
|
429
|
+
function buildInsertSql(table, row, options, dialect = SQLITE) {
|
|
397
430
|
const entries = Object.entries(row).filter(([, value]) => value !== undefined);
|
|
398
431
|
const columns = entries.map(([key]) => camelToSnake(key));
|
|
399
432
|
const params = entries.map(([, value]) => encodeParam(value));
|
|
400
433
|
const tableSql = quoteIdentifier(table);
|
|
401
434
|
const columnSql = columns.map(quoteIdentifier).join(", ");
|
|
402
435
|
const placeholderSql = columns.map(() => "?").join(", ");
|
|
403
|
-
|
|
436
|
+
// SQLite spells the ignore-on-conflict shorthand `INSERT OR IGNORE`;
|
|
437
|
+
// PostgreSQL has no such prefix and instead appends `ON CONFLICT DO NOTHING`.
|
|
438
|
+
const orIgnorePrefix = options?.orIgnore && dialect !== POSTGRES ? " OR IGNORE" : "";
|
|
439
|
+
let statement = `INSERT${orIgnorePrefix} INTO ${tableSql} (${columnSql}) ` +
|
|
404
440
|
`VALUES (${placeholderSql})`;
|
|
405
441
|
if (options?.conflictColumns && options.conflictColumns.length > 0) {
|
|
406
442
|
const conflictSql = options.conflictColumns.map(camelToSnake).map(quoteIdentifier).join(", ");
|
|
@@ -417,6 +453,9 @@ function buildInsertSql(table, row, options) {
|
|
|
417
453
|
statement += ` ON CONFLICT (${conflictSql}) DO UPDATE SET ${updateSql}`;
|
|
418
454
|
}
|
|
419
455
|
}
|
|
456
|
+
else if (options?.orIgnore && dialect === POSTGRES) {
|
|
457
|
+
statement += ` ON CONFLICT DO NOTHING`;
|
|
458
|
+
}
|
|
420
459
|
return { statement, params };
|
|
421
460
|
}
|
|
422
461
|
/**
|
|
@@ -494,7 +533,7 @@ function createConnection(sqlite) {
|
|
|
494
533
|
}
|
|
495
534
|
}),
|
|
496
535
|
executeUnprepared: (statement, params, transformRows) => execute(statement, params, transformRows),
|
|
497
|
-
executeStream: () =>
|
|
536
|
+
executeStream: (statement, params, transformRows) => Stream.fromIterableEffect(execute(statement, params, transformRows)),
|
|
498
537
|
};
|
|
499
538
|
}
|
|
500
539
|
/**
|
|
@@ -524,18 +563,118 @@ function makeSqlClientEffect(sqlite) {
|
|
|
524
563
|
function makeSqlClientLayer(sqlite) {
|
|
525
564
|
return Layer.scoped(SqlClient.SqlClient, makeSqlClientEffect(sqlite));
|
|
526
565
|
}
|
|
566
|
+
/**
|
|
567
|
+
* @param {SqliteParam} value
|
|
568
|
+
* @returns {unknown}
|
|
569
|
+
*/
|
|
570
|
+
function toPostgresParam(value) {
|
|
571
|
+
// node-postgres maps Buffer → bytea; a bare Uint8Array does not round-trip.
|
|
572
|
+
if (value instanceof Uint8Array || Buffer.isBuffer(value)) {
|
|
573
|
+
return Buffer.isBuffer(value) ? value : Buffer.from(value);
|
|
574
|
+
}
|
|
575
|
+
// Smithers mirrors SQLite's storage model on Postgres: JSON lives in TEXT
|
|
576
|
+
// columns. SQLite/Drizzle stringify objects automatically; node-postgres does
|
|
577
|
+
// not, so do it here to keep a single encoding contract across dialects.
|
|
578
|
+
if (value !== null && typeof value === "object" && !(value instanceof Date)) {
|
|
579
|
+
return JSON.stringify(value);
|
|
580
|
+
}
|
|
581
|
+
return value;
|
|
582
|
+
}
|
|
583
|
+
/**
|
|
584
|
+
* A `@effect/sql` connection backed by a single node-postgres connection (any
|
|
585
|
+
* object exposing `query({ text, values, rowMode })` — a `pg.Client`, or a
|
|
586
|
+
* PGlite socket connection). Smithers writes SQL with `?` placeholders; this
|
|
587
|
+
* connection rewrites them to PostgreSQL's `$n` on the way out, mirroring the
|
|
588
|
+
* SQLite connection so the rest of the adapter is dialect-agnostic.
|
|
589
|
+
* @param {{ query: (config: { text: string; values?: ReadonlyArray<unknown>; rowMode?: "array" }) => Promise<{ rows?: ReadonlyArray<any> }> }} pgConn
|
|
590
|
+
* @returns {Connection}
|
|
591
|
+
*/
|
|
592
|
+
function createPostgresConnection(pgConn) {
|
|
593
|
+
const run = (statement, params, transformRows) => Effect.tryPromise({
|
|
594
|
+
try: async () => {
|
|
595
|
+
const text = translatePlaceholders(POSTGRES, statement);
|
|
596
|
+
const result = await pgConn.query({ text, values: params.map(toPostgresParam) });
|
|
597
|
+
const rows = result.rows ?? [];
|
|
598
|
+
return transformRows ? transformRows(rows) : rows;
|
|
599
|
+
},
|
|
600
|
+
catch: (cause) => new SqlError({ cause, message: "Failed to execute Postgres statement" }),
|
|
601
|
+
});
|
|
602
|
+
return {
|
|
603
|
+
execute: (statement, params, transformRows) => run(statement, params, transformRows),
|
|
604
|
+
executeRaw: (statement, params) => run(statement, params, undefined),
|
|
605
|
+
executeValues: (statement, params) => Effect.tryPromise({
|
|
606
|
+
try: async () => {
|
|
607
|
+
const text = translatePlaceholders(POSTGRES, statement);
|
|
608
|
+
const result = await pgConn.query({
|
|
609
|
+
text,
|
|
610
|
+
values: params.map(toPostgresParam),
|
|
611
|
+
rowMode: "array",
|
|
612
|
+
});
|
|
613
|
+
return result.rows ?? [];
|
|
614
|
+
},
|
|
615
|
+
catch: (cause) => new SqlError({ cause, message: "Failed to execute Postgres values statement" }),
|
|
616
|
+
}),
|
|
617
|
+
executeUnprepared: (statement, params, transformRows) => run(statement, params, transformRows),
|
|
618
|
+
executeStream: (statement, params, transformRows) => Stream.fromIterableEffect(run(statement, params, transformRows)),
|
|
619
|
+
};
|
|
620
|
+
}
|
|
621
|
+
/**
|
|
622
|
+
* @param {object} pgConn
|
|
623
|
+
* @returns {Effect.Effect<SqlClient.SqlClient, never>}
|
|
624
|
+
*/
|
|
625
|
+
function makePostgresSqlClientEffect(pgConn) {
|
|
626
|
+
// The compiler is only exercised by the `sql``` tagged template, which this
|
|
627
|
+
// storage never uses — every query is a pre-built string run through the raw
|
|
628
|
+
// connection, where the `?`→`$n` rewrite happens. So the SQLite compiler is
|
|
629
|
+
// an inert placeholder here.
|
|
630
|
+
const compiler = Statement.makeCompilerSqlite(camelToSnake);
|
|
631
|
+
const connection = createPostgresConnection(pgConn);
|
|
632
|
+
return Effect.gen(function* () {
|
|
633
|
+
const semaphore = yield* Effect.makeSemaphore(1);
|
|
634
|
+
const acquirer = semaphore.withPermits(1)(Effect.succeed(connection));
|
|
635
|
+
const transactionAcquirer = Effect.uninterruptibleMask((restore) => Effect.as(Effect.zipRight(restore(semaphore.take(1)), Effect.tap(Effect.scope, (scope) => Scope.addFinalizer(scope, semaphore.release(1)))), connection));
|
|
636
|
+
const reactivity = yield* Reactivity.make;
|
|
637
|
+
return yield* SqlClient.make({
|
|
638
|
+
acquirer,
|
|
639
|
+
compiler,
|
|
640
|
+
transactionAcquirer,
|
|
641
|
+
spanAttributes: [[ATTR_DB_SYSTEM_NAME, "postgresql"]],
|
|
642
|
+
transformRows: transformRowKeys,
|
|
643
|
+
}).pipe(Effect.provideService(Reactivity.Reactivity, reactivity));
|
|
644
|
+
});
|
|
645
|
+
}
|
|
646
|
+
/**
|
|
647
|
+
* @param {object} pgConn
|
|
648
|
+
*/
|
|
649
|
+
function makePostgresSqlClientLayer(pgConn) {
|
|
650
|
+
return Layer.scoped(SqlClient.SqlClient, makePostgresSqlClientEffect(pgConn));
|
|
651
|
+
}
|
|
527
652
|
export class SqlMessageStorage {
|
|
528
653
|
sqlite;
|
|
654
|
+
/** @type {import("./dialect.js").Dialect} */
|
|
655
|
+
dialect;
|
|
656
|
+
/** @type {object | null} */
|
|
657
|
+
pgConn;
|
|
529
658
|
// TODO(Phase 8): Keep this per-DB runtime until the unified runtime can
|
|
530
659
|
// inject a scoped SqlClient without rebuilding the per-connection semaphore.
|
|
531
660
|
runtime;
|
|
532
661
|
tableColumnsCache = new Map();
|
|
533
662
|
/**
|
|
534
|
-
* @param {BunSQLiteDatabase<any> | Database} db
|
|
663
|
+
* @param {BunSQLiteDatabase<any> | Database | { dialect: "postgres"; connection: object }} db
|
|
535
664
|
*/
|
|
536
665
|
constructor(db) {
|
|
537
|
-
|
|
538
|
-
|
|
666
|
+
if (db && typeof db === "object" && /** @type {any} */ (db).dialect === POSTGRES) {
|
|
667
|
+
this.dialect = POSTGRES;
|
|
668
|
+
this.pgConn = /** @type {any} */ (db).connection;
|
|
669
|
+
this.sqlite = null;
|
|
670
|
+
this.runtime = ManagedRuntime.make(makePostgresSqlClientLayer(this.pgConn));
|
|
671
|
+
}
|
|
672
|
+
else {
|
|
673
|
+
this.dialect = SQLITE;
|
|
674
|
+
this.sqlite = resolveSqliteDatabase(db);
|
|
675
|
+
this.pgConn = null;
|
|
676
|
+
this.runtime = ManagedRuntime.make(makeSqlClientLayer(this.sqlite));
|
|
677
|
+
}
|
|
539
678
|
}
|
|
540
679
|
/**
|
|
541
680
|
* @param {string} table
|
|
@@ -546,6 +685,13 @@ export class SqlMessageStorage {
|
|
|
546
685
|
if (cached) {
|
|
547
686
|
return cached;
|
|
548
687
|
}
|
|
688
|
+
if (this.dialect === POSTGRES) {
|
|
689
|
+
// A fresh PostgreSQL schema has no historical column drift to defend
|
|
690
|
+
// against, and PRAGMA is unavailable. Returning null tells
|
|
691
|
+
// filterKnownColumns to skip filtering, so a genuinely missing column
|
|
692
|
+
// surfaces as a loud insert error rather than a silently dropped field.
|
|
693
|
+
return null;
|
|
694
|
+
}
|
|
549
695
|
const rows = this.sqlite
|
|
550
696
|
.query(`PRAGMA table_info(${quoteIdentifier(table)})`)
|
|
551
697
|
.all();
|
|
@@ -562,7 +708,7 @@ export class SqlMessageStorage {
|
|
|
562
708
|
*/
|
|
563
709
|
filterKnownColumns(table, row) {
|
|
564
710
|
const knownColumns = this.getTableColumns(table);
|
|
565
|
-
return Object.fromEntries(Object.entries(row).filter(([key, value]) => value !== undefined && knownColumns.has(key)));
|
|
711
|
+
return Object.fromEntries(Object.entries(row).filter(([key, value]) => value !== undefined && (knownColumns === null || knownColumns.has(key))));
|
|
566
712
|
}
|
|
567
713
|
/**
|
|
568
714
|
* @template A, E
|
|
@@ -584,6 +730,16 @@ export class SqlMessageStorage {
|
|
|
584
730
|
* @returns {Effect.Effect<void, never>}
|
|
585
731
|
*/
|
|
586
732
|
ensureSchemaEffect() {
|
|
733
|
+
if (this.dialect === POSTGRES) {
|
|
734
|
+
const pgConn = this.pgConn;
|
|
735
|
+
return Effect.tryPromise({
|
|
736
|
+
try: () => runSmithersSchemaInitPostgres(pgConn, {
|
|
737
|
+
createTableStatements: CREATE_TABLE_STATEMENTS,
|
|
738
|
+
createIndexStatements: CREATE_INDEX_STATEMENTS,
|
|
739
|
+
}),
|
|
740
|
+
catch: (cause) => new SqlError({ cause, message: "Failed to initialize Postgres schema" }),
|
|
741
|
+
});
|
|
742
|
+
}
|
|
587
743
|
const sqlite = this.sqlite;
|
|
588
744
|
return Effect.sync(() => {
|
|
589
745
|
runSmithersSchemaMigrations(sqlite, {
|
|
@@ -622,6 +778,28 @@ export class SqlMessageStorage {
|
|
|
622
778
|
return rows[0];
|
|
623
779
|
}
|
|
624
780
|
/**
|
|
781
|
+
* Like {@link queryAll} but returns rows with their on-disk column names (no
|
|
782
|
+
* snake→camel transform). Used for "raw" output-table reads where callers
|
|
783
|
+
* expect the storage column names verbatim.
|
|
784
|
+
* @template T
|
|
785
|
+
* @param {string} statement
|
|
786
|
+
* @param {ReadonlyArray<SqliteParam>} [params]
|
|
787
|
+
* @returns {Promise<Array<T>>}
|
|
788
|
+
*/
|
|
789
|
+
queryAllRaw(statement, params = []) {
|
|
790
|
+
return this.withConnection((connection) => connection.execute(statement, params.map(encodeParam), undefined));
|
|
791
|
+
}
|
|
792
|
+
/**
|
|
793
|
+
* @template T
|
|
794
|
+
* @param {string} statement
|
|
795
|
+
* @param {ReadonlyArray<SqliteParam>} [params]
|
|
796
|
+
* @returns {Promise<T | undefined>}
|
|
797
|
+
*/
|
|
798
|
+
async queryOneRaw(statement, params = []) {
|
|
799
|
+
const rows = await this.queryAllRaw(statement, params);
|
|
800
|
+
return rows[0];
|
|
801
|
+
}
|
|
802
|
+
/**
|
|
625
803
|
* @param {string} statement
|
|
626
804
|
* @param {ReadonlyArray<SqliteParam>} [params]
|
|
627
805
|
* @returns {Promise<void>}
|
|
@@ -636,7 +814,7 @@ export class SqlMessageStorage {
|
|
|
636
814
|
*/
|
|
637
815
|
insertIgnore(table, row) {
|
|
638
816
|
const filteredRow = this.filterKnownColumns(table, row);
|
|
639
|
-
const { statement, params } = buildInsertSql(table, filteredRow, { orIgnore: true });
|
|
817
|
+
const { statement, params } = buildInsertSql(table, filteredRow, { orIgnore: true }, this.dialect);
|
|
640
818
|
return this.execute(statement, params);
|
|
641
819
|
}
|
|
642
820
|
/**
|
|
@@ -651,7 +829,7 @@ export class SqlMessageStorage {
|
|
|
651
829
|
const { statement, params } = buildInsertSql(table, filteredRow, {
|
|
652
830
|
conflictColumns,
|
|
653
831
|
updateColumns,
|
|
654
|
-
});
|
|
832
|
+
}, this.dialect);
|
|
655
833
|
return this.execute(statement, params);
|
|
656
834
|
}
|
|
657
835
|
/**
|
|
@@ -694,7 +872,7 @@ export class SqlMessageStorage {
|
|
|
694
872
|
params.push(...query.types);
|
|
695
873
|
}
|
|
696
874
|
if (query.nodeId) {
|
|
697
|
-
clauses.push("
|
|
875
|
+
clauses.push(`${jsonExtractText(this.dialect, "payload_json", "$.nodeId")} = ?`);
|
|
698
876
|
params.push(query.nodeId);
|
|
699
877
|
}
|
|
700
878
|
return {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { unwrapZodType } from "./unwrapZodType.js";
|
|
2
|
+
import { columnType, SQLITE } from "./dialect.js";
|
|
2
3
|
import { camelToSnake } from "./utils/camelToSnake.js";
|
|
3
4
|
/**
|
|
4
5
|
* Determines the Zod base type name from a (possibly unwrapped) Zod type.
|
|
@@ -34,18 +35,22 @@ export function zodSchemaColumns(schema) {
|
|
|
34
35
|
}
|
|
35
36
|
/**
|
|
36
37
|
* Generates a CREATE TABLE IF NOT EXISTS SQL statement from a Zod schema.
|
|
37
|
-
* Used for runtime table creation without Drizzle migrations.
|
|
38
|
+
* Used for runtime table creation without Drizzle migrations. Pass
|
|
39
|
+
* `opts.dialect` to emit PostgreSQL-compatible column types; the default
|
|
40
|
+
* (`sqlite`) is byte-identical to the historical output.
|
|
38
41
|
*/
|
|
39
42
|
export function zodToCreateTableSQL(tableName, schema, opts) {
|
|
43
|
+
const dialect = opts?.dialect ?? SQLITE;
|
|
44
|
+
const integer = columnType(dialect, "INTEGER");
|
|
40
45
|
const colDefs = opts?.isInput
|
|
41
46
|
? [`run_id TEXT NOT NULL PRIMARY KEY`]
|
|
42
47
|
: [
|
|
43
48
|
`run_id TEXT NOT NULL`,
|
|
44
49
|
`node_id TEXT NOT NULL`,
|
|
45
|
-
`iteration
|
|
50
|
+
`iteration ${integer} NOT NULL DEFAULT 0`,
|
|
46
51
|
];
|
|
47
52
|
for (const { name, sqliteType } of zodSchemaColumns(schema)) {
|
|
48
|
-
colDefs.push(`"${name}" ${sqliteType}`);
|
|
53
|
+
colDefs.push(`"${name}" ${columnType(dialect, sqliteType)}`);
|
|
49
54
|
}
|
|
50
55
|
if (!opts?.isInput) {
|
|
51
56
|
colDefs.push(`PRIMARY KEY (run_id, node_id, iteration)`);
|