@smithers-orchestrator/db 0.16.9 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -6
- package/src/SqlMessageStorage.js +11 -6
- package/src/adapter/SmithersDb.js +107 -26
- package/src/adapter.js +107 -26
- package/src/ensureSqlMessageStorage.js +0 -1
- package/src/ensureSqlMessageStorageEffect.js +0 -2
- package/src/getSqlMessageStorage.js +0 -1
- package/src/index.d.ts +9 -3
- package/src/input.js +0 -1
- package/src/internal-schema/index.js +4 -2
- package/src/internal-schema/smithersMemoryFacts.js +13 -0
- package/src/internal-schema/smithersMemoryMessages.js +11 -0
- package/src/internal-schema/smithersMemoryThreads.js +10 -0
- package/src/internal-schema/smithersScorers.js +20 -0
- package/src/internal-schema.js +4 -2
- package/src/loadOutputsEffect.js +1 -2
- package/src/output/buildOutputRow.js +1 -1
- package/src/output/describeSchemaShape.js +0 -1
- package/src/output/getAgentOutputSchema.js +1 -1
- package/src/output/validateExistingOutput.js +0 -1
- package/src/output/validateOutput.js +0 -1
- package/src/output.js +1 -1
- package/src/snapshot.js +1 -1
- package/src/sql-message-storage.js +11 -6
- package/src/storage/StorageService.js +1 -1
- package/src/zodToCreateTableSQL.js +60 -16
- package/src/zodToTable.js +0 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@smithers-orchestrator/db",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.17.0",
|
|
4
4
|
"description": "SQLite and Drizzle persistence adapter for Smithers workflows",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"sideEffects": false,
|
|
@@ -24,12 +24,14 @@
|
|
|
24
24
|
"@effect/sql": "^0.51.0",
|
|
25
25
|
"drizzle-orm": "^0.45.2",
|
|
26
26
|
"drizzle-zod": "^0.8.3",
|
|
27
|
+
"effect": "^3.21.1",
|
|
27
28
|
"zod": "^4.3.6",
|
|
28
|
-
"@smithers-orchestrator/
|
|
29
|
-
"@smithers-orchestrator/
|
|
30
|
-
"@smithers-orchestrator/
|
|
31
|
-
"@smithers-orchestrator/
|
|
32
|
-
"@smithers-orchestrator/
|
|
29
|
+
"@smithers-orchestrator/errors": "0.17.0",
|
|
30
|
+
"@smithers-orchestrator/graph": "0.17.0",
|
|
31
|
+
"@smithers-orchestrator/memory": "0.17.0",
|
|
32
|
+
"@smithers-orchestrator/scorers": "0.17.0",
|
|
33
|
+
"@smithers-orchestrator/observability": "0.17.0",
|
|
34
|
+
"@smithers-orchestrator/scheduler": "0.17.0"
|
|
33
35
|
},
|
|
34
36
|
"devDependencies": {
|
|
35
37
|
"@types/bun": "latest",
|
package/src/SqlMessageStorage.js
CHANGED
|
@@ -34,8 +34,6 @@ const CREATE_TABLE_STATEMENTS = [
|
|
|
34
34
|
error_json TEXT,
|
|
35
35
|
config_json TEXT
|
|
36
36
|
)`,
|
|
37
|
-
`CREATE INDEX IF NOT EXISTS _smithers_runs_status_heartbeat_idx
|
|
38
|
-
ON _smithers_runs (status, heartbeat_at_ms)`,
|
|
39
37
|
`CREATE TABLE IF NOT EXISTS _smithers_nodes (
|
|
40
38
|
run_id TEXT NOT NULL,
|
|
41
39
|
node_id TEXT NOT NULL,
|
|
@@ -143,8 +141,6 @@ const CREATE_TABLE_STATEMENTS = [
|
|
|
143
141
|
received_by TEXT,
|
|
144
142
|
PRIMARY KEY (run_id, seq)
|
|
145
143
|
)`,
|
|
146
|
-
`CREATE INDEX IF NOT EXISTS _smithers_signals_lookup_idx
|
|
147
|
-
ON _smithers_signals (run_id, signal_name, correlation_id, received_at_ms)`,
|
|
148
144
|
`CREATE TABLE IF NOT EXISTS _smithers_cache (
|
|
149
145
|
cache_key TEXT PRIMARY KEY,
|
|
150
146
|
created_at_ms INTEGER NOT NULL,
|
|
@@ -177,8 +173,6 @@ const CREATE_TABLE_STATEMENTS = [
|
|
|
177
173
|
result TEXT NOT NULL,
|
|
178
174
|
duration_ms INTEGER
|
|
179
175
|
)`,
|
|
180
|
-
`CREATE INDEX IF NOT EXISTS _smithers_time_travel_audit_lookup_idx
|
|
181
|
-
ON _smithers_time_travel_audit (run_id, caller, timestamp_ms)`,
|
|
182
176
|
`CREATE TABLE IF NOT EXISTS _smithers_sandboxes (
|
|
183
177
|
run_id TEXT NOT NULL,
|
|
184
178
|
sandbox_id TEXT NOT NULL,
|
|
@@ -322,6 +316,14 @@ const CREATE_TABLE_STATEMENTS = [
|
|
|
322
316
|
created_at_ms INTEGER NOT NULL
|
|
323
317
|
)`,
|
|
324
318
|
];
|
|
319
|
+
const CREATE_INDEX_STATEMENTS = [
|
|
320
|
+
`CREATE INDEX IF NOT EXISTS _smithers_runs_status_heartbeat_idx
|
|
321
|
+
ON _smithers_runs (status, heartbeat_at_ms)`,
|
|
322
|
+
`CREATE INDEX IF NOT EXISTS _smithers_signals_lookup_idx
|
|
323
|
+
ON _smithers_signals (run_id, signal_name, correlation_id, received_at_ms)`,
|
|
324
|
+
`CREATE INDEX IF NOT EXISTS _smithers_time_travel_audit_lookup_idx
|
|
325
|
+
ON _smithers_time_travel_audit (run_id, caller, timestamp_ms)`,
|
|
326
|
+
];
|
|
325
327
|
const MIGRATION_STATEMENTS = [
|
|
326
328
|
`ALTER TABLE _smithers_attempts ADD COLUMN response_text TEXT`,
|
|
327
329
|
`ALTER TABLE _smithers_attempts ADD COLUMN jj_cwd TEXT`,
|
|
@@ -643,6 +645,9 @@ export class SqlMessageStorage {
|
|
|
643
645
|
// Ignore if another caller added it first.
|
|
644
646
|
}
|
|
645
647
|
}
|
|
648
|
+
for (const statement of CREATE_INDEX_STATEMENTS) {
|
|
649
|
+
sqlite.run(statement);
|
|
650
|
+
}
|
|
646
651
|
});
|
|
647
652
|
}
|
|
648
653
|
/**
|
|
@@ -2,7 +2,7 @@ import { getTableName, sql } from "drizzle-orm";
|
|
|
2
2
|
import { Effect, Exit, FiberId, Metric } from "effect";
|
|
3
3
|
import { toSmithersError } from "@smithers-orchestrator/errors/toSmithersError";
|
|
4
4
|
import { getSqlMessageStorage } from "../sql-message-storage.js";
|
|
5
|
-
import { alertsAcknowledgedTotal, alertsFiredTotal, dbQueryDuration, dbTransactionDuration,
|
|
5
|
+
import { alertsAcknowledgedTotal, alertsFiredTotal, dbQueryDuration, dbTransactionDuration, dbTransactionRollbacks, } from "@smithers-orchestrator/observability/metrics";
|
|
6
6
|
import { assertOptionalStringMaxLength, assertPositiveFiniteNumber, } from "../input-bounds.js";
|
|
7
7
|
import { FRAME_KEYFRAME_INTERVAL, applyFrameDeltaJson, encodeFrameDelta, normalizeFrameEncoding, serializeFrameDelta, } from "../frame-codec.js";
|
|
8
8
|
import { getKeyColumns } from "../output.js";
|
|
@@ -64,7 +64,6 @@ import { alertsActive } from "@smithers-orchestrator/observability/metrics";
|
|
|
64
64
|
|
|
65
65
|
|
|
66
66
|
const FRAME_XML_CACHE_MAX = 512;
|
|
67
|
-
const RUN_HEARTBEAT_STALE_MS = 30_000;
|
|
68
67
|
const RAW_QUERY_ALLOWED_PREFIX = /^(?:select|with|explain|values)\b/i;
|
|
69
68
|
const RAW_QUERY_FORBIDDEN_KEYWORDS = /\b(?:drop|delete|insert|update|alter|create|attach|detach|pragma)\b/i;
|
|
70
69
|
const ACTIVE_ALERT_STATUSES = new Set([
|
|
@@ -337,6 +336,73 @@ function runnableEffect(effect) {
|
|
|
337
336
|
}
|
|
338
337
|
return runnable;
|
|
339
338
|
}
|
|
339
|
+
/**
|
|
340
|
+
* @typedef {{ depth: number; ownerThread: string | null; tail: Promise<unknown> }} SqliteTransactionState
|
|
341
|
+
*/
|
|
342
|
+
/** @type {WeakMap<object, SqliteTransactionState>} */
|
|
343
|
+
// Cross-adapter coordination: one sqlite connection cannot run overlapping BEGIN IMMEDIATE statements.
|
|
344
|
+
const sqliteTransactionStateByClient = (() => {
|
|
345
|
+
const key = Symbol.for("smithers.sqliteTransactionStateByClient");
|
|
346
|
+
const registry = /** @type {Record<PropertyKey, unknown>} */ (globalThis);
|
|
347
|
+
const existing = registry[key];
|
|
348
|
+
if (existing instanceof WeakMap) {
|
|
349
|
+
return /** @type {WeakMap<object, SqliteTransactionState>} */ (existing);
|
|
350
|
+
}
|
|
351
|
+
const stateByClient = new WeakMap();
|
|
352
|
+
Object.defineProperty(globalThis, key, {
|
|
353
|
+
configurable: false,
|
|
354
|
+
enumerable: false,
|
|
355
|
+
value: stateByClient,
|
|
356
|
+
});
|
|
357
|
+
return stateByClient;
|
|
358
|
+
})();
|
|
359
|
+
/**
|
|
360
|
+
* @param {unknown} client
|
|
361
|
+
* @returns {object}
|
|
362
|
+
*/
|
|
363
|
+
function assertSqliteClientKey(client) {
|
|
364
|
+
if ((typeof client !== "object" && typeof client !== "function") ||
|
|
365
|
+
client === null) {
|
|
366
|
+
throw new Error("SmithersDb requires an object sqlite client for transaction coordination.");
|
|
367
|
+
}
|
|
368
|
+
return /** @type {object} */ (client);
|
|
369
|
+
}
|
|
370
|
+
/**
|
|
371
|
+
* @param {unknown} db
|
|
372
|
+
* @returns {object}
|
|
373
|
+
*/
|
|
374
|
+
function resolveSqliteClientKey(db) {
|
|
375
|
+
const source = /** @type {{ session?: { client?: unknown }; $client?: unknown }} */ (db);
|
|
376
|
+
return assertSqliteClientKey(source?.session?.client ?? source?.$client ?? db);
|
|
377
|
+
}
|
|
378
|
+
/**
|
|
379
|
+
* @param {unknown} client
|
|
380
|
+
* @returns {SqliteTransactionState}
|
|
381
|
+
*/
|
|
382
|
+
function getSqliteTransactionState(client) {
|
|
383
|
+
const key = assertSqliteClientKey(client);
|
|
384
|
+
let state = sqliteTransactionStateByClient.get(key);
|
|
385
|
+
if (!state) {
|
|
386
|
+
state = {
|
|
387
|
+
depth: 0,
|
|
388
|
+
ownerThread: null,
|
|
389
|
+
tail: Promise.resolve(),
|
|
390
|
+
};
|
|
391
|
+
sqliteTransactionStateByClient.set(key, state);
|
|
392
|
+
}
|
|
393
|
+
return state;
|
|
394
|
+
}
|
|
395
|
+
/**
|
|
396
|
+
* @param {unknown} db
|
|
397
|
+
* @returns {{ run: (sql: string) => unknown; query: (sql: string) => { run: (...args: unknown[]) => unknown; get: (...args: unknown[]) => Record<string, unknown> | null | undefined; all: () => Array<Record<string, unknown>> }; exec: (sql: string) => unknown; $client?: unknown }}
|
|
398
|
+
*/
|
|
399
|
+
function resolveSqliteTransactionClient(db) {
|
|
400
|
+
const candidate = /** @type {{ run?: unknown; query?: unknown; exec?: unknown; $client?: unknown }} */ (resolveSqliteClientKey(db));
|
|
401
|
+
if (typeof candidate.run !== "function") {
|
|
402
|
+
throw new Error("SmithersDb.withTransaction requires Bun SQLite client transaction primitives.");
|
|
403
|
+
}
|
|
404
|
+
return /** @type {{ run: (sql: string) => unknown; query: (sql: string) => { run: (...args: unknown[]) => unknown; get: (...args: unknown[]) => Record<string, unknown> | null | undefined; all: () => Array<Record<string, unknown>> }; exec: (sql: string) => unknown; $client?: unknown }} */ (candidate);
|
|
405
|
+
}
|
|
340
406
|
export class SmithersDb {
|
|
341
407
|
/** @type {BunSQLiteDatabase<Record<string, unknown>>} */
|
|
342
408
|
db;
|
|
@@ -433,8 +499,12 @@ export class SmithersDb {
|
|
|
433
499
|
* @returns {boolean}
|
|
434
500
|
*/
|
|
435
501
|
ownsActiveTransaction(currentFiberThread) {
|
|
436
|
-
|
|
437
|
-
|
|
502
|
+
const state = getSqliteTransactionState(resolveSqliteClientKey(this.db));
|
|
503
|
+
this.transactionDepth = state.depth;
|
|
504
|
+
this.transactionOwnerThread = state.ownerThread;
|
|
505
|
+
this.transactionTail = state.tail;
|
|
506
|
+
return (state.depth > 0 &&
|
|
507
|
+
state.ownerThread === currentFiberThread);
|
|
438
508
|
}
|
|
439
509
|
/**
|
|
440
510
|
* @template A
|
|
@@ -508,11 +578,7 @@ export class SmithersDb {
|
|
|
508
578
|
getSqliteTransactionClient() {
|
|
509
579
|
return Effect.try({
|
|
510
580
|
try: () => {
|
|
511
|
-
|
|
512
|
-
if (!candidate || typeof candidate.run !== "function") {
|
|
513
|
-
throw new Error("SmithersDb.withTransaction requires Bun SQLite client transaction primitives.");
|
|
514
|
-
}
|
|
515
|
-
return candidate;
|
|
581
|
+
return resolveSqliteTransactionClient(this.db);
|
|
516
582
|
},
|
|
517
583
|
catch: (cause) => toSmithersError(cause, "resolve sqlite transaction client", {
|
|
518
584
|
code: "DB_WRITE_FAILED",
|
|
@@ -526,12 +592,14 @@ export class SmithersDb {
|
|
|
526
592
|
acquireTransactionTurn() {
|
|
527
593
|
return Effect.tryPromise({
|
|
528
594
|
try: async () => {
|
|
595
|
+
const state = getSqliteTransactionState(resolveSqliteClientKey(this.db));
|
|
529
596
|
let release;
|
|
530
597
|
const gate = new Promise((resolve) => {
|
|
531
598
|
release = resolve;
|
|
532
599
|
});
|
|
533
|
-
const previous =
|
|
534
|
-
|
|
600
|
+
const previous = state.tail.catch(() => undefined);
|
|
601
|
+
state.tail = previous.then(() => gate);
|
|
602
|
+
this.transactionTail = state.tail;
|
|
535
603
|
await previous;
|
|
536
604
|
return release;
|
|
537
605
|
},
|
|
@@ -560,6 +628,7 @@ export class SmithersDb {
|
|
|
560
628
|
}));
|
|
561
629
|
}
|
|
562
630
|
const releaseTurn = yield* self.acquireTransactionTurn();
|
|
631
|
+
const transactionState = getSqliteTransactionState(resolveSqliteClientKey(self.db));
|
|
563
632
|
const start = performance.now();
|
|
564
633
|
return yield* Effect.gen(function* () {
|
|
565
634
|
const client = yield* self.getSqliteTransactionClient();
|
|
@@ -586,8 +655,11 @@ export class SmithersDb {
|
|
|
586
655
|
yield* Effect.try({
|
|
587
656
|
try: () => {
|
|
588
657
|
client.run("BEGIN IMMEDIATE");
|
|
589
|
-
|
|
590
|
-
|
|
658
|
+
transactionState.depth += 1;
|
|
659
|
+
transactionState.ownerThread = currentFiberThread;
|
|
660
|
+
self.transactionDepth = transactionState.depth;
|
|
661
|
+
self.transactionOwnerThread = transactionState.ownerThread;
|
|
662
|
+
self.transactionTail = transactionState.tail;
|
|
591
663
|
},
|
|
592
664
|
catch: (cause) => toSmithersError(cause, "begin sqlite transaction", {
|
|
593
665
|
code: "DB_WRITE_FAILED",
|
|
@@ -614,10 +686,13 @@ export class SmithersDb {
|
|
|
614
686
|
}
|
|
615
687
|
return operationExit.value;
|
|
616
688
|
}).pipe(Effect.ensuring(Effect.gen(function* () {
|
|
617
|
-
|
|
618
|
-
if (
|
|
619
|
-
|
|
689
|
+
transactionState.depth = Math.max(0, transactionState.depth - 1);
|
|
690
|
+
if (transactionState.depth === 0) {
|
|
691
|
+
transactionState.ownerThread = null;
|
|
620
692
|
}
|
|
693
|
+
self.transactionDepth = transactionState.depth;
|
|
694
|
+
self.transactionOwnerThread = transactionState.ownerThread;
|
|
695
|
+
self.transactionTail = transactionState.tail;
|
|
621
696
|
yield* Metric.update(dbTransactionDuration, performance.now() - start);
|
|
622
697
|
}))).pipe(Effect.ensuring(Effect.sync(() => {
|
|
623
698
|
releaseTurn();
|
|
@@ -1577,10 +1652,10 @@ export class SmithersDb {
|
|
|
1577
1652
|
if (existing?.seq !== undefined) {
|
|
1578
1653
|
return existing.seq;
|
|
1579
1654
|
}
|
|
1580
|
-
const client = self.db
|
|
1581
|
-
if (
|
|
1582
|
-
typeof client.
|
|
1583
|
-
typeof client.
|
|
1655
|
+
const client = /** @type {{ exec?: unknown; query?: unknown; run?: unknown }} */ (resolveSqliteClientKey(self.db));
|
|
1656
|
+
if (typeof client.exec !== "function" ||
|
|
1657
|
+
typeof client.query !== "function" ||
|
|
1658
|
+
typeof client.run !== "function") {
|
|
1584
1659
|
const lastSeq = (yield* self.getLastSignalSeq(row.runId)) ?? -1;
|
|
1585
1660
|
const seq = lastSeq + 1;
|
|
1586
1661
|
yield* Effect.tryPromise({
|
|
@@ -1593,6 +1668,7 @@ export class SmithersDb {
|
|
|
1593
1668
|
});
|
|
1594
1669
|
return seq;
|
|
1595
1670
|
}
|
|
1671
|
+
const releaseTurn = yield* self.acquireTransactionTurn();
|
|
1596
1672
|
return yield* Effect.try({
|
|
1597
1673
|
try: () => {
|
|
1598
1674
|
client.run("BEGIN IMMEDIATE");
|
|
@@ -1618,7 +1694,9 @@ export class SmithersDb {
|
|
|
1618
1694
|
}
|
|
1619
1695
|
},
|
|
1620
1696
|
catch: (cause) => toSmithersError(cause, "insert signal transaction"),
|
|
1621
|
-
})
|
|
1697
|
+
}).pipe(Effect.ensuring(Effect.sync(() => {
|
|
1698
|
+
releaseTurn();
|
|
1699
|
+
})));
|
|
1622
1700
|
}), { label }).pipe(Effect.annotateLogs({
|
|
1623
1701
|
runId: row.runId,
|
|
1624
1702
|
signalName: row.signalName,
|
|
@@ -1738,10 +1816,10 @@ export class SmithersDb {
|
|
|
1738
1816
|
if (existing?.seq !== undefined) {
|
|
1739
1817
|
return existing.seq;
|
|
1740
1818
|
}
|
|
1741
|
-
const client = self.db
|
|
1742
|
-
if (
|
|
1743
|
-
typeof client.
|
|
1744
|
-
typeof client.
|
|
1819
|
+
const client = /** @type {{ exec?: unknown; query?: unknown; run?: unknown }} */ (resolveSqliteClientKey(self.db));
|
|
1820
|
+
if (typeof client.exec !== "function" ||
|
|
1821
|
+
typeof client.query !== "function" ||
|
|
1822
|
+
typeof client.run !== "function") {
|
|
1745
1823
|
const lastSeq = (yield* self.getLastEventSeq(row.runId)) ?? -1;
|
|
1746
1824
|
const seq = lastSeq + 1;
|
|
1747
1825
|
yield* Effect.tryPromise({
|
|
@@ -1750,6 +1828,7 @@ export class SmithersDb {
|
|
|
1750
1828
|
});
|
|
1751
1829
|
return seq;
|
|
1752
1830
|
}
|
|
1831
|
+
const releaseTurn = yield* self.acquireTransactionTurn();
|
|
1753
1832
|
return yield* Effect.try({
|
|
1754
1833
|
try: () => {
|
|
1755
1834
|
client.run("BEGIN IMMEDIATE");
|
|
@@ -1775,7 +1854,9 @@ export class SmithersDb {
|
|
|
1775
1854
|
}
|
|
1776
1855
|
},
|
|
1777
1856
|
catch: (cause) => toSmithersError(cause, "insert event transaction"),
|
|
1778
|
-
})
|
|
1857
|
+
}).pipe(Effect.ensuring(Effect.sync(() => {
|
|
1858
|
+
releaseTurn();
|
|
1859
|
+
})));
|
|
1779
1860
|
}), { label }).pipe(Effect.annotateLogs({ dbOperation: label }), Effect.withLogSpan(`db:${label}`)));
|
|
1780
1861
|
}
|
|
1781
1862
|
/**
|
package/src/adapter.js
CHANGED
|
@@ -14,7 +14,7 @@ import { getTableName, sql } from "drizzle-orm";
|
|
|
14
14
|
import { Effect, Exit, FiberId, Metric } from "effect";
|
|
15
15
|
import { toSmithersError } from "@smithers-orchestrator/errors/toSmithersError";
|
|
16
16
|
import { getSqlMessageStorage } from "./sql-message-storage.js";
|
|
17
|
-
import { alertsAcknowledgedTotal, alertsActive, alertsFiredTotal, dbQueryDuration, dbTransactionDuration,
|
|
17
|
+
import { alertsAcknowledgedTotal, alertsActive, alertsFiredTotal, dbQueryDuration, dbTransactionDuration, dbTransactionRollbacks, } from "@smithers-orchestrator/observability/metrics";
|
|
18
18
|
import { assertOptionalStringMaxLength, assertPositiveFiniteNumber, } from "./input-bounds.js";
|
|
19
19
|
import { FRAME_KEYFRAME_INTERVAL, applyFrameDeltaJson, encodeFrameDelta, normalizeFrameEncoding, serializeFrameDelta, } from "./frame-codec.js";
|
|
20
20
|
import { getKeyColumns } from "./output.js";
|
|
@@ -77,7 +77,6 @@ export const DB_RUN_ALLOWED_STATUSES = [
|
|
|
77
77
|
"cancelled",
|
|
78
78
|
"continued",
|
|
79
79
|
];
|
|
80
|
-
const RUN_HEARTBEAT_STALE_MS = 30_000;
|
|
81
80
|
const RAW_QUERY_ALLOWED_PREFIX = /^(?:select|with|explain|values)\b/i;
|
|
82
81
|
const RAW_QUERY_FORBIDDEN_KEYWORDS = /\b(?:drop|delete|insert|update|alter|create|attach|detach|pragma)\b/i;
|
|
83
82
|
const ACTIVE_ALERT_STATUSES = new Set([
|
|
@@ -354,6 +353,73 @@ function runnableEffect(effect) {
|
|
|
354
353
|
}
|
|
355
354
|
return runnable;
|
|
356
355
|
}
|
|
356
|
+
/**
|
|
357
|
+
* @typedef {{ depth: number; ownerThread: string | null; tail: Promise<unknown> }} SqliteTransactionState
|
|
358
|
+
*/
|
|
359
|
+
/** @type {WeakMap<object, SqliteTransactionState>} */
|
|
360
|
+
// Cross-adapter coordination: one sqlite connection cannot run overlapping BEGIN IMMEDIATE statements.
|
|
361
|
+
const sqliteTransactionStateByClient = (() => {
|
|
362
|
+
const key = Symbol.for("smithers.sqliteTransactionStateByClient");
|
|
363
|
+
const registry = /** @type {Record<PropertyKey, unknown>} */ (globalThis);
|
|
364
|
+
const existing = registry[key];
|
|
365
|
+
if (existing instanceof WeakMap) {
|
|
366
|
+
return /** @type {WeakMap<object, SqliteTransactionState>} */ (existing);
|
|
367
|
+
}
|
|
368
|
+
const stateByClient = new WeakMap();
|
|
369
|
+
Object.defineProperty(globalThis, key, {
|
|
370
|
+
configurable: false,
|
|
371
|
+
enumerable: false,
|
|
372
|
+
value: stateByClient,
|
|
373
|
+
});
|
|
374
|
+
return stateByClient;
|
|
375
|
+
})();
|
|
376
|
+
/**
|
|
377
|
+
* @param {unknown} client
|
|
378
|
+
* @returns {object}
|
|
379
|
+
*/
|
|
380
|
+
function assertSqliteClientKey(client) {
|
|
381
|
+
if ((typeof client !== "object" && typeof client !== "function") ||
|
|
382
|
+
client === null) {
|
|
383
|
+
throw new Error("SmithersDb requires an object sqlite client for transaction coordination.");
|
|
384
|
+
}
|
|
385
|
+
return /** @type {object} */ (client);
|
|
386
|
+
}
|
|
387
|
+
/**
|
|
388
|
+
* @param {unknown} db
|
|
389
|
+
* @returns {object}
|
|
390
|
+
*/
|
|
391
|
+
function resolveSqliteClientKey(db) {
|
|
392
|
+
const source = /** @type {{ session?: { client?: unknown }; $client?: unknown }} */ (db);
|
|
393
|
+
return assertSqliteClientKey(source?.session?.client ?? source?.$client ?? db);
|
|
394
|
+
}
|
|
395
|
+
/**
|
|
396
|
+
* @param {unknown} client
|
|
397
|
+
* @returns {SqliteTransactionState}
|
|
398
|
+
*/
|
|
399
|
+
function getSqliteTransactionState(client) {
|
|
400
|
+
const key = assertSqliteClientKey(client);
|
|
401
|
+
let state = sqliteTransactionStateByClient.get(key);
|
|
402
|
+
if (!state) {
|
|
403
|
+
state = {
|
|
404
|
+
depth: 0,
|
|
405
|
+
ownerThread: null,
|
|
406
|
+
tail: Promise.resolve(),
|
|
407
|
+
};
|
|
408
|
+
sqliteTransactionStateByClient.set(key, state);
|
|
409
|
+
}
|
|
410
|
+
return state;
|
|
411
|
+
}
|
|
412
|
+
/**
|
|
413
|
+
* @param {unknown} db
|
|
414
|
+
* @returns {{ run: (sql: string) => unknown; query: (sql: string) => { run: (...args: unknown[]) => unknown; get: (...args: unknown[]) => Record<string, unknown> | null | undefined; all: () => Array<Record<string, unknown>> }; exec: (sql: string) => unknown; $client?: unknown }}
|
|
415
|
+
*/
|
|
416
|
+
function resolveSqliteTransactionClient(db) {
|
|
417
|
+
const candidate = /** @type {{ run?: unknown; query?: unknown; exec?: unknown; $client?: unknown }} */ (resolveSqliteClientKey(db));
|
|
418
|
+
if (typeof candidate.run !== "function") {
|
|
419
|
+
throw new Error("SmithersDb.withTransaction requires Bun SQLite client transaction primitives.");
|
|
420
|
+
}
|
|
421
|
+
return /** @type {{ run: (sql: string) => unknown; query: (sql: string) => { run: (...args: unknown[]) => unknown; get: (...args: unknown[]) => Record<string, unknown> | null | undefined; all: () => Array<Record<string, unknown>> }; exec: (sql: string) => unknown; $client?: unknown }} */ (candidate);
|
|
422
|
+
}
|
|
357
423
|
export class SmithersDb {
|
|
358
424
|
/** @type {BunSQLiteDatabase<Record<string, unknown>>} */
|
|
359
425
|
db;
|
|
@@ -450,8 +516,12 @@ export class SmithersDb {
|
|
|
450
516
|
* @returns {boolean}
|
|
451
517
|
*/
|
|
452
518
|
ownsActiveTransaction(currentFiberThread) {
|
|
453
|
-
|
|
454
|
-
|
|
519
|
+
const state = getSqliteTransactionState(resolveSqliteClientKey(this.db));
|
|
520
|
+
this.transactionDepth = state.depth;
|
|
521
|
+
this.transactionOwnerThread = state.ownerThread;
|
|
522
|
+
this.transactionTail = state.tail;
|
|
523
|
+
return (state.depth > 0 &&
|
|
524
|
+
state.ownerThread === currentFiberThread);
|
|
455
525
|
}
|
|
456
526
|
/**
|
|
457
527
|
* @template A
|
|
@@ -525,11 +595,7 @@ export class SmithersDb {
|
|
|
525
595
|
getSqliteTransactionClient() {
|
|
526
596
|
return Effect.try({
|
|
527
597
|
try: () => {
|
|
528
|
-
|
|
529
|
-
if (!candidate || typeof candidate.run !== "function") {
|
|
530
|
-
throw new Error("SmithersDb.withTransaction requires Bun SQLite client transaction primitives.");
|
|
531
|
-
}
|
|
532
|
-
return candidate;
|
|
598
|
+
return resolveSqliteTransactionClient(this.db);
|
|
533
599
|
},
|
|
534
600
|
catch: (cause) => toSmithersError(cause, "resolve sqlite transaction client", {
|
|
535
601
|
code: "DB_WRITE_FAILED",
|
|
@@ -543,12 +609,14 @@ export class SmithersDb {
|
|
|
543
609
|
acquireTransactionTurn() {
|
|
544
610
|
return Effect.tryPromise({
|
|
545
611
|
try: async () => {
|
|
612
|
+
const state = getSqliteTransactionState(resolveSqliteClientKey(this.db));
|
|
546
613
|
let release;
|
|
547
614
|
const gate = new Promise((resolve) => {
|
|
548
615
|
release = resolve;
|
|
549
616
|
});
|
|
550
|
-
const previous =
|
|
551
|
-
|
|
617
|
+
const previous = state.tail.catch(() => undefined);
|
|
618
|
+
state.tail = previous.then(() => gate);
|
|
619
|
+
this.transactionTail = state.tail;
|
|
552
620
|
await previous;
|
|
553
621
|
return release;
|
|
554
622
|
},
|
|
@@ -577,6 +645,7 @@ export class SmithersDb {
|
|
|
577
645
|
}));
|
|
578
646
|
}
|
|
579
647
|
const releaseTurn = yield* self.acquireTransactionTurn();
|
|
648
|
+
const transactionState = getSqliteTransactionState(resolveSqliteClientKey(self.db));
|
|
580
649
|
const start = performance.now();
|
|
581
650
|
return yield* Effect.gen(function* () {
|
|
582
651
|
const client = yield* self.getSqliteTransactionClient();
|
|
@@ -603,8 +672,11 @@ export class SmithersDb {
|
|
|
603
672
|
yield* Effect.try({
|
|
604
673
|
try: () => {
|
|
605
674
|
client.run("BEGIN IMMEDIATE");
|
|
606
|
-
|
|
607
|
-
|
|
675
|
+
transactionState.depth += 1;
|
|
676
|
+
transactionState.ownerThread = currentFiberThread;
|
|
677
|
+
self.transactionDepth = transactionState.depth;
|
|
678
|
+
self.transactionOwnerThread = transactionState.ownerThread;
|
|
679
|
+
self.transactionTail = transactionState.tail;
|
|
608
680
|
},
|
|
609
681
|
catch: (cause) => toSmithersError(cause, "begin sqlite transaction", {
|
|
610
682
|
code: "DB_WRITE_FAILED",
|
|
@@ -631,10 +703,13 @@ export class SmithersDb {
|
|
|
631
703
|
}
|
|
632
704
|
return operationExit.value;
|
|
633
705
|
}).pipe(Effect.ensuring(Effect.gen(function* () {
|
|
634
|
-
|
|
635
|
-
if (
|
|
636
|
-
|
|
706
|
+
transactionState.depth = Math.max(0, transactionState.depth - 1);
|
|
707
|
+
if (transactionState.depth === 0) {
|
|
708
|
+
transactionState.ownerThread = null;
|
|
637
709
|
}
|
|
710
|
+
self.transactionDepth = transactionState.depth;
|
|
711
|
+
self.transactionOwnerThread = transactionState.ownerThread;
|
|
712
|
+
self.transactionTail = transactionState.tail;
|
|
638
713
|
yield* Metric.update(dbTransactionDuration, performance.now() - start);
|
|
639
714
|
}))).pipe(Effect.ensuring(Effect.sync(() => {
|
|
640
715
|
releaseTurn();
|
|
@@ -1594,10 +1669,10 @@ export class SmithersDb {
|
|
|
1594
1669
|
if (existing?.seq !== undefined) {
|
|
1595
1670
|
return existing.seq;
|
|
1596
1671
|
}
|
|
1597
|
-
const client = self.db
|
|
1598
|
-
if (
|
|
1599
|
-
typeof client.
|
|
1600
|
-
typeof client.
|
|
1672
|
+
const client = /** @type {{ exec?: unknown; query?: unknown; run?: unknown }} */ (resolveSqliteClientKey(self.db));
|
|
1673
|
+
if (typeof client.exec !== "function" ||
|
|
1674
|
+
typeof client.query !== "function" ||
|
|
1675
|
+
typeof client.run !== "function") {
|
|
1601
1676
|
const lastSeq = (yield* self.getLastSignalSeq(row.runId)) ?? -1;
|
|
1602
1677
|
const seq = lastSeq + 1;
|
|
1603
1678
|
yield* Effect.tryPromise({
|
|
@@ -1610,6 +1685,7 @@ export class SmithersDb {
|
|
|
1610
1685
|
});
|
|
1611
1686
|
return seq;
|
|
1612
1687
|
}
|
|
1688
|
+
const releaseTurn = yield* self.acquireTransactionTurn();
|
|
1613
1689
|
return yield* Effect.try({
|
|
1614
1690
|
try: () => {
|
|
1615
1691
|
client.run("BEGIN IMMEDIATE");
|
|
@@ -1635,7 +1711,9 @@ export class SmithersDb {
|
|
|
1635
1711
|
}
|
|
1636
1712
|
},
|
|
1637
1713
|
catch: (cause) => toSmithersError(cause, "insert signal transaction"),
|
|
1638
|
-
})
|
|
1714
|
+
}).pipe(Effect.ensuring(Effect.sync(() => {
|
|
1715
|
+
releaseTurn();
|
|
1716
|
+
})));
|
|
1639
1717
|
}), { label }).pipe(Effect.annotateLogs({
|
|
1640
1718
|
runId: row.runId,
|
|
1641
1719
|
signalName: row.signalName,
|
|
@@ -1755,10 +1833,10 @@ export class SmithersDb {
|
|
|
1755
1833
|
if (existing?.seq !== undefined) {
|
|
1756
1834
|
return existing.seq;
|
|
1757
1835
|
}
|
|
1758
|
-
const client = self.db
|
|
1759
|
-
if (
|
|
1760
|
-
typeof client.
|
|
1761
|
-
typeof client.
|
|
1836
|
+
const client = /** @type {{ exec?: unknown; query?: unknown; run?: unknown }} */ (resolveSqliteClientKey(self.db));
|
|
1837
|
+
if (typeof client.exec !== "function" ||
|
|
1838
|
+
typeof client.query !== "function" ||
|
|
1839
|
+
typeof client.run !== "function") {
|
|
1762
1840
|
const lastSeq = (yield* self.getLastEventSeq(row.runId)) ?? -1;
|
|
1763
1841
|
const seq = lastSeq + 1;
|
|
1764
1842
|
yield* Effect.tryPromise({
|
|
@@ -1767,6 +1845,7 @@ export class SmithersDb {
|
|
|
1767
1845
|
});
|
|
1768
1846
|
return seq;
|
|
1769
1847
|
}
|
|
1848
|
+
const releaseTurn = yield* self.acquireTransactionTurn();
|
|
1770
1849
|
return yield* Effect.try({
|
|
1771
1850
|
try: () => {
|
|
1772
1851
|
client.run("BEGIN IMMEDIATE");
|
|
@@ -1792,7 +1871,9 @@ export class SmithersDb {
|
|
|
1792
1871
|
}
|
|
1793
1872
|
},
|
|
1794
1873
|
catch: (cause) => toSmithersError(cause, "insert event transaction"),
|
|
1795
|
-
})
|
|
1874
|
+
}).pipe(Effect.ensuring(Effect.sync(() => {
|
|
1875
|
+
releaseTurn();
|
|
1876
|
+
})));
|
|
1796
1877
|
}), { label }).pipe(Effect.annotateLogs({ dbOperation: label }), Effect.withLogSpan(`db:${label}`)));
|
|
1797
1878
|
}
|
|
1798
1879
|
/**
|
package/src/index.d.ts
CHANGED
|
@@ -9,7 +9,6 @@ import { Database } from 'bun:sqlite';
|
|
|
9
9
|
import * as SqlClient from '@effect/sql/SqlClient';
|
|
10
10
|
import { SqlError } from '@effect/sql/SqlError';
|
|
11
11
|
import * as drizzle_orm_sqlite_core from 'drizzle-orm/sqlite-core';
|
|
12
|
-
import * as _smithers_driver_OutputSnapshot from '@smithers-orchestrator/driver/OutputSnapshot';
|
|
13
12
|
|
|
14
13
|
type SchemaRegistryEntry$1 = {
|
|
15
14
|
table: Table$1;
|
|
@@ -5018,7 +5017,7 @@ declare function loadOutputsEffect(db: BunSQLiteDatabase<Record<string, unknown>
|
|
|
5018
5017
|
* @returns {Promise<OutputSnapshot>}
|
|
5019
5018
|
*/
|
|
5020
5019
|
declare function loadOutputs(db: BunSQLiteDatabase<Record<string, unknown>>, schema: Record<string, _Table | unknown>, runId: string): Promise<OutputSnapshot>;
|
|
5021
|
-
type OutputSnapshot =
|
|
5020
|
+
type OutputSnapshot = Record<string, Array<unknown>>;
|
|
5022
5021
|
type BunSQLiteDatabase = drizzle_orm_bun_sqlite.BunSQLiteDatabase;
|
|
5023
5022
|
type _Table = drizzle_orm.Table;
|
|
5024
5023
|
|
|
@@ -5156,6 +5155,13 @@ type SqliteWriteRetryOptions = SqliteWriteRetryOptions$2;
|
|
|
5156
5155
|
*/
|
|
5157
5156
|
declare function zodToCreateTableSQL(tableName: any, schema: any, opts: any): string;
|
|
5158
5157
|
|
|
5158
|
+
declare function zodSchemaColumns(schema: any): {
|
|
5159
|
+
name: any;
|
|
5160
|
+
sqliteType: string;
|
|
5161
|
+
}[];
|
|
5162
|
+
|
|
5163
|
+
declare function syncZodTableSchema(sqlite: any, tableName: any, schema: any, opts: any): void;
|
|
5164
|
+
|
|
5159
5165
|
/**
|
|
5160
5166
|
* Generates a Drizzle sqliteTable from a Zod object schema.
|
|
5161
5167
|
*
|
|
@@ -5200,4 +5206,4 @@ declare function camelToSnake(str: any): any;
|
|
|
5200
5206
|
|
|
5201
5207
|
type SchemaRegistryEntry = SchemaRegistryEntry$1;
|
|
5202
5208
|
|
|
5203
|
-
export { type AlertRow, type AlertSeverity, type AlertStatus, type AnyColumn, type ApprovalRow, type AttemptRow, type CacheRow, type CacheRowLike, type CountRow, DB_ALERT_ALLOWED_SEVERITIES, DB_ALERT_ALLOWED_STATUSES, DB_ALERT_ID_MAX_LENGTH, DB_ALERT_MESSAGE_MAX_LENGTH, DB_ALERT_POLICY_NAME_MAX_LENGTH, DB_RUN_ALLOWED_STATUSES, DB_RUN_ID_MAX_LENGTH, DB_RUN_WORKFLOW_NAME_MAX_LENGTH, type EventHistoryQuery, FRAME_KEYFRAME_INTERVAL, type FrameDelta, type FrameDeltaOp, type FrameEncoding, type FrameRow, type HumanRequestRow, type JsonBounds, type JsonPath, type JsonPathSegment, NODE_DIFF_MAX_BYTES, NodeDiffCache, type NodeDiffCacheResult, type NodeDiffCacheRow$1 as NodeDiffCacheRow, NodeDiffTooLargeError, type NodeRow, type OutputKey, type OutputSnapshot, type PendingHumanRequestRow, type RalphRow, type RunAncestryRow, type RunRow, type RunnableEffect, type SchemaRegistryEntry, type SignalQuery, type SignalRow, SmithersDb, type SmithersError$1 as SmithersError, SqlMessageStorage, type SqlMessageStorageEventHistoryQuery, type SqliteParam, type SqliteWriteRetryOptions, type StaleRunRecord, type _BunSQLiteDatabase, type _NodeDiffCacheRow, type _OutputKey, type _SmithersDb, type _SmithersError, applyFrameDelta, applyFrameDeltaJson, assertJsonPayloadWithinBounds, assertMaxBytes, assertMaxJsonDepth, assertMaxStringLength, assertOptionalArrayMaxLength, assertOptionalStringMaxLength, assertPositiveFiniteInteger, assertPositiveFiniteNumber, buildKeyWhere, buildOutputRow, camelToSnake, describeSchemaShape, encodeFrameDelta, ensureSmithersTables, ensureSmithersTablesEffect, ensureSqlMessageStorage, ensureSqlMessageStorageEffect, getAgentOutputSchema, getKeyColumns, getSqlMessageStorage, isRetryableSqliteWriteError, loadInput, loadInputEffect, loadOutputs, loadOutputsEffect, normalizeFrameEncoding, parseFrameDelta, schemaSignature, selectOutputRow, selectOutputRowEffect, serializeFrameDelta, smithersAlerts, smithersApprovals, smithersAttempts, smithersCache, smithersCron, smithersEvents, smithersFrames, smithersHumanRequests, smithersNodeDiffs, smithersNodes, smithersRalph, smithersRuns, smithersSandboxes, smithersSignals, smithersTimeTravelAudit, smithersToolCalls, smithersVectors, stripAutoColumns, unwrapZodType, upsertOutputRow, upsertOutputRowEffect, validateExistingOutput, validateInput, validateOutput, withSqliteWriteRetry, withSqliteWriteRetryEffect, zodToCreateTableSQL, zodToTable };
|
|
5209
|
+
export { type AlertRow, type AlertSeverity, type AlertStatus, type AnyColumn, type ApprovalRow, type AttemptRow, type CacheRow, type CacheRowLike, type CountRow, DB_ALERT_ALLOWED_SEVERITIES, DB_ALERT_ALLOWED_STATUSES, DB_ALERT_ID_MAX_LENGTH, DB_ALERT_MESSAGE_MAX_LENGTH, DB_ALERT_POLICY_NAME_MAX_LENGTH, DB_RUN_ALLOWED_STATUSES, DB_RUN_ID_MAX_LENGTH, DB_RUN_WORKFLOW_NAME_MAX_LENGTH, type EventHistoryQuery, FRAME_KEYFRAME_INTERVAL, type FrameDelta, type FrameDeltaOp, type FrameEncoding, type FrameRow, type HumanRequestRow, type JsonBounds, type JsonPath, type JsonPathSegment, NODE_DIFF_MAX_BYTES, NodeDiffCache, type NodeDiffCacheResult, type NodeDiffCacheRow$1 as NodeDiffCacheRow, NodeDiffTooLargeError, type NodeRow, type OutputKey, type OutputSnapshot, type PendingHumanRequestRow, type RalphRow, type RunAncestryRow, type RunRow, type RunnableEffect, type SchemaRegistryEntry, type SignalQuery, type SignalRow, SmithersDb, type SmithersError$1 as SmithersError, SqlMessageStorage, type SqlMessageStorageEventHistoryQuery, type SqliteParam, type SqliteWriteRetryOptions, type StaleRunRecord, type _BunSQLiteDatabase, type _NodeDiffCacheRow, type _OutputKey, type _SmithersDb, type _SmithersError, applyFrameDelta, applyFrameDeltaJson, assertJsonPayloadWithinBounds, assertMaxBytes, assertMaxJsonDepth, assertMaxStringLength, assertOptionalArrayMaxLength, assertOptionalStringMaxLength, assertPositiveFiniteInteger, assertPositiveFiniteNumber, buildKeyWhere, buildOutputRow, camelToSnake, describeSchemaShape, encodeFrameDelta, ensureSmithersTables, ensureSmithersTablesEffect, ensureSqlMessageStorage, ensureSqlMessageStorageEffect, getAgentOutputSchema, getKeyColumns, getSqlMessageStorage, isRetryableSqliteWriteError, loadInput, loadInputEffect, loadOutputs, loadOutputsEffect, normalizeFrameEncoding, parseFrameDelta, schemaSignature, selectOutputRow, selectOutputRowEffect, serializeFrameDelta, smithersAlerts, smithersApprovals, smithersAttempts, smithersCache, smithersCron, smithersEvents, smithersFrames, smithersHumanRequests, smithersNodeDiffs, smithersNodes, smithersRalph, smithersRuns, smithersSandboxes, smithersSignals, smithersTimeTravelAudit, smithersToolCalls, smithersVectors, stripAutoColumns, syncZodTableSchema, unwrapZodType, upsertOutputRow, upsertOutputRowEffect, validateExistingOutput, validateInput, validateOutput, withSqliteWriteRetry, withSqliteWriteRetryEffect, zodSchemaColumns, zodToCreateTableSQL, zodToTable };
|
package/src/input.js
CHANGED
|
@@ -13,7 +13,9 @@ export { smithersSandboxes } from "./smithersSandboxes.js";
|
|
|
13
13
|
export { smithersToolCalls } from "./smithersToolCalls.js";
|
|
14
14
|
export { smithersEvents } from "./smithersEvents.js";
|
|
15
15
|
export { smithersRalph } from "./smithersRalph.js";
|
|
16
|
-
export { smithersScorers } from "
|
|
17
|
-
export { smithersMemoryFacts
|
|
16
|
+
export { smithersScorers } from "./smithersScorers.js";
|
|
17
|
+
export { smithersMemoryFacts } from "./smithersMemoryFacts.js";
|
|
18
|
+
export { smithersMemoryThreads } from "./smithersMemoryThreads.js";
|
|
19
|
+
export { smithersMemoryMessages } from "./smithersMemoryMessages.js";
|
|
18
20
|
export { smithersVectors } from "./smithersVectors.js";
|
|
19
21
|
export { smithersCron } from "./smithersCron.js";
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { integer, sqliteTable, text, primaryKey } from "drizzle-orm/sqlite-core";
|
|
2
|
+
|
|
3
|
+
export const smithersMemoryFacts = sqliteTable("_smithers_memory_facts", {
|
|
4
|
+
namespace: text("namespace").notNull(),
|
|
5
|
+
key: text("key").notNull(),
|
|
6
|
+
valueJson: text("value_json").notNull(),
|
|
7
|
+
schemaSig: text("schema_sig"),
|
|
8
|
+
createdAtMs: integer("created_at_ms").notNull(),
|
|
9
|
+
updatedAtMs: integer("updated_at_ms").notNull(),
|
|
10
|
+
ttlMs: integer("ttl_ms"),
|
|
11
|
+
}, (t) => ({
|
|
12
|
+
pk: primaryKey({ columns: [t.namespace, t.key] }),
|
|
13
|
+
}));
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
|
|
2
|
+
|
|
3
|
+
export const smithersMemoryMessages = sqliteTable("_smithers_memory_messages", {
|
|
4
|
+
id: text("id").primaryKey(),
|
|
5
|
+
threadId: text("thread_id").notNull(),
|
|
6
|
+
role: text("role").notNull(),
|
|
7
|
+
contentJson: text("content_json").notNull(),
|
|
8
|
+
runId: text("run_id"),
|
|
9
|
+
nodeId: text("node_id"),
|
|
10
|
+
createdAtMs: integer("created_at_ms").notNull(),
|
|
11
|
+
});
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
|
|
2
|
+
|
|
3
|
+
export const smithersMemoryThreads = sqliteTable("_smithers_memory_threads", {
|
|
4
|
+
threadId: text("thread_id").primaryKey(),
|
|
5
|
+
namespace: text("namespace").notNull(),
|
|
6
|
+
title: text("title"),
|
|
7
|
+
metadataJson: text("metadata_json"),
|
|
8
|
+
createdAtMs: integer("created_at_ms").notNull(),
|
|
9
|
+
updatedAtMs: integer("updated_at_ms").notNull(),
|
|
10
|
+
});
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { integer, real, sqliteTable, text } from "drizzle-orm/sqlite-core";
|
|
2
|
+
|
|
3
|
+
export const smithersScorers = sqliteTable("_smithers_scorers", {
|
|
4
|
+
id: text("id").primaryKey(),
|
|
5
|
+
runId: text("run_id").notNull(),
|
|
6
|
+
nodeId: text("node_id").notNull(),
|
|
7
|
+
iteration: integer("iteration").notNull().default(0),
|
|
8
|
+
attempt: integer("attempt").notNull().default(0),
|
|
9
|
+
scorerId: text("scorer_id").notNull(),
|
|
10
|
+
scorerName: text("scorer_name").notNull(),
|
|
11
|
+
source: text("source").notNull(),
|
|
12
|
+
score: real("score").notNull(),
|
|
13
|
+
reason: text("reason"),
|
|
14
|
+
metaJson: text("meta_json"),
|
|
15
|
+
inputJson: text("input_json"),
|
|
16
|
+
outputJson: text("output_json"),
|
|
17
|
+
latencyMs: real("latency_ms"),
|
|
18
|
+
scoredAtMs: integer("scored_at_ms").notNull(),
|
|
19
|
+
durationMs: real("duration_ms"),
|
|
20
|
+
});
|
package/src/internal-schema.js
CHANGED
|
@@ -220,8 +220,10 @@ export const smithersRalph = sqliteTable("_smithers_ralph", {
|
|
|
220
220
|
}, (t) => ({
|
|
221
221
|
pk: primaryKey({ columns: [t.runId, t.ralphId] }),
|
|
222
222
|
}));
|
|
223
|
-
export { smithersScorers } from "
|
|
224
|
-
export { smithersMemoryFacts
|
|
223
|
+
export { smithersScorers } from "./internal-schema/smithersScorers.js";
|
|
224
|
+
export { smithersMemoryFacts } from "./internal-schema/smithersMemoryFacts.js";
|
|
225
|
+
export { smithersMemoryThreads } from "./internal-schema/smithersMemoryThreads.js";
|
|
226
|
+
export { smithersMemoryMessages } from "./internal-schema/smithersMemoryMessages.js";
|
|
225
227
|
export const smithersVectors = sqliteTable("_smithers_vectors", {
|
|
226
228
|
id: text("id").primaryKey(),
|
|
227
229
|
namespace: text("namespace").notNull(),
|
package/src/loadOutputsEffect.js
CHANGED
|
@@ -2,8 +2,7 @@ import { eq, getTableName } from "drizzle-orm";
|
|
|
2
2
|
import { getTableColumns } from "drizzle-orm/utils";
|
|
3
3
|
import { Effect, Option } from "effect";
|
|
4
4
|
import { toSmithersError } from "@smithers-orchestrator/errors/toSmithersError";
|
|
5
|
-
|
|
6
|
-
/** @typedef {import("@smithers-orchestrator/driver/OutputSnapshot").OutputSnapshot} OutputSnapshot */
|
|
5
|
+
/** @typedef {Record<string, Array<unknown>>} OutputSnapshot */
|
|
7
6
|
/** @typedef {import("drizzle-orm/bun-sqlite").BunSQLiteDatabase} BunSQLiteDatabase */
|
|
8
7
|
/** @typedef {import("drizzle-orm").Table} Table */
|
|
9
8
|
|
|
@@ -8,6 +8,6 @@ export function getAgentOutputSchema(table) {
|
|
|
8
8
|
const baseSchema = createInsertSchema(table);
|
|
9
9
|
// Remove the key columns that smithers populates automatically
|
|
10
10
|
const shape = baseSchema.shape;
|
|
11
|
-
const { runId, nodeId, iteration, ...rest } = shape;
|
|
11
|
+
const { runId: _runId, nodeId: _nodeId, iteration: _iteration, ...rest } = shape;
|
|
12
12
|
return z.object(rest);
|
|
13
13
|
}
|
package/src/output.js
CHANGED
|
@@ -28,7 +28,7 @@ export function buildOutputRow(table, runId, nodeId, iteration, payload) {
|
|
|
28
28
|
return { runId, nodeId, iteration, payload: (payload ?? null) };
|
|
29
29
|
}
|
|
30
30
|
return {
|
|
31
|
-
...(/** @type {Record<string, unknown>} */ (payload
|
|
31
|
+
...(/** @type {Record<string, unknown>} */ (payload)),
|
|
32
32
|
runId, nodeId, iteration,
|
|
33
33
|
};
|
|
34
34
|
}
|
package/src/snapshot.js
CHANGED
|
@@ -3,7 +3,7 @@ import { getTableColumns } from "drizzle-orm/utils";
|
|
|
3
3
|
import { Effect, Option } from "effect";
|
|
4
4
|
import { toSmithersError } from "@smithers-orchestrator/errors/toSmithersError";
|
|
5
5
|
import { SmithersError } from "@smithers-orchestrator/errors/SmithersError";
|
|
6
|
-
/** @typedef {
|
|
6
|
+
/** @typedef {Record<string, Array<unknown>>} OutputSnapshot */
|
|
7
7
|
/** @typedef {import("drizzle-orm/bun-sqlite").BunSQLiteDatabase} BunSQLiteDatabase */
|
|
8
8
|
/** @typedef {import("drizzle-orm").Table} _Table */
|
|
9
9
|
|
|
@@ -34,8 +34,6 @@ const CREATE_TABLE_STATEMENTS = [
|
|
|
34
34
|
error_json TEXT,
|
|
35
35
|
config_json TEXT
|
|
36
36
|
)`,
|
|
37
|
-
`CREATE INDEX IF NOT EXISTS _smithers_runs_status_heartbeat_idx
|
|
38
|
-
ON _smithers_runs (status, heartbeat_at_ms)`,
|
|
39
37
|
`CREATE TABLE IF NOT EXISTS _smithers_nodes (
|
|
40
38
|
run_id TEXT NOT NULL,
|
|
41
39
|
node_id TEXT NOT NULL,
|
|
@@ -143,8 +141,6 @@ const CREATE_TABLE_STATEMENTS = [
|
|
|
143
141
|
received_by TEXT,
|
|
144
142
|
PRIMARY KEY (run_id, seq)
|
|
145
143
|
)`,
|
|
146
|
-
`CREATE INDEX IF NOT EXISTS _smithers_signals_lookup_idx
|
|
147
|
-
ON _smithers_signals (run_id, signal_name, correlation_id, received_at_ms)`,
|
|
148
144
|
`CREATE TABLE IF NOT EXISTS _smithers_cache (
|
|
149
145
|
cache_key TEXT PRIMARY KEY,
|
|
150
146
|
created_at_ms INTEGER NOT NULL,
|
|
@@ -177,8 +173,6 @@ const CREATE_TABLE_STATEMENTS = [
|
|
|
177
173
|
result TEXT NOT NULL,
|
|
178
174
|
duration_ms INTEGER
|
|
179
175
|
)`,
|
|
180
|
-
`CREATE INDEX IF NOT EXISTS _smithers_time_travel_audit_lookup_idx
|
|
181
|
-
ON _smithers_time_travel_audit (run_id, caller, timestamp_ms)`,
|
|
182
176
|
`CREATE TABLE IF NOT EXISTS _smithers_sandboxes (
|
|
183
177
|
run_id TEXT NOT NULL,
|
|
184
178
|
sandbox_id TEXT NOT NULL,
|
|
@@ -322,6 +316,14 @@ const CREATE_TABLE_STATEMENTS = [
|
|
|
322
316
|
created_at_ms INTEGER NOT NULL
|
|
323
317
|
)`,
|
|
324
318
|
];
|
|
319
|
+
const CREATE_INDEX_STATEMENTS = [
|
|
320
|
+
`CREATE INDEX IF NOT EXISTS _smithers_runs_status_heartbeat_idx
|
|
321
|
+
ON _smithers_runs (status, heartbeat_at_ms)`,
|
|
322
|
+
`CREATE INDEX IF NOT EXISTS _smithers_signals_lookup_idx
|
|
323
|
+
ON _smithers_signals (run_id, signal_name, correlation_id, received_at_ms)`,
|
|
324
|
+
`CREATE INDEX IF NOT EXISTS _smithers_time_travel_audit_lookup_idx
|
|
325
|
+
ON _smithers_time_travel_audit (run_id, caller, timestamp_ms)`,
|
|
326
|
+
];
|
|
325
327
|
const MIGRATION_STATEMENTS = [
|
|
326
328
|
`ALTER TABLE _smithers_attempts ADD COLUMN response_text TEXT`,
|
|
327
329
|
`ALTER TABLE _smithers_attempts ADD COLUMN jj_cwd TEXT`,
|
|
@@ -643,6 +645,9 @@ export class SqlMessageStorage {
|
|
|
643
645
|
// Ignore if another caller added it first.
|
|
644
646
|
}
|
|
645
647
|
}
|
|
648
|
+
for (const statement of CREATE_INDEX_STATEMENTS) {
|
|
649
|
+
sqlite.run(statement);
|
|
650
|
+
}
|
|
646
651
|
});
|
|
647
652
|
}
|
|
648
653
|
/**
|
|
@@ -2,6 +2,6 @@
|
|
|
2
2
|
/** @typedef {import("./StorageServiceShape.ts").StorageServiceShape} StorageServiceShape */
|
|
3
3
|
// @smithers-type-exports-end
|
|
4
4
|
|
|
5
|
-
import { Context
|
|
5
|
+
import { Context } from "effect";
|
|
6
6
|
export class StorageService extends Context.Tag("StorageService")() {
|
|
7
7
|
}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { z } from "zod";
|
|
2
1
|
import { unwrapZodType } from "./unwrapZodType.js";
|
|
3
2
|
import { camelToSnake } from "./utils/camelToSnake.js";
|
|
4
3
|
/**
|
|
@@ -7,6 +6,32 @@ import { camelToSnake } from "./utils/camelToSnake.js";
|
|
|
7
6
|
function getZodBaseTypeName(zodType) {
|
|
8
7
|
return zodType._zod?.def?.type ?? "unknown";
|
|
9
8
|
}
|
|
9
|
+
function sqliteTypeFor(zodFieldSchema) {
|
|
10
|
+
const baseType = unwrapZodType(zodFieldSchema);
|
|
11
|
+
const baseTypeName = getZodBaseTypeName(baseType);
|
|
12
|
+
if (baseTypeName === "number" ||
|
|
13
|
+
baseTypeName === "int" ||
|
|
14
|
+
baseTypeName === "float" ||
|
|
15
|
+
baseTypeName === "boolean") {
|
|
16
|
+
return "INTEGER";
|
|
17
|
+
}
|
|
18
|
+
return "TEXT";
|
|
19
|
+
}
|
|
20
|
+
function quoteIdentifier(identifier) {
|
|
21
|
+
return `"${String(identifier).replaceAll(`"`, `""`)}"`;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Returns the user-defined columns derived from a Zod schema (excluding the
|
|
25
|
+
* fixed run_id/node_id/iteration prefix). Each entry is `{ name, sqliteType }`.
|
|
26
|
+
*/
|
|
27
|
+
export function zodSchemaColumns(schema) {
|
|
28
|
+
const out = [];
|
|
29
|
+
const shape = schema.shape;
|
|
30
|
+
for (const [key] of Object.entries(shape)) {
|
|
31
|
+
out.push({ name: camelToSnake(key), sqliteType: sqliteTypeFor(shape[key]) });
|
|
32
|
+
}
|
|
33
|
+
return out;
|
|
34
|
+
}
|
|
10
35
|
/**
|
|
11
36
|
* Generates a CREATE TABLE IF NOT EXISTS SQL statement from a Zod schema.
|
|
12
37
|
* Used for runtime table creation without Drizzle migrations.
|
|
@@ -19,23 +44,42 @@ export function zodToCreateTableSQL(tableName, schema, opts) {
|
|
|
19
44
|
`node_id TEXT NOT NULL`,
|
|
20
45
|
`iteration INTEGER NOT NULL DEFAULT 0`,
|
|
21
46
|
];
|
|
22
|
-
const
|
|
23
|
-
|
|
24
|
-
const colName = camelToSnake(key);
|
|
25
|
-
const baseType = unwrapZodType(shape[key]);
|
|
26
|
-
const baseTypeName = getZodBaseTypeName(baseType);
|
|
27
|
-
if (baseTypeName === "number" ||
|
|
28
|
-
baseTypeName === "int" ||
|
|
29
|
-
baseTypeName === "float" ||
|
|
30
|
-
baseTypeName === "boolean") {
|
|
31
|
-
colDefs.push(`"${colName}" INTEGER`);
|
|
32
|
-
}
|
|
33
|
-
else {
|
|
34
|
-
colDefs.push(`"${colName}" TEXT`);
|
|
35
|
-
}
|
|
47
|
+
for (const { name, sqliteType } of zodSchemaColumns(schema)) {
|
|
48
|
+
colDefs.push(`"${name}" ${sqliteType}`);
|
|
36
49
|
}
|
|
37
50
|
if (!opts?.isInput) {
|
|
38
51
|
colDefs.push(`PRIMARY KEY (run_id, node_id, iteration)`);
|
|
39
52
|
}
|
|
40
|
-
return `CREATE TABLE IF NOT EXISTS
|
|
53
|
+
return `CREATE TABLE IF NOT EXISTS ${quoteIdentifier(tableName)} (${colDefs.join(", ")})`;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Ensures `tableName` exists with all columns implied by `schema`, adding
|
|
57
|
+
* any missing columns via ALTER TABLE. Idempotent and safe to call on every
|
|
58
|
+
* boot — fixes the case where the schema evolves but the table was created
|
|
59
|
+
* by an earlier version (CREATE TABLE IF NOT EXISTS would silently no-op).
|
|
60
|
+
*
|
|
61
|
+
* `sqlite` must be a `bun:sqlite` Database (or compatible) exposing
|
|
62
|
+
* `.run(sql)` and `.query(sql).all()`.
|
|
63
|
+
*/
|
|
64
|
+
export function syncZodTableSchema(sqlite, tableName, schema, opts) {
|
|
65
|
+
sqlite.run(zodToCreateTableSQL(tableName, schema, opts));
|
|
66
|
+
let existing;
|
|
67
|
+
const quotedTable = quoteIdentifier(tableName);
|
|
68
|
+
try {
|
|
69
|
+
existing = sqlite.query(`PRAGMA table_info(${quotedTable})`).all();
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
const have = new Set(existing.map((row) => row?.name).filter((n) => typeof n === "string"));
|
|
75
|
+
for (const { name, sqliteType } of zodSchemaColumns(schema)) {
|
|
76
|
+
if (have.has(name))
|
|
77
|
+
continue;
|
|
78
|
+
try {
|
|
79
|
+
sqlite.run(`ALTER TABLE ${quotedTable} ADD COLUMN ${quoteIdentifier(name)} ${sqliteType}`);
|
|
80
|
+
}
|
|
81
|
+
catch {
|
|
82
|
+
// Concurrent boot, or column added since the PRAGMA snapshot — ignore.
|
|
83
|
+
}
|
|
84
|
+
}
|
|
41
85
|
}
|