@smithers-orchestrator/db 0.24.2 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -5
- package/src/adapter/DB_RUN_ALLOWED_STATUSES.js +1 -0
- package/src/adapter.js +240 -67
- package/src/assertJsonPayloadWithinBounds.js +1 -1
- package/src/assertNoReservedColumns.js +38 -0
- package/src/dialect.js +2 -31
- package/src/docWatcher.js +162 -0
- package/src/frame-codec.js +5 -1
- package/src/getSmithersSchemaSignature.js +70 -0
- package/src/index.d.ts +60 -12
- package/src/index.js +3 -0
- package/src/internal-schema/index.js +1 -0
- package/src/internal-schema/smithersDocs.js +27 -0
- package/src/internal-schema/smithersScorers.js +2 -0
- package/src/internal-schema.js +1 -0
- package/src/react-output.js +3 -10
- package/src/runState/DeriveRunStateInput.ts +4 -0
- package/src/runState/ReasonBlocked.ts +4 -1
- package/src/runState/RunState.ts +1 -0
- package/src/runState/computeRunStateFromRow.js +21 -0
- package/src/runState/deriveRunState.js +39 -10
- package/src/schema-migrations.js +346 -14
- package/src/sha256Hex.js +14 -0
- package/src/sql-message-storage.js +13 -0
- package/src/zodToCreateTableSQL.js +12 -4
- package/src/zodToTable.js +20 -5
- package/src/frame-codec/index.js +0 -15
- package/src/output/index.js +0 -14
- package/src/storage/InMemoryStorage.js +0 -484
- package/src/storage/StorageService.js +0 -7
- package/src/storage/StorageServiceShape.ts +0 -122
- package/src/storage/StorageServiceTypes.ts +0 -150
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@smithers-orchestrator/db",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.25.0",
|
|
4
4
|
"description": "SQLite and Drizzle persistence adapter for Smithers workflows",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"sideEffects": false,
|
|
@@ -30,10 +30,10 @@
|
|
|
30
30
|
"drizzle-zod": "^0.8.3",
|
|
31
31
|
"effect": "^3.21.1",
|
|
32
32
|
"zod": "^4.3.6",
|
|
33
|
-
"@smithers-orchestrator/
|
|
34
|
-
"@smithers-orchestrator/
|
|
35
|
-
"@smithers-orchestrator/
|
|
36
|
-
"@smithers-orchestrator/
|
|
33
|
+
"@smithers-orchestrator/errors": "0.25.0",
|
|
34
|
+
"@smithers-orchestrator/graph": "0.25.0",
|
|
35
|
+
"@smithers-orchestrator/observability": "0.25.0",
|
|
36
|
+
"@smithers-orchestrator/scheduler": "0.25.0"
|
|
37
37
|
},
|
|
38
38
|
"devDependencies": {
|
|
39
39
|
"@electric-sql/pglite": "0.5.1",
|
package/src/adapter.js
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
|
|
13
13
|
import { getTableName } from "drizzle-orm";
|
|
14
14
|
import { getTableColumns } from "drizzle-orm/utils";
|
|
15
|
+
import { createHash } from "node:crypto";
|
|
15
16
|
import { Effect, Exit, FiberId, Metric } from "effect";
|
|
16
17
|
import { toSmithersError } from "@smithers-orchestrator/errors/toSmithersError";
|
|
17
18
|
import { getSqlMessageStorage } from "./sql-message-storage.js";
|
|
@@ -51,6 +52,9 @@ import { camelToSnake } from "./utils/camelToSnake.js";
|
|
|
51
52
|
/**
|
|
52
53
|
* @typedef {{ cacheKey: string; createdAtMs?: number; nodeId: string; outputTable: string }} CacheRowLike
|
|
53
54
|
*/
|
|
55
|
+
/**
|
|
56
|
+
* @typedef {{ path: string; kind: string; content: string; contentHash: string; updatedAtMs: number; deletedAtMs?: number | null }} DocRow
|
|
57
|
+
*/
|
|
54
58
|
|
|
55
59
|
|
|
56
60
|
export const DB_ALERT_ID_MAX_LENGTH = 256;
|
|
@@ -68,6 +72,7 @@ export const DB_ALERT_ALLOWED_STATUSES = [
|
|
|
68
72
|
"silenced",
|
|
69
73
|
];
|
|
70
74
|
const FRAME_XML_CACHE_MAX = 512;
|
|
75
|
+
const DOC_CONFLICT_KIND = "conflict";
|
|
71
76
|
export const DB_RUN_ID_MAX_LENGTH = 256;
|
|
72
77
|
export const DB_RUN_WORKFLOW_NAME_MAX_LENGTH = 256;
|
|
73
78
|
export const DB_RUN_ALLOWED_STATUSES = [
|
|
@@ -75,6 +80,7 @@ export const DB_RUN_ALLOWED_STATUSES = [
|
|
|
75
80
|
"waiting-approval",
|
|
76
81
|
"waiting-event",
|
|
77
82
|
"waiting-timer",
|
|
83
|
+
"waiting-quota",
|
|
78
84
|
"finished",
|
|
79
85
|
"failed",
|
|
80
86
|
"cancelled",
|
|
@@ -87,6 +93,71 @@ const ACTIVE_ALERT_STATUSES = new Set([
|
|
|
87
93
|
"acknowledged",
|
|
88
94
|
"silenced",
|
|
89
95
|
]);
|
|
96
|
+
/**
|
|
97
|
+
* @param {string} value
|
|
98
|
+
* @returns {string}
|
|
99
|
+
*/
|
|
100
|
+
function sha256Hex(value) {
|
|
101
|
+
return createHash("sha256").update(value).digest("hex");
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* @param {string} path
|
|
105
|
+
* @param {number} updatedAtMs
|
|
106
|
+
* @returns {string}
|
|
107
|
+
*/
|
|
108
|
+
function docConflictPath(path, updatedAtMs) {
|
|
109
|
+
const key = createHash("sha1").update(path).digest("hex").slice(0, 16);
|
|
110
|
+
return `conflicts/${key}-${updatedAtMs}.json`;
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Decide whether an incoming doc write diverges from the row already on record.
|
|
114
|
+
*
|
|
115
|
+
* The single writer (`syncDocsFromDisk`) re-reads and re-upserts a file on every
|
|
116
|
+
* watcher settle, so a plain "the content hash changed" rule would mint a marker
|
|
117
|
+
* on every ordinary sequential save and turn the conflict feature into an
|
|
118
|
+
* unbounded per-save change-log. A real conflict is a divergent write: the
|
|
119
|
+
* content differs AND the incoming write does not strictly advance past the row
|
|
120
|
+
* we already hold (its `updatedAtMs` is the same or older — a stale or
|
|
121
|
+
* out-of-order writer). A strict forward edit (`row.updatedAtMs >
|
|
122
|
+
* existing.updatedAtMs`) is the happy path and records no marker.
|
|
123
|
+
*
|
|
124
|
+
* @param {DocRow} row
|
|
125
|
+
* @param {DocRow} existing
|
|
126
|
+
* @returns {DocRow | null}
|
|
127
|
+
*/
|
|
128
|
+
function buildDocConflictRow(row, existing) {
|
|
129
|
+
if (row.kind === DOC_CONFLICT_KIND || existing.kind === DOC_CONFLICT_KIND) {
|
|
130
|
+
return null;
|
|
131
|
+
}
|
|
132
|
+
if (row.deletedAtMs != null || existing.deletedAtMs != null) {
|
|
133
|
+
return null;
|
|
134
|
+
}
|
|
135
|
+
if (row.contentHash === existing.contentHash) {
|
|
136
|
+
return null;
|
|
137
|
+
}
|
|
138
|
+
if (row.updatedAtMs > existing.updatedAtMs) {
|
|
139
|
+
return null;
|
|
140
|
+
}
|
|
141
|
+
const updatedAtMs = row.updatedAtMs;
|
|
142
|
+
const content = JSON.stringify({
|
|
143
|
+
path: row.path,
|
|
144
|
+
kind: row.kind,
|
|
145
|
+
previousHash: existing.contentHash,
|
|
146
|
+
incomingHash: row.contentHash,
|
|
147
|
+
previousUpdatedAtMs: existing.updatedAtMs,
|
|
148
|
+
incomingUpdatedAtMs: row.updatedAtMs,
|
|
149
|
+
resolution: "last-write-wins",
|
|
150
|
+
recordedAtMs: updatedAtMs,
|
|
151
|
+
}, null, 2);
|
|
152
|
+
return {
|
|
153
|
+
path: docConflictPath(row.path, updatedAtMs),
|
|
154
|
+
kind: DOC_CONFLICT_KIND,
|
|
155
|
+
content,
|
|
156
|
+
contentHash: sha256Hex(content),
|
|
157
|
+
updatedAtMs,
|
|
158
|
+
deletedAtMs: null,
|
|
159
|
+
};
|
|
160
|
+
}
|
|
90
161
|
/**
|
|
91
162
|
* @param {string} queryString
|
|
92
163
|
* @returns {string}
|
|
@@ -324,23 +395,6 @@ function validateAlertRow(row) {
|
|
|
324
395
|
function isAlertActiveStatus(status) {
|
|
325
396
|
return status !== undefined && status !== null && ACTIVE_ALERT_STATUSES.has(status);
|
|
326
397
|
}
|
|
327
|
-
/**
|
|
328
|
-
* Returns the row unchanged. Historically this helper relabeled
|
|
329
|
-
* stale-heartbeat "running" rows as "continued", which is wrong: "continued"
|
|
330
|
-
* is reserved for runs that successfully forked into a new run, and
|
|
331
|
-
* `deriveRunState` then maps "continued" → "succeeded" — so a dead workflow
|
|
332
|
-
* was being reported as a success. Heartbeat-based classification now lives
|
|
333
|
-
* exclusively in `deriveRunState`, which correctly returns "stale" or
|
|
334
|
-
* "orphaned". This shim is kept so the `listRuns` / `getRun` call sites
|
|
335
|
-
* continue to compile; remove it once they call `deriveRunState` directly.
|
|
336
|
-
*
|
|
337
|
-
* @template T
|
|
338
|
-
* @param {T} row
|
|
339
|
-
* @returns {T}
|
|
340
|
-
*/
|
|
341
|
-
function classifyRunRowStatus(row) {
|
|
342
|
-
return row;
|
|
343
|
-
}
|
|
344
398
|
/**
|
|
345
399
|
* @template A, E
|
|
346
400
|
* @param {Effect.Effect<A, E>} effect
|
|
@@ -590,10 +644,11 @@ export class SmithersDb {
|
|
|
590
644
|
}
|
|
591
645
|
}
|
|
592
646
|
/**
|
|
593
|
-
|
|
647
|
+
* @param {string} queryString
|
|
648
|
+
* @param {unknown[]} [params]
|
|
594
649
|
* @returns {RunnableEffect<unknown[], SmithersError>}
|
|
595
650
|
*/
|
|
596
|
-
rawQuery(queryString) {
|
|
651
|
+
rawQuery(queryString, params = []) {
|
|
597
652
|
const self = this;
|
|
598
653
|
return runnableEffect(Effect.gen(function* () {
|
|
599
654
|
const validatedQuery = yield* Effect.try({
|
|
@@ -605,11 +660,11 @@ export class SmithersDb {
|
|
|
605
660
|
});
|
|
606
661
|
return yield* self.read(`raw query ${validatedQuery.slice(0, 20)}`, () => {
|
|
607
662
|
if (self.internalStorage.dialect === POSTGRES) {
|
|
608
|
-
return self.internalStorage.queryAllRaw(validatedQuery);
|
|
663
|
+
return self.internalStorage.queryAllRaw(validatedQuery, params);
|
|
609
664
|
}
|
|
610
665
|
const client = self.db.session.client;
|
|
611
666
|
const stmt = client.query(validatedQuery);
|
|
612
|
-
return Promise.resolve(stmt.all());
|
|
667
|
+
return Promise.resolve(stmt.all(...params));
|
|
613
668
|
});
|
|
614
669
|
}));
|
|
615
670
|
}
|
|
@@ -899,11 +954,10 @@ export class SmithersDb {
|
|
|
899
954
|
*/
|
|
900
955
|
getRun(runId) {
|
|
901
956
|
return this.read(`get run ${runId}`, async () => {
|
|
902
|
-
|
|
957
|
+
return this.internalStorage.queryOne(`SELECT *
|
|
903
958
|
FROM _smithers_runs
|
|
904
959
|
WHERE run_id = ?
|
|
905
960
|
LIMIT 1`, [runId]);
|
|
906
|
-
return row ? classifyRunRowStatus(row) : undefined;
|
|
907
961
|
});
|
|
908
962
|
}
|
|
909
963
|
/**
|
|
@@ -962,7 +1016,7 @@ export class SmithersDb {
|
|
|
962
1016
|
${whereSql}
|
|
963
1017
|
ORDER BY created_at_ms DESC
|
|
964
1018
|
LIMIT ?`, [...params, limit]);
|
|
965
|
-
return rows
|
|
1019
|
+
return rows;
|
|
966
1020
|
});
|
|
967
1021
|
}
|
|
968
1022
|
/**
|
|
@@ -1871,17 +1925,30 @@ export class SmithersDb {
|
|
|
1871
1925
|
if (typeof client.exec !== "function" ||
|
|
1872
1926
|
typeof client.query !== "function" ||
|
|
1873
1927
|
typeof client.run !== "function") {
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1928
|
+
// Non-bun:sqlite (Postgres/pglite) fallback. Serialize the
|
|
1929
|
+
// read-MAX-then-insert under the shared transaction turn so two
|
|
1930
|
+
// concurrent allocations can't both read the same lastSeq and
|
|
1931
|
+
// collide on the (run_id, seq) primary key — insertIgnore would
|
|
1932
|
+
// otherwise silently drop the loser, losing a signal.
|
|
1933
|
+
const releaseTurn = yield* self.acquireTransactionTurn();
|
|
1934
|
+
return yield* Effect.gen(function* () {
|
|
1935
|
+
const lastSeq = (yield* Effect.tryPromise({
|
|
1936
|
+
try: () => self.internalStorage.getLastSignalSeq(row.runId),
|
|
1937
|
+
catch: (cause) => toSmithersError(cause, "get fallback last signal seq"),
|
|
1938
|
+
})) ?? -1;
|
|
1939
|
+
const seq = lastSeq + 1;
|
|
1940
|
+
yield* Effect.tryPromise({
|
|
1941
|
+
try: () => self.internalStorage.insertIgnore("_smithers_signals", {
|
|
1942
|
+
...row,
|
|
1943
|
+
receivedBy: row.receivedBy ?? null,
|
|
1944
|
+
seq,
|
|
1945
|
+
}),
|
|
1946
|
+
catch: (cause) => toSmithersError(cause, "insert fallback signal row"),
|
|
1947
|
+
});
|
|
1948
|
+
return seq;
|
|
1949
|
+
}).pipe(Effect.ensuring(Effect.sync(() => {
|
|
1950
|
+
releaseTurn();
|
|
1951
|
+
})));
|
|
1885
1952
|
}
|
|
1886
1953
|
const releaseTurn = yield* self.acquireTransactionTurn();
|
|
1887
1954
|
return yield* Effect.try({
|
|
@@ -2075,6 +2142,25 @@ export class SmithersDb {
|
|
|
2075
2142
|
) sub WHERE rn <= ?)`, [runId, runId, keep]));
|
|
2076
2143
|
}
|
|
2077
2144
|
/**
|
|
2145
|
+
* Upsert a DB-backed markdown artifact. When an existing live row has a
|
|
2146
|
+
* different content hash, last-write-wins still applies, but a conflict marker
|
|
2147
|
+
* row is inserted so the UI can surface the mismatch.
|
|
2148
|
+
* @param {DocRow} row
|
|
2149
|
+
* @returns {RunnableEffect<void, SmithersError>}
|
|
2150
|
+
*/
|
|
2151
|
+
upsertDocRow(row) {
|
|
2152
|
+
return this.write(`upsert doc ${row.path}`, async () => {
|
|
2153
|
+
const existing = /** @type {DocRow | undefined} */ (await this.internalStorage.queryOne(`SELECT *
|
|
2154
|
+
FROM _smithers_docs
|
|
2155
|
+
WHERE path = ?`, [row.path]));
|
|
2156
|
+
const conflict = existing ? buildDocConflictRow(row, existing) : null;
|
|
2157
|
+
if (conflict) {
|
|
2158
|
+
await this.internalStorage.upsert("_smithers_docs", conflict, ["path"], ["kind", "content", "contentHash", "updatedAtMs", "deletedAtMs"]);
|
|
2159
|
+
}
|
|
2160
|
+
await this.internalStorage.upsert("_smithers_docs", row, ["path"], ["kind", "content", "contentHash", "updatedAtMs", "deletedAtMs"]);
|
|
2161
|
+
});
|
|
2162
|
+
}
|
|
2163
|
+
/**
|
|
2078
2164
|
* @param {Record<string, unknown>} row
|
|
2079
2165
|
* @returns {RunnableEffect<void, SmithersError>}
|
|
2080
2166
|
*/
|
|
@@ -2104,13 +2190,28 @@ export class SmithersDb {
|
|
|
2104
2190
|
if (typeof client.exec !== "function" ||
|
|
2105
2191
|
typeof client.query !== "function" ||
|
|
2106
2192
|
typeof client.run !== "function") {
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2193
|
+
// Non-bun:sqlite (Postgres/pglite) fallback. Serialize the
|
|
2194
|
+
// read-MAX-then-insert under the shared transaction turn — the
|
|
2195
|
+
// same primitive the bun:sqlite path below relies on — so two
|
|
2196
|
+
// concurrent allocations can't both read the same lastSeq and
|
|
2197
|
+
// collide on the (run_id, seq) primary key. Without the turn,
|
|
2198
|
+
// insertIgnore would silently drop the loser, losing an event
|
|
2199
|
+
// from the durable log that replay/live-stream depend on.
|
|
2200
|
+
const releaseTurn = yield* self.acquireTransactionTurn();
|
|
2201
|
+
return yield* Effect.gen(function* () {
|
|
2202
|
+
const lastSeq = (yield* Effect.tryPromise({
|
|
2203
|
+
try: () => self.internalStorage.getLastEventSeq(row.runId),
|
|
2204
|
+
catch: (cause) => toSmithersError(cause, "get fallback last event seq"),
|
|
2205
|
+
})) ?? -1;
|
|
2206
|
+
const seq = lastSeq + 1;
|
|
2207
|
+
yield* Effect.tryPromise({
|
|
2208
|
+
try: () => self.internalStorage.insertIgnore("_smithers_events", { ...row, seq }),
|
|
2209
|
+
catch: (cause) => toSmithersError(cause, "insert fallback event row"),
|
|
2210
|
+
});
|
|
2211
|
+
return seq;
|
|
2212
|
+
}).pipe(Effect.ensuring(Effect.sync(() => {
|
|
2213
|
+
releaseTurn();
|
|
2214
|
+
})));
|
|
2114
2215
|
}
|
|
2115
2216
|
const releaseTurn = yield* self.acquireTransactionTurn();
|
|
2116
2217
|
return yield* Effect.try({
|
|
@@ -2153,31 +2254,6 @@ export class SmithersDb {
|
|
|
2153
2254
|
/**
|
|
2154
2255
|
* @param {string} runId
|
|
2155
2256
|
* @param {EventHistoryQuery} [query]
|
|
2156
|
-
* @returns {{ whereSql: string; params: Array<string | number> }}
|
|
2157
|
-
*/
|
|
2158
|
-
buildEventHistoryWhere(runId, query = {}) {
|
|
2159
|
-
const clauses = ["run_id = ?", "seq > ?"];
|
|
2160
|
-
const params = [runId, query.afterSeq ?? -1];
|
|
2161
|
-
if (typeof query.sinceTimestampMs === "number") {
|
|
2162
|
-
clauses.push("timestamp_ms >= ?");
|
|
2163
|
-
params.push(query.sinceTimestampMs);
|
|
2164
|
-
}
|
|
2165
|
-
if (query.types && query.types.length > 0) {
|
|
2166
|
-
clauses.push(`type IN (${query.types.map(() => "?").join(", ")})`);
|
|
2167
|
-
params.push(...query.types);
|
|
2168
|
-
}
|
|
2169
|
-
if (query.nodeId) {
|
|
2170
|
-
clauses.push("json_extract(payload_json, '$.nodeId') = ?");
|
|
2171
|
-
params.push(query.nodeId);
|
|
2172
|
-
}
|
|
2173
|
-
return {
|
|
2174
|
-
whereSql: clauses.join(" AND "),
|
|
2175
|
-
params,
|
|
2176
|
-
};
|
|
2177
|
-
}
|
|
2178
|
-
/**
|
|
2179
|
-
* @param {string} runId
|
|
2180
|
-
* @param {EventHistoryQuery} [query]
|
|
2181
2257
|
* @returns {RunnableEffect<Array<Record<string, unknown>>, SmithersError>}
|
|
2182
2258
|
*/
|
|
2183
2259
|
listEventHistory(runId, query = {}) {
|
|
@@ -2546,6 +2622,103 @@ export class SmithersDb {
|
|
|
2546
2622
|
return this.write(`delete cron ${cronId}`, () => this.internalStorage.deleteWhere("_smithers_cron", "cron_id = ?", [cronId]));
|
|
2547
2623
|
}
|
|
2548
2624
|
// ---------------------------------------------------------------------------
|
|
2625
|
+
// Memory facts
|
|
2626
|
+
// ---------------------------------------------------------------------------
|
|
2627
|
+
/**
|
|
2628
|
+
* List cross-run memory facts, optionally scoped to a namespace. Reads the
|
|
2629
|
+
* `_smithers_memory_facts` table written by `@smithers-orchestrator/memory`'s
|
|
2630
|
+
* MemoryStore (`setFact`) — the SAME table the `smithers memory list` CLI reads
|
|
2631
|
+
* — so a fact set by any run/workflow surfaces here. Columns are snake→camel
|
|
2632
|
+
* cased by the storage layer (`value_json → valueJson`, etc.). A null/undefined
|
|
2633
|
+
* namespace returns every namespace's facts; ordering is stable (namespace, key)
|
|
2634
|
+
* so the gateway's `listMemoryFacts` RPC returns a deterministic list.
|
|
2635
|
+
* @param {string | null} [namespace]
|
|
2636
|
+
* @returns {RunnableEffect<Array<Record<string, unknown>>, SmithersError>}
|
|
2637
|
+
*/
|
|
2638
|
+
listMemoryFacts(namespace = null) {
|
|
2639
|
+
const ns = namespace ?? null;
|
|
2640
|
+
return this.read("list memory facts", () => this.internalStorage.queryAll(`SELECT namespace, key, value_json, schema_sig, created_at_ms, updated_at_ms, ttl_ms
|
|
2641
|
+
FROM _smithers_memory_facts
|
|
2642
|
+
WHERE (? IS NULL OR namespace = ?)
|
|
2643
|
+
ORDER BY namespace, key`, [ns, ns]));
|
|
2644
|
+
}
|
|
2645
|
+
// ---------------------------------------------------------------------------
|
|
2646
|
+
// Docs (tickets / plans / specs / proposals)
|
|
2647
|
+
// ---------------------------------------------------------------------------
|
|
2648
|
+
/**
|
|
2649
|
+
* List LIVE docs from `_smithers_docs` (tombstones — `deleted_at_ms IS NOT
|
|
2650
|
+
* NULL` — are NEVER returned), optionally scoped to one `kind`. Backs the
|
|
2651
|
+
* gateway's `listTickets` RPC and the file-watcher's reconcile read. Columns
|
|
2652
|
+
* are snake→camel cased by the storage layer (`content_hash → contentHash`,
|
|
2653
|
+
* etc.); newest-updated first so the surface shows recent edits on top.
|
|
2654
|
+
* @param {string | null | { kind?: string; includeDeleted?: boolean; updatedAfterMs?: number; limit?: number }} [arg]
|
|
2655
|
+
* Either a positional `kind` filter (string) or an options object.
|
|
2656
|
+
* @returns {RunnableEffect<Array<Record<string, unknown>>, SmithersError>}
|
|
2657
|
+
*/
|
|
2658
|
+
listDocs(arg = null) {
|
|
2659
|
+
const options = typeof arg === "string" ? { kind: arg } : (arg ?? {});
|
|
2660
|
+
const clauses = [];
|
|
2661
|
+
const params = [];
|
|
2662
|
+
if (!options.includeDeleted) {
|
|
2663
|
+
clauses.push("deleted_at_ms IS NULL");
|
|
2664
|
+
}
|
|
2665
|
+
if (options.kind) {
|
|
2666
|
+
clauses.push("kind = ?");
|
|
2667
|
+
params.push(options.kind);
|
|
2668
|
+
}
|
|
2669
|
+
if (typeof options.updatedAfterMs === "number" && Number.isFinite(options.updatedAfterMs)) {
|
|
2670
|
+
clauses.push("updated_at_ms > ?");
|
|
2671
|
+
params.push(Math.floor(options.updatedAfterMs));
|
|
2672
|
+
}
|
|
2673
|
+
const whereSql = clauses.length > 0 ? `WHERE ${clauses.join(" AND ")}` : "";
|
|
2674
|
+
const limit = Math.max(1, Math.min(10_000, Math.floor(options.limit ?? 4_096)));
|
|
2675
|
+
// Newest-updated first so when the doc set exceeds the cap the LIMIT keeps
|
|
2676
|
+
// the most recent edits (the gateway re-sorts before serving).
|
|
2677
|
+
return this.read("list docs", () => this.internalStorage.queryAll(`SELECT path, kind, content, content_hash, status, updated_at_ms, deleted_at_ms
|
|
2678
|
+
FROM _smithers_docs
|
|
2679
|
+
${whereSql}
|
|
2680
|
+
ORDER BY updated_at_ms DESC, path ASC
|
|
2681
|
+
LIMIT ?`, [...params, limit]));
|
|
2682
|
+
}
|
|
2683
|
+
/**
|
|
2684
|
+
* Read a single doc row by path (INCLUDING a tombstone, so the watcher's
|
|
2685
|
+
* last-write-wins can compare against a soft-deleted row). Returns `undefined`
|
|
2686
|
+
* when the path was never written.
|
|
2687
|
+
* @param {string} path
|
|
2688
|
+
* @returns {RunnableEffect<Record<string, unknown> | undefined, SmithersError>}
|
|
2689
|
+
*/
|
|
2690
|
+
getDoc(path, options = {}) {
|
|
2691
|
+
// Returns a tombstone by default (the watcher's last-write-wins compares
|
|
2692
|
+
// against soft-deleted rows). Pass `includeDeleted: false` to hide them.
|
|
2693
|
+
const filter = options.includeDeleted === false ? " AND deleted_at_ms IS NULL" : "";
|
|
2694
|
+
return this.read(`get doc ${path}`, () => this.internalStorage.queryOne(`SELECT path, kind, content, content_hash, status, updated_at_ms, deleted_at_ms
|
|
2695
|
+
FROM _smithers_docs
|
|
2696
|
+
WHERE path = ?${filter}`, [path]));
|
|
2697
|
+
}
|
|
2698
|
+
/**
|
|
2699
|
+
* Upsert a doc row (insert-or-replace by `path`). The caller supplies the
|
|
2700
|
+
* already-computed `contentHash` (`sha256(content)`) and `updatedAtMs` so the
|
|
2701
|
+
* RPC handler and the file-watcher hash/stamp identically. Writing a row with
|
|
2702
|
+
* `deletedAtMs: null` REVIVES a previously soft-deleted path (a re-create or a
|
|
2703
|
+
* fresh file write), which is the intended last-write-wins behaviour.
|
|
2704
|
+
* @param {Record<string, unknown>} row
|
|
2705
|
+
* @returns {RunnableEffect<void, SmithersError>}
|
|
2706
|
+
*/
|
|
2707
|
+
upsertDoc(row) {
|
|
2708
|
+
return this.write(`upsert doc ${row.path}`, () => this.internalStorage.upsert("_smithers_docs", row, ["path"], ["kind", "content", "contentHash", "status", "updatedAtMs", "deletedAtMs"]));
|
|
2709
|
+
}
|
|
2710
|
+
/**
|
|
2711
|
+
* Soft-delete a doc by stamping `deleted_at_ms` (a tombstone) rather than
|
|
2712
|
+
* removing the row, so `listTickets` hides it while the watcher can still see
|
|
2713
|
+
* it survived. `updated_at_ms` is bumped so the change orders correctly.
|
|
2714
|
+
* @param {string} path
|
|
2715
|
+
* @param {number} deletedAtMs
|
|
2716
|
+
* @returns {RunnableEffect<void, SmithersError>}
|
|
2717
|
+
*/
|
|
2718
|
+
softDeleteDoc(path, deletedAtMs) {
|
|
2719
|
+
return this.write(`soft delete doc ${path}`, () => this.internalStorage.updateWhere("_smithers_docs", { deletedAtMs, updatedAtMs: deletedAtMs }, "path = ?", [path]));
|
|
2720
|
+
}
|
|
2721
|
+
// ---------------------------------------------------------------------------
|
|
2549
2722
|
// Scorer results
|
|
2550
2723
|
// ---------------------------------------------------------------------------
|
|
2551
2724
|
/**
|
|
@@ -17,7 +17,7 @@ function validateJsonValue(field, value, bounds, path, seen) {
|
|
|
17
17
|
if (typeof value === "string") {
|
|
18
18
|
if (typeof bounds.maxStringLength === "number" &&
|
|
19
19
|
value.length > bounds.maxStringLength) {
|
|
20
|
-
throw new SmithersError("INVALID_INPUT", `${field} contains a string exceeding ${bounds.maxStringLength} characters.`, {
|
|
20
|
+
throw new SmithersError("INVALID_INPUT", `${field} contains a string exceeding ${bounds.maxStringLength} characters (at ${path}: ${value.length} characters).`, {
|
|
21
21
|
field,
|
|
22
22
|
path,
|
|
23
23
|
maxLength: bounds.maxStringLength,
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { SmithersError } from "@smithers-orchestrator/errors/SmithersError";
|
|
2
|
+
import { camelToSnake } from "./utils/camelToSnake.js";
|
|
3
|
+
|
|
4
|
+
const OUTPUT_RESERVED = new Set(["run_id", "node_id", "iteration"]);
|
|
5
|
+
const INPUT_RESERVED = new Set(["run_id"]);
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Throw a clear error if a user schema field collides with a smithers internal
|
|
9
|
+
* key column. Output tables reserve `run_id`/`node_id`/`iteration`; input tables
|
|
10
|
+
* reserve only `run_id`. Without this guard a colliding field either crashes DDL
|
|
11
|
+
* with a raw `duplicate column name` (the SQL path) or silently overwrites the
|
|
12
|
+
* internal key column and corrupts the composite primary key (the drizzle path),
|
|
13
|
+
* with no diagnostic naming the offending field.
|
|
14
|
+
*
|
|
15
|
+
* @param {{ shape?: Record<string, unknown> }} schema
|
|
16
|
+
* @param {string} [tableName]
|
|
17
|
+
* @param {{ isInput?: boolean }} [opts]
|
|
18
|
+
* @returns {void}
|
|
19
|
+
*/
|
|
20
|
+
export function assertNoReservedColumns(schema, tableName, opts) {
|
|
21
|
+
const shape = schema?.shape;
|
|
22
|
+
if (!shape || typeof shape !== "object")
|
|
23
|
+
return;
|
|
24
|
+
const reserved = opts?.isInput ? INPUT_RESERVED : OUTPUT_RESERVED;
|
|
25
|
+
// camelToSnake handles both camelCase and already-snake keys
|
|
26
|
+
// (camelToSnake("node_id") === "node_id"), so one snake-set check catches
|
|
27
|
+
// `nodeId` AND a literal `node_id`.
|
|
28
|
+
const offenders = Object.keys(shape).filter((key) => reserved.has(camelToSnake(key)));
|
|
29
|
+
if (offenders.length === 0)
|
|
30
|
+
return;
|
|
31
|
+
const where = tableName ? ` for "${tableName}"` : "";
|
|
32
|
+
const reservedList = opts?.isInput
|
|
33
|
+
? "run_id (camelCase runId)"
|
|
34
|
+
: "run_id, node_id and iteration (camelCase runId/nodeId/iteration)";
|
|
35
|
+
throw new SmithersError("INVALID_INPUT", `${opts?.isInput ? "Input" : "Output"} schema${where} uses reserved field name(s): ${offenders.join(", ")}. ` +
|
|
36
|
+
`smithers persists every ${opts?.isInput ? "input" : "node output"} with internal column(s) ${reservedList}. ` +
|
|
37
|
+
`Rename the conflicting field(s) - e.g. nodeId -> targetNodeId, runId -> sourceRunId, iteration -> attempt.`, { table: tableName, offenders });
|
|
38
|
+
}
|
package/src/dialect.js
CHANGED
|
@@ -24,14 +24,6 @@
|
|
|
24
24
|
export const SQLITE = /** @type {const} */ ("sqlite");
|
|
25
25
|
export const POSTGRES = /** @type {const} */ ("postgres");
|
|
26
26
|
|
|
27
|
-
/**
|
|
28
|
-
* @param {unknown} value
|
|
29
|
-
* @returns {value is Dialect}
|
|
30
|
-
*/
|
|
31
|
-
export function isDialect(value) {
|
|
32
|
-
return value === SQLITE || value === POSTGRES;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
27
|
/**
|
|
36
28
|
* @param {string} identifier
|
|
37
29
|
* @returns {string}
|
|
@@ -131,8 +123,8 @@ export function translatePlaceholders(dialect, sql) {
|
|
|
131
123
|
|
|
132
124
|
/**
|
|
133
125
|
* Map a single SQLite column type keyword (`INTEGER`, `REAL`, `BLOB`, `TEXT`) to
|
|
134
|
-
* the dialect equivalent. Used by the Zod→DDL generator
|
|
135
|
-
* `INTEGER`
|
|
126
|
+
* the dialect equivalent. Used by the Zod→DDL generator for schema-derived
|
|
127
|
+
* columns such as `INTEGER`, `REAL`, and `TEXT`.
|
|
136
128
|
*
|
|
137
129
|
* @param {Dialect} dialect
|
|
138
130
|
* @param {string} sqliteType
|
|
@@ -177,27 +169,6 @@ export function translateDdl(dialect, ddl) {
|
|
|
177
169
|
.replace(/\bINTEGER\b/gi, "BIGINT");
|
|
178
170
|
}
|
|
179
171
|
|
|
180
|
-
/**
|
|
181
|
-
* SQL that lists the column names of a table, normalized to rows shaped
|
|
182
|
-
* `{ name: string }`.
|
|
183
|
-
*
|
|
184
|
-
* @param {Dialect} dialect
|
|
185
|
-
* @param {string} table
|
|
186
|
-
* @returns {{ sql: string; params: ReadonlyArray<string> }}
|
|
187
|
-
*/
|
|
188
|
-
export function tableColumnsSql(dialect, table) {
|
|
189
|
-
if (dialect === POSTGRES) {
|
|
190
|
-
return {
|
|
191
|
-
sql: "SELECT column_name AS name FROM information_schema.columns WHERE table_name = ? AND table_schema = current_schema()",
|
|
192
|
-
params: [table],
|
|
193
|
-
};
|
|
194
|
-
}
|
|
195
|
-
return {
|
|
196
|
-
sql: `PRAGMA table_info(${quoteIdentifier(table)})`,
|
|
197
|
-
params: [],
|
|
198
|
-
};
|
|
199
|
-
}
|
|
200
|
-
|
|
201
172
|
/**
|
|
202
173
|
* The statement that begins a write transaction. SQLite takes the write lock
|
|
203
174
|
* eagerly with `BEGIN IMMEDIATE`; PostgreSQL uses a plain `BEGIN`.
|