@staix/agent-hub 0.6.4 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/README.md +16 -15
- package/docs/operations.md +217 -0
- package/docs/quickstart.md +121 -0
- package/docs/security.md +57 -0
- package/docs/smoke.md +525 -0
- package/docs/specs/2026-09-19-agent-hub-design.md +675 -0
- package/docs/specs/2026-09-19-multi-project-design.md +95 -0
- package/docs/specs/2026-09-20-durable-delivery-design.md +67 -0
- package/docs/specs/2026-09-20-pi-local-models-design.md +40 -0
- package/docs/specs/2026-09-20-upgrade-recovery-design.md +151 -0
- package/package.json +2 -1
- package/plugins/agent-hub/.claude-plugin/plugin.json +1 -1
- package/plugins/agent-hub/server.js +31 -10
- package/src/adapters/acp.ts +40 -7
- package/src/adapters/claude-channel.ts +22 -5
- package/src/adapters/codex-appserver.ts +50 -7
- package/src/adapters/local-worker.ts +20 -3
- package/src/adapters/pi.ts +33 -7
- package/src/cli/main.ts +49 -0
- package/src/cli/status-lines.ts +4 -1
- package/src/cli/terminal-recovery.ts +41 -1
- package/src/cli/upgrade-runtime.ts +8 -8
- package/src/hub/bus.ts +290 -38
- package/src/hub/control-client.ts +5 -3
- package/src/hub/daemon.ts +75 -19
- package/src/hub/delivery-journal.ts +830 -0
- package/src/hub/manager.ts +6 -4
- package/src/hub/peers.ts +16 -3
- package/src/hub/restart.ts +2 -0
- package/src/ui/index.html +7 -3
|
@@ -0,0 +1,830 @@
|
|
|
1
|
+
import { chmodSync, existsSync, mkdirSync, readFileSync } from "node:fs";
|
|
2
|
+
import { dirname } from "node:path";
|
|
3
|
+
import { Database } from "bun:sqlite";
|
|
4
|
+
import type { BusSnapshot } from "./bus.ts";
|
|
5
|
+
import type { Envelope } from "./envelope.ts";
|
|
6
|
+
|
|
7
|
+
export type JournalDeliveryState =
|
|
8
|
+
| "queued"
|
|
9
|
+
| "dispatching"
|
|
10
|
+
| "accepted"
|
|
11
|
+
| "completed"
|
|
12
|
+
| "needs_review"
|
|
13
|
+
| "failed"
|
|
14
|
+
| "discarded";
|
|
15
|
+
export type JournalResolution = "completed" | "retry" | "discard";
|
|
16
|
+
const SCHEMA_VERSION = 1,
|
|
17
|
+
TERMINAL_CAP = 2048,
|
|
18
|
+
MAX_ROWS = 100_000,
|
|
19
|
+
MAX_JSON = 20 * 1024 * 1024;
|
|
20
|
+
const STATES = new Set<JournalDeliveryState>([
|
|
21
|
+
"queued",
|
|
22
|
+
"dispatching",
|
|
23
|
+
"accepted",
|
|
24
|
+
"completed",
|
|
25
|
+
"needs_review",
|
|
26
|
+
"failed",
|
|
27
|
+
"discarded",
|
|
28
|
+
]);
|
|
29
|
+
const ACTIONS = new Set<JournalResolution>(["completed", "retry", "discard"]);
|
|
30
|
+
|
|
31
|
+
export interface JournalDelivery {
|
|
32
|
+
id: string;
|
|
33
|
+
peer: string;
|
|
34
|
+
state: JournalDeliveryState;
|
|
35
|
+
revision: number;
|
|
36
|
+
createdAt: number;
|
|
37
|
+
updatedAt: number;
|
|
38
|
+
originals: Envelope[];
|
|
39
|
+
out: Envelope[];
|
|
40
|
+
reason?: string;
|
|
41
|
+
attempt?: number;
|
|
42
|
+
previousId?: string;
|
|
43
|
+
}
|
|
44
|
+
export interface DeliveryJournalOptions {
|
|
45
|
+
file: string;
|
|
46
|
+
projectRoot: string;
|
|
47
|
+
projectId: string;
|
|
48
|
+
instanceId: string;
|
|
49
|
+
operationId?: string;
|
|
50
|
+
}
|
|
51
|
+
export interface JournalSnapshot {
|
|
52
|
+
schemaVersion: number;
|
|
53
|
+
revision: number;
|
|
54
|
+
instanceId: string;
|
|
55
|
+
bus: BusSnapshot;
|
|
56
|
+
deliveries: JournalDelivery[];
|
|
57
|
+
manualPaused: string[];
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function fail(message: string): never {
|
|
61
|
+
throw new Error(`invalid delivery journal: ${message}`);
|
|
62
|
+
}
|
|
63
|
+
function obj(value: unknown, name: string): Record<string, unknown> {
|
|
64
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
65
|
+
fail(`${name} must be an object`);
|
|
66
|
+
return value as Record<string, unknown>;
|
|
67
|
+
}
|
|
68
|
+
function text(value: unknown, name: string, max = 4096): string {
|
|
69
|
+
if (typeof value !== "string" || value.length < 1 || value.length > max)
|
|
70
|
+
fail(`${name} must be a non-empty string`);
|
|
71
|
+
return value;
|
|
72
|
+
}
|
|
73
|
+
function integer(
|
|
74
|
+
value: unknown,
|
|
75
|
+
name: string,
|
|
76
|
+
min = 0,
|
|
77
|
+
max = Number.MAX_SAFE_INTEGER,
|
|
78
|
+
): number {
|
|
79
|
+
if (
|
|
80
|
+
!Number.isSafeInteger(value) ||
|
|
81
|
+
(value as number) < min ||
|
|
82
|
+
(value as number) > max
|
|
83
|
+
)
|
|
84
|
+
fail(`${name} is out of range`);
|
|
85
|
+
return value as number;
|
|
86
|
+
}
|
|
87
|
+
function list(value: unknown, name: string, max = MAX_ROWS): unknown[] {
|
|
88
|
+
if (!Array.isArray(value) || value.length > max)
|
|
89
|
+
fail(`${name} must be an array`);
|
|
90
|
+
return value;
|
|
91
|
+
}
|
|
92
|
+
function checkedJSON(value: unknown, name: string): unknown {
|
|
93
|
+
let encoded: string;
|
|
94
|
+
try {
|
|
95
|
+
encoded = JSON.stringify(value);
|
|
96
|
+
} catch {
|
|
97
|
+
fail(`${name} is not serializable`);
|
|
98
|
+
}
|
|
99
|
+
if (encoded.length > MAX_JSON) fail(`${name} is too large`);
|
|
100
|
+
return value;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function validateEnvelope(value: unknown, name: string, seen = false): Envelope {
|
|
104
|
+
const e = obj(value, name);
|
|
105
|
+
text(e.id, `${name}.id`, 256);
|
|
106
|
+
text(e.trace, `${name}.trace`, 256);
|
|
107
|
+
text(e.from, `${name}.from`, 256);
|
|
108
|
+
integer(e.hop, `${name}.hop`, 0, seen ? Number.MAX_SAFE_INTEGER : 3);
|
|
109
|
+
if (typeof e.body !== "string" || e.body.length > 1_000_000) fail(`${name}.body is invalid`);
|
|
110
|
+
integer(e.ts, `${name}.ts`);
|
|
111
|
+
if (
|
|
112
|
+
!new Set(["chat", "task", "review", "status", "budget", "presence"]).has(
|
|
113
|
+
e.kind as string,
|
|
114
|
+
)
|
|
115
|
+
)
|
|
116
|
+
fail(`${name}.kind is invalid`);
|
|
117
|
+
if (!new Set(["important", "status", "fyi"]).has(e.priority as string))
|
|
118
|
+
fail(`${name}.priority is invalid`);
|
|
119
|
+
if (e.to !== undefined)
|
|
120
|
+
for (const p of list(e.to, `${name}.to`, 256)) text(p, `${name}.to[]`, 256);
|
|
121
|
+
if (e.private !== undefined && typeof e.private !== "boolean")
|
|
122
|
+
fail(`${name}.private is invalid`);
|
|
123
|
+
if (e.refs !== undefined) {
|
|
124
|
+
const refs = obj(e.refs, `${name}.refs`);
|
|
125
|
+
for (const k of ["repo", "branch", "commit", "task"])
|
|
126
|
+
if (refs[k] !== undefined) text(refs[k], `${name}.refs.${k}`);
|
|
127
|
+
if (refs.paths !== undefined)
|
|
128
|
+
for (const p of list(refs.paths, `${name}.refs.paths`, 256))
|
|
129
|
+
text(p, `${name}.refs.paths[]`);
|
|
130
|
+
}
|
|
131
|
+
return e as unknown as Envelope;
|
|
132
|
+
}
|
|
133
|
+
function validateBus(value: unknown, name = "bus"): BusSnapshot {
|
|
134
|
+
const b = obj(value, name);
|
|
135
|
+
if (b.schemaVersion !== 1) fail(`${name}.schemaVersion is unsupported`);
|
|
136
|
+
const queues = obj(b.queues, `${name}.queues`);
|
|
137
|
+
if (Object.keys(queues).length > 4096) fail(`${name}.queues is too large`);
|
|
138
|
+
for (const [peer, items] of Object.entries(queues)) {
|
|
139
|
+
text(peer, `${name}.queues key`, 256);
|
|
140
|
+
list(items, `${name}.queues.${peer}`, 2048).forEach((e, i) =>
|
|
141
|
+
validateEnvelope(e, `${name}.queues.${peer}[${i}]`),
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
const prefaces = obj(b.prefaces, `${name}.prefaces`);
|
|
145
|
+
for (const [peer, e] of Object.entries(prefaces)) {
|
|
146
|
+
text(peer, `${name}.prefaces key`, 256);
|
|
147
|
+
validateEnvelope(e, `${name}.prefaces.${peer}`);
|
|
148
|
+
}
|
|
149
|
+
list(b.seen, `${name}.seen`, 2048).forEach((e, i) =>
|
|
150
|
+
validateEnvelope(e, `${name}.seen[${i}]`, true),
|
|
151
|
+
);
|
|
152
|
+
const attempts = obj(b.attempts, `${name}.attempts`);
|
|
153
|
+
for (const [k, n] of Object.entries(attempts)) {
|
|
154
|
+
text(k, `${name}.attempts key`);
|
|
155
|
+
integer(n, `${name}.attempts.${k}`, 1, 100);
|
|
156
|
+
}
|
|
157
|
+
list(b.withdrawn, `${name}.withdrawn`, 100_000).forEach((id) =>
|
|
158
|
+
text(id, `${name}.withdrawn[]`, 256),
|
|
159
|
+
);
|
|
160
|
+
if (b.manualPaused !== undefined)
|
|
161
|
+
list(b.manualPaused, `${name}.manualPaused`, 4096).forEach((id) =>
|
|
162
|
+
text(id, `${name}.manualPaused[]`, 256),
|
|
163
|
+
);
|
|
164
|
+
if (b.journal !== undefined) fail("nested journal snapshot is invalid");
|
|
165
|
+
return b as unknown as BusSnapshot;
|
|
166
|
+
}
|
|
167
|
+
function validateDelivery(value: unknown, name = "delivery"): JournalDelivery {
|
|
168
|
+
const d = obj(value, name);
|
|
169
|
+
text(d.id, `${name}.id`, 256);
|
|
170
|
+
text(d.peer, `${name}.peer`, 256);
|
|
171
|
+
if (!STATES.has(d.state as JournalDeliveryState))
|
|
172
|
+
fail(`${name}.state is invalid`);
|
|
173
|
+
integer(d.revision, `${name}.revision`);
|
|
174
|
+
integer(d.createdAt, `${name}.createdAt`);
|
|
175
|
+
integer(d.updatedAt, `${name}.updatedAt`);
|
|
176
|
+
const originals = list(d.originals, `${name}.originals`, 2048).map((e, i) =>
|
|
177
|
+
validateEnvelope(e, `${name}.originals[${i}]`),
|
|
178
|
+
);
|
|
179
|
+
const out = list(d.out, `${name}.out`, 2048).map((e, i) =>
|
|
180
|
+
validateEnvelope(e, `${name}.out[${i}]`),
|
|
181
|
+
);
|
|
182
|
+
if (d.reason !== undefined) text(d.reason, `${name}.reason`, 1_000_000);
|
|
183
|
+
if (d.attempt !== undefined) integer(d.attempt, `${name}.attempt`, 0, 100);
|
|
184
|
+
if (d.previousId !== undefined) text(d.previousId, `${name}.previousId`, 256);
|
|
185
|
+
return {
|
|
186
|
+
id: d.id as string,
|
|
187
|
+
peer: d.peer as string,
|
|
188
|
+
state: d.state as JournalDeliveryState,
|
|
189
|
+
revision: d.revision as number,
|
|
190
|
+
createdAt: d.createdAt as number,
|
|
191
|
+
updatedAt: d.updatedAt as number,
|
|
192
|
+
originals,
|
|
193
|
+
out,
|
|
194
|
+
...(d.reason === undefined ? {} : { reason: d.reason as string }),
|
|
195
|
+
...(d.attempt === undefined ? {} : { attempt: d.attempt as number }),
|
|
196
|
+
...(d.previousId === undefined
|
|
197
|
+
? {}
|
|
198
|
+
: { previousId: d.previousId as string }),
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
/** Validate an externally supplied restart journal before it can affect a database. */
|
|
202
|
+
export function validateJournalSnapshot(
|
|
203
|
+
value: unknown,
|
|
204
|
+
): value is JournalSnapshot {
|
|
205
|
+
const s = obj(value, "snapshot");
|
|
206
|
+
if (s.schemaVersion !== SCHEMA_VERSION)
|
|
207
|
+
fail("snapshot.schemaVersion is unsupported");
|
|
208
|
+
integer(s.revision, "snapshot.revision");
|
|
209
|
+
text(s.instanceId, "snapshot.instanceId", 256);
|
|
210
|
+
validateBus(s.bus);
|
|
211
|
+
list(s.manualPaused, "snapshot.manualPaused", 4096).forEach((id) =>
|
|
212
|
+
text(id, "snapshot.manualPaused[]", 256),
|
|
213
|
+
);
|
|
214
|
+
const deliveries = list(s.deliveries, "snapshot.deliveries").map((d, i) =>
|
|
215
|
+
validateDelivery(d, `snapshot.deliveries[${i}]`),
|
|
216
|
+
);
|
|
217
|
+
const ids = new Set<string>();
|
|
218
|
+
for (const d of deliveries) {
|
|
219
|
+
if (ids.has(d.id)) fail(`duplicate delivery ${d.id}`);
|
|
220
|
+
ids.add(d.id);
|
|
221
|
+
}
|
|
222
|
+
return true;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
export class DeliveryJournal {
|
|
226
|
+
readonly file: string;
|
|
227
|
+
readonly projectRoot: string;
|
|
228
|
+
readonly projectId: string;
|
|
229
|
+
readonly instanceId: string;
|
|
230
|
+
private readonly db: Database;
|
|
231
|
+
private closed = false;
|
|
232
|
+
private readonly operationId?: string;
|
|
233
|
+
constructor(options: DeliveryJournalOptions) {
|
|
234
|
+
this.file = options.file;
|
|
235
|
+
this.projectRoot = text(options.projectRoot, "projectRoot", 4096);
|
|
236
|
+
this.projectId = text(options.projectId, "projectId", 256);
|
|
237
|
+
this.instanceId = text(options.instanceId, "instanceId", 256);
|
|
238
|
+
this.operationId = options.operationId;
|
|
239
|
+
mkdirSync(dirname(options.file), { recursive: true, mode: 0o700 });
|
|
240
|
+
this.db = new Database(options.file, { create: true });
|
|
241
|
+
this.harden();
|
|
242
|
+
this.db.run("PRAGMA journal_mode = WAL");
|
|
243
|
+
this.db.run("PRAGMA synchronous = FULL");
|
|
244
|
+
this.db.run("PRAGMA busy_timeout = 3000");
|
|
245
|
+
this.createSchema();
|
|
246
|
+
const meta = this.readMeta();
|
|
247
|
+
if (!meta)
|
|
248
|
+
this.db
|
|
249
|
+
.query(
|
|
250
|
+
"INSERT INTO delivery_meta (project_id, project_root, revision, instance_id, operation_id) VALUES (?, ?, 0, ?, ?)",
|
|
251
|
+
)
|
|
252
|
+
.run(
|
|
253
|
+
this.projectId,
|
|
254
|
+
this.projectRoot,
|
|
255
|
+
this.instanceId,
|
|
256
|
+
this.operationId ?? null,
|
|
257
|
+
);
|
|
258
|
+
else {
|
|
259
|
+
this.validateStored(meta);
|
|
260
|
+
this.rejectLiveOwner(meta);
|
|
261
|
+
this.db.transaction(() => {
|
|
262
|
+
this.db
|
|
263
|
+
.query(
|
|
264
|
+
"UPDATE delivery_meta SET project_root = ?, instance_id = ?, operation_id = COALESCE(?, operation_id) WHERE project_id = ?",
|
|
265
|
+
)
|
|
266
|
+
.run(
|
|
267
|
+
this.projectRoot,
|
|
268
|
+
this.instanceId,
|
|
269
|
+
this.operationId ?? null,
|
|
270
|
+
this.projectId,
|
|
271
|
+
);
|
|
272
|
+
// A new daemon cannot infer completion of an old process's handoffs, including
|
|
273
|
+
// controlled restarts with silent native/bridge receipts still outstanding.
|
|
274
|
+
this.recoverInterrupted();
|
|
275
|
+
})();
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
private harden(): void {
|
|
279
|
+
chmodSync(this.file, 0o600);
|
|
280
|
+
for (const suffix of ["-wal", "-shm"])
|
|
281
|
+
if (existsSync(`${this.file}${suffix}`))
|
|
282
|
+
chmodSync(`${this.file}${suffix}`, 0o600);
|
|
283
|
+
try {
|
|
284
|
+
chmodSync(dirname(this.file), 0o700);
|
|
285
|
+
} catch {
|
|
286
|
+
/* caller may own parent */
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
private createSchema(): void {
|
|
290
|
+
this.db.run(
|
|
291
|
+
`CREATE TABLE IF NOT EXISTS delivery_meta (project_id TEXT PRIMARY KEY, project_root TEXT, schema_version INTEGER NOT NULL DEFAULT 1, revision INTEGER NOT NULL, instance_id TEXT NOT NULL, operation_id TEXT, manual_paused TEXT NOT NULL DEFAULT '[]', bus_snapshot TEXT NOT NULL DEFAULT '{}')`,
|
|
292
|
+
);
|
|
293
|
+
for (const [column, definition] of [
|
|
294
|
+
["project_root", "TEXT"],
|
|
295
|
+
["schema_version", "INTEGER NOT NULL DEFAULT 1"],
|
|
296
|
+
["operation_id", "TEXT"],
|
|
297
|
+
] as const) {
|
|
298
|
+
const columns = this.db
|
|
299
|
+
.query("PRAGMA table_info(delivery_meta)")
|
|
300
|
+
.all() as { name: string }[];
|
|
301
|
+
if (!columns.some((c) => c.name === column))
|
|
302
|
+
this.db.run(
|
|
303
|
+
`ALTER TABLE delivery_meta ADD COLUMN ${column} ${definition}`,
|
|
304
|
+
);
|
|
305
|
+
}
|
|
306
|
+
this.db.run(
|
|
307
|
+
`CREATE TABLE IF NOT EXISTS deliveries (id TEXT PRIMARY KEY, project_id TEXT NOT NULL, peer TEXT NOT NULL, state TEXT NOT NULL, revision INTEGER NOT NULL, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, originals TEXT NOT NULL, out_json TEXT NOT NULL, reason TEXT, attempt INTEGER, previous_id TEXT)`,
|
|
308
|
+
);
|
|
309
|
+
this.db.run(
|
|
310
|
+
`CREATE TABLE IF NOT EXISTS resolution_history (id INTEGER PRIMARY KEY AUTOINCREMENT, project_id TEXT NOT NULL, delivery_id TEXT NOT NULL, observed_revision INTEGER NOT NULL, action TEXT NOT NULL, reason TEXT NOT NULL, result_revision INTEGER NOT NULL, result_state TEXT NOT NULL, UNIQUE(project_id, delivery_id, observed_revision, action, reason))`,
|
|
311
|
+
);
|
|
312
|
+
}
|
|
313
|
+
private readMeta(): Record<string, unknown> | null {
|
|
314
|
+
return this.db
|
|
315
|
+
.query("SELECT * FROM delivery_meta WHERE project_id = ?")
|
|
316
|
+
.get(this.projectId) as Record<string, unknown> | null;
|
|
317
|
+
}
|
|
318
|
+
private rejectLiveOwner(meta: Record<string, unknown>): void {
|
|
319
|
+
if (String(meta.instance_id) === this.instanceId)
|
|
320
|
+
return;
|
|
321
|
+
const statusPath = dirname(this.file) + "/status.json";
|
|
322
|
+
try {
|
|
323
|
+
const status = JSON.parse(readFileSync(statusPath, "utf8")) as Record<
|
|
324
|
+
string,
|
|
325
|
+
unknown
|
|
326
|
+
>;
|
|
327
|
+
if (
|
|
328
|
+
status.projectId === this.projectId &&
|
|
329
|
+
status.cwd === this.projectRoot &&
|
|
330
|
+
status.instanceId === meta.instance_id &&
|
|
331
|
+
Number.isInteger(status.pid) &&
|
|
332
|
+
Number(status.pid) > 0
|
|
333
|
+
) {
|
|
334
|
+
try {
|
|
335
|
+
process.kill(Number(status.pid), 0);
|
|
336
|
+
throw new Error("delivery journal is owned by a live instance");
|
|
337
|
+
} catch (e) {
|
|
338
|
+
if (e instanceof Error && e.message.includes("owned by a live"))
|
|
339
|
+
throw e;
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
} catch (e) {
|
|
343
|
+
if (e instanceof Error && e.message.includes("owned by a live")) throw e;
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
private validateStored(meta: Record<string, unknown>): void {
|
|
347
|
+
if (
|
|
348
|
+
meta.project_root != null &&
|
|
349
|
+
String(meta.project_root) !== this.projectRoot
|
|
350
|
+
)
|
|
351
|
+
fail("project root does not match database");
|
|
352
|
+
if (
|
|
353
|
+
Number(meta.schema_version) !== SCHEMA_VERSION ||
|
|
354
|
+
!Number.isSafeInteger(Number(meta.revision)) ||
|
|
355
|
+
Number(meta.revision) < 0
|
|
356
|
+
)
|
|
357
|
+
fail("metadata is corrupt");
|
|
358
|
+
try {
|
|
359
|
+
const bus = JSON.parse(String(meta.bus_snapshot || "{}"));
|
|
360
|
+
if (!(bus && typeof bus === "object" && Object.keys(bus).length === 0))
|
|
361
|
+
validateBus(bus);
|
|
362
|
+
list(
|
|
363
|
+
JSON.parse(String(meta.manual_paused || "[]")),
|
|
364
|
+
"stored manualPaused",
|
|
365
|
+
);
|
|
366
|
+
} catch (e) {
|
|
367
|
+
throw e instanceof Error &&
|
|
368
|
+
e.message.startsWith("invalid delivery journal")
|
|
369
|
+
? e
|
|
370
|
+
: new Error("invalid delivery journal: corrupt metadata JSON");
|
|
371
|
+
}
|
|
372
|
+
const rows = this.db
|
|
373
|
+
.query("SELECT * FROM deliveries WHERE project_id = ? LIMIT ?")
|
|
374
|
+
.all(this.projectId, MAX_ROWS) as Record<string, unknown>[];
|
|
375
|
+
for (const row of rows) validateDelivery(parseDelivery(row));
|
|
376
|
+
if (
|
|
377
|
+
rows.length === MAX_ROWS &&
|
|
378
|
+
this.db
|
|
379
|
+
.query("SELECT 1 FROM deliveries WHERE project_id = ? LIMIT 1 OFFSET ?")
|
|
380
|
+
.get(this.projectId, MAX_ROWS)
|
|
381
|
+
)
|
|
382
|
+
fail("too many delivery rows");
|
|
383
|
+
}
|
|
384
|
+
private recoverInterrupted(): void {
|
|
385
|
+
this.db
|
|
386
|
+
.query(
|
|
387
|
+
"UPDATE deliveries SET state = 'needs_review', revision = revision + 1, updated_at = ?, reason = COALESCE(reason, 'daemon stopped during delivery') WHERE project_id = ? AND state IN ('dispatching', 'accepted')",
|
|
388
|
+
)
|
|
389
|
+
.run(Date.now(), this.projectId);
|
|
390
|
+
}
|
|
391
|
+
private ensureOpen(): void {
|
|
392
|
+
if (this.closed) throw new Error("delivery journal is closed");
|
|
393
|
+
}
|
|
394
|
+
private bump(): number {
|
|
395
|
+
const row = this.readMeta();
|
|
396
|
+
if (!row) throw new Error("delivery journal metadata missing");
|
|
397
|
+
if (String(row.instance_id) !== this.instanceId)
|
|
398
|
+
throw new Error("delivery journal instance fence lost");
|
|
399
|
+
const revision = Number(row.revision) + 1;
|
|
400
|
+
this.db
|
|
401
|
+
.query(
|
|
402
|
+
"UPDATE delivery_meta SET revision = ? WHERE project_id = ? AND instance_id = ?",
|
|
403
|
+
)
|
|
404
|
+
.run(revision, this.projectId, this.instanceId);
|
|
405
|
+
if (
|
|
406
|
+
Number(
|
|
407
|
+
(this.db.query("SELECT changes() AS n").get() as { n: number }).n,
|
|
408
|
+
) !== 1
|
|
409
|
+
)
|
|
410
|
+
throw new Error("delivery journal instance fence lost");
|
|
411
|
+
return revision;
|
|
412
|
+
}
|
|
413
|
+
transaction<T>(fn: () => T): T {
|
|
414
|
+
this.ensureOpen();
|
|
415
|
+
return this.db.transaction(fn)();
|
|
416
|
+
}
|
|
417
|
+
snapshot(): JournalSnapshot {
|
|
418
|
+
this.ensureOpen();
|
|
419
|
+
const meta = this.readMeta();
|
|
420
|
+
if (!meta) throw new Error("delivery journal metadata missing");
|
|
421
|
+
this.validateStored(meta);
|
|
422
|
+
const rows = this.db
|
|
423
|
+
.query(
|
|
424
|
+
"SELECT * FROM deliveries WHERE project_id = ? ORDER BY created_at, id",
|
|
425
|
+
)
|
|
426
|
+
.all(this.projectId) as Record<string, unknown>[];
|
|
427
|
+
const storedBus = JSON.parse(String(meta.bus_snapshot || "{}"));
|
|
428
|
+
const bus =
|
|
429
|
+
Object.keys(storedBus).length === 0
|
|
430
|
+
? {
|
|
431
|
+
schemaVersion: 1,
|
|
432
|
+
queues: {},
|
|
433
|
+
prefaces: {},
|
|
434
|
+
seen: [],
|
|
435
|
+
attempts: {},
|
|
436
|
+
withdrawn: [],
|
|
437
|
+
}
|
|
438
|
+
: storedBus;
|
|
439
|
+
const result = {
|
|
440
|
+
schemaVersion: SCHEMA_VERSION,
|
|
441
|
+
revision: Number(meta.revision),
|
|
442
|
+
instanceId: String(meta.instance_id),
|
|
443
|
+
bus,
|
|
444
|
+
manualPaused: JSON.parse(String(meta.manual_paused || "[]")),
|
|
445
|
+
deliveries: rows.map(parseDelivery),
|
|
446
|
+
};
|
|
447
|
+
validateJournalSnapshot(result);
|
|
448
|
+
return result;
|
|
449
|
+
}
|
|
450
|
+
persistBus(bus: unknown, manualPaused: string[]): number {
|
|
451
|
+
this.ensureOpen();
|
|
452
|
+
validateBus(bus);
|
|
453
|
+
list(manualPaused, "manualPaused", 4096).forEach((id) =>
|
|
454
|
+
text(id, "manualPaused[]", 256),
|
|
455
|
+
);
|
|
456
|
+
return this.db.transaction(() => {
|
|
457
|
+
const revision = this.bump();
|
|
458
|
+
this.db
|
|
459
|
+
.query(
|
|
460
|
+
"UPDATE delivery_meta SET bus_snapshot = ?, manual_paused = ? WHERE project_id = ?",
|
|
461
|
+
)
|
|
462
|
+
.run(
|
|
463
|
+
JSON.stringify(checkedJSON(bus, "bus")),
|
|
464
|
+
JSON.stringify([...new Set(manualPaused)].sort()),
|
|
465
|
+
this.projectId,
|
|
466
|
+
);
|
|
467
|
+
this.harden();
|
|
468
|
+
return revision;
|
|
469
|
+
})();
|
|
470
|
+
}
|
|
471
|
+
/** Atomically persist a bus snapshot and any fan-out rows created for that snapshot. */
|
|
472
|
+
persistBusAndDeliveries(
|
|
473
|
+
bus: BusSnapshot,
|
|
474
|
+
manualPaused: string[],
|
|
475
|
+
deliveries: Array<
|
|
476
|
+
Omit<JournalDelivery, "revision" | "updatedAt"> & { updatedAt?: number }
|
|
477
|
+
>,
|
|
478
|
+
): number {
|
|
479
|
+
this.ensureOpen();
|
|
480
|
+
validateBus(bus);
|
|
481
|
+
list(manualPaused, "manualPaused", 4096).forEach((id) =>
|
|
482
|
+
text(id, "manualPaused[]", 256),
|
|
483
|
+
);
|
|
484
|
+
deliveries.forEach((d) =>
|
|
485
|
+
validateDelivery({
|
|
486
|
+
...d,
|
|
487
|
+
revision: 0,
|
|
488
|
+
updatedAt: d.updatedAt ?? d.createdAt,
|
|
489
|
+
}),
|
|
490
|
+
);
|
|
491
|
+
return this.db.transaction(() => {
|
|
492
|
+
const revision = this.bump();
|
|
493
|
+
this.db
|
|
494
|
+
.query(
|
|
495
|
+
"UPDATE delivery_meta SET bus_snapshot = ?, manual_paused = ? WHERE project_id = ?",
|
|
496
|
+
)
|
|
497
|
+
.run(
|
|
498
|
+
JSON.stringify(bus),
|
|
499
|
+
JSON.stringify([...new Set(manualPaused)].sort()),
|
|
500
|
+
this.projectId,
|
|
501
|
+
);
|
|
502
|
+
for (const d of deliveries)
|
|
503
|
+
this.db
|
|
504
|
+
.query(
|
|
505
|
+
"INSERT INTO deliveries (id, project_id, peer, state, revision, created_at, updated_at, originals, out_json, reason, attempt, previous_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
506
|
+
)
|
|
507
|
+
.run(
|
|
508
|
+
d.id,
|
|
509
|
+
this.projectId,
|
|
510
|
+
d.peer,
|
|
511
|
+
d.state,
|
|
512
|
+
revision,
|
|
513
|
+
d.createdAt,
|
|
514
|
+
d.updatedAt ?? Date.now(),
|
|
515
|
+
JSON.stringify(d.originals),
|
|
516
|
+
JSON.stringify(d.out),
|
|
517
|
+
d.reason ?? null,
|
|
518
|
+
d.attempt ?? null,
|
|
519
|
+
d.previousId ?? null,
|
|
520
|
+
);
|
|
521
|
+
this.harden();
|
|
522
|
+
return revision;
|
|
523
|
+
})();
|
|
524
|
+
}
|
|
525
|
+
checkpointDelivery(
|
|
526
|
+
bus: BusSnapshot,
|
|
527
|
+
delivery: Omit<JournalDelivery, "revision" | "updatedAt"> & {
|
|
528
|
+
state: "dispatching";
|
|
529
|
+
},
|
|
530
|
+
): JournalDelivery {
|
|
531
|
+
validateBus(bus);
|
|
532
|
+
validateDelivery({
|
|
533
|
+
...delivery,
|
|
534
|
+
revision: 0,
|
|
535
|
+
updatedAt: delivery.createdAt,
|
|
536
|
+
});
|
|
537
|
+
return this.db.transaction(() => {
|
|
538
|
+
const ids = new Set(delivery.originals.map((env) => env.id));
|
|
539
|
+
for (const row of this.list(delivery.peer)) {
|
|
540
|
+
if (row.state !== "queued" || !row.originals.some((env) => ids.has(env.id))) continue;
|
|
541
|
+
this.transition(row.id, "discarded", `grouped into delivery ${delivery.id}`);
|
|
542
|
+
delivery = { ...delivery, previousId: delivery.previousId ?? row.id };
|
|
543
|
+
}
|
|
544
|
+
const revision = this.bump();
|
|
545
|
+
const now = Date.now();
|
|
546
|
+
this.db
|
|
547
|
+
.query(
|
|
548
|
+
"UPDATE delivery_meta SET bus_snapshot = ?, manual_paused = ? WHERE project_id = ?",
|
|
549
|
+
)
|
|
550
|
+
.run(
|
|
551
|
+
JSON.stringify(bus),
|
|
552
|
+
JSON.stringify(bus.manualPaused ?? []),
|
|
553
|
+
this.projectId,
|
|
554
|
+
);
|
|
555
|
+
this.db
|
|
556
|
+
.query(
|
|
557
|
+
"INSERT INTO deliveries (id, project_id, peer, state, revision, created_at, updated_at, originals, out_json, reason, attempt, previous_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
558
|
+
)
|
|
559
|
+
.run(
|
|
560
|
+
delivery.id,
|
|
561
|
+
this.projectId,
|
|
562
|
+
delivery.peer,
|
|
563
|
+
delivery.state,
|
|
564
|
+
revision,
|
|
565
|
+
delivery.createdAt,
|
|
566
|
+
now,
|
|
567
|
+
JSON.stringify(delivery.originals),
|
|
568
|
+
JSON.stringify(delivery.out),
|
|
569
|
+
delivery.reason ?? null,
|
|
570
|
+
delivery.attempt ?? null,
|
|
571
|
+
delivery.previousId ?? null,
|
|
572
|
+
);
|
|
573
|
+
this.harden();
|
|
574
|
+
return { ...delivery, revision, updatedAt: now };
|
|
575
|
+
})();
|
|
576
|
+
}
|
|
577
|
+
createDelivery(
|
|
578
|
+
input: Omit<JournalDelivery, "revision" | "updatedAt"> & {
|
|
579
|
+
updatedAt?: number;
|
|
580
|
+
},
|
|
581
|
+
): JournalDelivery {
|
|
582
|
+
this.ensureOpen();
|
|
583
|
+
validateDelivery({
|
|
584
|
+
...input,
|
|
585
|
+
revision: 0,
|
|
586
|
+
updatedAt: input.updatedAt ?? input.createdAt,
|
|
587
|
+
});
|
|
588
|
+
return this.db.transaction(() => {
|
|
589
|
+
const revision = this.bump();
|
|
590
|
+
const updatedAt = input.updatedAt ?? Date.now();
|
|
591
|
+
this.db
|
|
592
|
+
.query(
|
|
593
|
+
"INSERT INTO deliveries (id, project_id, peer, state, revision, created_at, updated_at, originals, out_json, reason, attempt, previous_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
594
|
+
)
|
|
595
|
+
.run(
|
|
596
|
+
input.id,
|
|
597
|
+
this.projectId,
|
|
598
|
+
input.peer,
|
|
599
|
+
input.state,
|
|
600
|
+
revision,
|
|
601
|
+
input.createdAt,
|
|
602
|
+
updatedAt,
|
|
603
|
+
JSON.stringify(input.originals),
|
|
604
|
+
JSON.stringify(input.out),
|
|
605
|
+
input.reason ?? null,
|
|
606
|
+
input.attempt ?? null,
|
|
607
|
+
input.previousId ?? null,
|
|
608
|
+
);
|
|
609
|
+
this.pruneTerminal();
|
|
610
|
+
return { ...input, revision, updatedAt };
|
|
611
|
+
})();
|
|
612
|
+
}
|
|
613
|
+
transition(
|
|
614
|
+
id: string,
|
|
615
|
+
state: JournalDeliveryState,
|
|
616
|
+
reason?: string,
|
|
617
|
+
expectedRevision?: number,
|
|
618
|
+
): JournalDelivery {
|
|
619
|
+
this.ensureOpen();
|
|
620
|
+
if (!STATES.has(state)) throw new Error("invalid delivery state");
|
|
621
|
+
return this.db.transaction(() => {
|
|
622
|
+
const old = this.get(id);
|
|
623
|
+
if (!old) throw new Error(`unknown delivery ${id}`);
|
|
624
|
+
if (expectedRevision !== undefined && old.revision !== expectedRevision)
|
|
625
|
+
throw new Error(`stale delivery revision for ${id}`);
|
|
626
|
+
const revision = this.bump();
|
|
627
|
+
this.db
|
|
628
|
+
.query(
|
|
629
|
+
"UPDATE deliveries SET state = ?, revision = ?, updated_at = ?, reason = ? WHERE id = ? AND project_id = ?",
|
|
630
|
+
)
|
|
631
|
+
.run(state, revision, Date.now(), reason ?? null, id, this.projectId);
|
|
632
|
+
this.pruneTerminal();
|
|
633
|
+
return this.get(id)!;
|
|
634
|
+
})();
|
|
635
|
+
}
|
|
636
|
+
get(id: string): JournalDelivery | undefined {
|
|
637
|
+
this.ensureOpen();
|
|
638
|
+
const row = this.db
|
|
639
|
+
.query("SELECT * FROM deliveries WHERE id = ? AND project_id = ?")
|
|
640
|
+
.get(id, this.projectId) as Record<string, unknown> | null;
|
|
641
|
+
return row ? parseDelivery(row) : undefined;
|
|
642
|
+
}
|
|
643
|
+
list(peer?: string): JournalDelivery[] {
|
|
644
|
+
this.ensureOpen();
|
|
645
|
+
const rows = (
|
|
646
|
+
peer === undefined
|
|
647
|
+
? this.db
|
|
648
|
+
.query(
|
|
649
|
+
"SELECT * FROM deliveries WHERE project_id = ? ORDER BY created_at, id",
|
|
650
|
+
)
|
|
651
|
+
.all(this.projectId)
|
|
652
|
+
: this.db
|
|
653
|
+
.query(
|
|
654
|
+
"SELECT * FROM deliveries WHERE project_id = ? AND peer = ? ORDER BY created_at, id",
|
|
655
|
+
)
|
|
656
|
+
.all(this.projectId, peer)
|
|
657
|
+
) as Record<string, unknown>[];
|
|
658
|
+
return rows.map(parseDelivery);
|
|
659
|
+
}
|
|
660
|
+
resolve(
|
|
661
|
+
id: string,
|
|
662
|
+
revision: number,
|
|
663
|
+
action: JournalResolution,
|
|
664
|
+
reason: string,
|
|
665
|
+
requestRevision = revision,
|
|
666
|
+
): JournalDelivery {
|
|
667
|
+
this.ensureOpen();
|
|
668
|
+
text(reason, "resolution reason", 1_000_000);
|
|
669
|
+
integer(revision, "observed revision");
|
|
670
|
+
if (!ACTIONS.has(action)) throw new Error("invalid resolution action");
|
|
671
|
+
return this.db.transaction(() => {
|
|
672
|
+
const resolved = this.resolution(id);
|
|
673
|
+
if (resolved) {
|
|
674
|
+
const current = this.get(id);
|
|
675
|
+
if (current && resolved.action === action && resolved.reason === reason && (requestRevision === resolved.observedRevision || requestRevision === current.revision)) return current;
|
|
676
|
+
throw new Error("stale delivery revision or conflicting operator resolution");
|
|
677
|
+
}
|
|
678
|
+
const prior = this.db
|
|
679
|
+
.query(
|
|
680
|
+
"SELECT result_revision FROM resolution_history WHERE project_id = ? AND delivery_id = ? AND observed_revision = ? AND action = ? AND reason = ?",
|
|
681
|
+
)
|
|
682
|
+
.get(this.projectId, id, requestRevision, action, reason) as {
|
|
683
|
+
result_revision: number;
|
|
684
|
+
} | null;
|
|
685
|
+
if (prior) return this.get(id)!;
|
|
686
|
+
const current = this.get(id);
|
|
687
|
+
if (!current) throw new Error(`unknown delivery ${id}`);
|
|
688
|
+
if (current.revision !== revision)
|
|
689
|
+
throw new Error(`stale delivery revision for ${id}`);
|
|
690
|
+
if (
|
|
691
|
+
["completed", "discarded"].includes(current.state) &&
|
|
692
|
+
actionState(action) !== current.state
|
|
693
|
+
)
|
|
694
|
+
throw new Error(`delivery ${id} is already terminal`);
|
|
695
|
+
const result = this.transition(id, actionState(action), reason, revision);
|
|
696
|
+
if (action === "retry") {
|
|
697
|
+
const childId = `${id}:retry:${(current.attempt ?? 0) + 1}`;
|
|
698
|
+
if (!this.get(childId))
|
|
699
|
+
this.createDelivery({
|
|
700
|
+
id: childId,
|
|
701
|
+
peer: current.peer,
|
|
702
|
+
state: "queued",
|
|
703
|
+
createdAt: Date.now(),
|
|
704
|
+
originals: current.originals,
|
|
705
|
+
out: current.out,
|
|
706
|
+
attempt: (current.attempt ?? 0) + 1,
|
|
707
|
+
previousId: id,
|
|
708
|
+
});
|
|
709
|
+
}
|
|
710
|
+
this.db
|
|
711
|
+
.query(
|
|
712
|
+
"INSERT INTO resolution_history (project_id, delivery_id, observed_revision, action, reason, result_revision, result_state) VALUES (?, ?, ?, ?, ?, ?, ?)",
|
|
713
|
+
)
|
|
714
|
+
.run(
|
|
715
|
+
this.projectId,
|
|
716
|
+
id,
|
|
717
|
+
requestRevision,
|
|
718
|
+
action,
|
|
719
|
+
reason,
|
|
720
|
+
result.revision,
|
|
721
|
+
result.state,
|
|
722
|
+
);
|
|
723
|
+
this.pruneTerminal();
|
|
724
|
+
return result;
|
|
725
|
+
})();
|
|
726
|
+
}
|
|
727
|
+
resolution(id: string): { action: JournalResolution; reason: string; observedRevision: number } | undefined {
|
|
728
|
+
const row = this.db.query("SELECT action,reason,observed_revision FROM resolution_history WHERE project_id=? AND delivery_id=? ORDER BY id DESC LIMIT 1").get(this.projectId, id) as { action: JournalResolution; reason: string; observed_revision: number } | null;
|
|
729
|
+
return row ? { action: row.action, reason: row.reason, observedRevision: row.observed_revision } : undefined;
|
|
730
|
+
}
|
|
731
|
+
private pruneTerminal(): void {
|
|
732
|
+
const count = Number(
|
|
733
|
+
(
|
|
734
|
+
this.db
|
|
735
|
+
.query(
|
|
736
|
+
"SELECT COUNT(*) AS n FROM deliveries WHERE project_id = ? AND state IN ('completed','discarded','failed')",
|
|
737
|
+
)
|
|
738
|
+
.get(this.projectId) as { n: number }
|
|
739
|
+
).n,
|
|
740
|
+
);
|
|
741
|
+
if (count <= TERMINAL_CAP) return;
|
|
742
|
+
this.db
|
|
743
|
+
.query(
|
|
744
|
+
"DELETE FROM deliveries WHERE id IN (SELECT d.id FROM deliveries d WHERE d.project_id = ? AND d.state IN ('completed','discarded','failed') AND NOT EXISTS (SELECT 1 FROM deliveries c WHERE c.project_id = d.project_id AND c.previous_id = d.id) ORDER BY d.updated_at, d.id LIMIT ?)",
|
|
745
|
+
)
|
|
746
|
+
.run(this.projectId, count - TERMINAL_CAP);
|
|
747
|
+
}
|
|
748
|
+
importSnapshot(snapshot: JournalSnapshot, operationId: string): boolean {
|
|
749
|
+
this.ensureOpen();
|
|
750
|
+
validateJournalSnapshot(snapshot);
|
|
751
|
+
text(operationId, "operationId", 256);
|
|
752
|
+
if (this.operationId !== undefined && this.operationId !== operationId)
|
|
753
|
+
throw new Error("recovery operation does not own delivery journal");
|
|
754
|
+
const meta = this.readMeta();
|
|
755
|
+
if (!meta) throw new Error("delivery journal metadata missing");
|
|
756
|
+
this.validateStored(meta);
|
|
757
|
+
const deliveries = this.db
|
|
758
|
+
.query("SELECT 1 FROM deliveries WHERE project_id = ? LIMIT 1")
|
|
759
|
+
.get(this.projectId);
|
|
760
|
+
const storedBus = JSON.parse(String(meta.bus_snapshot || "{}"));
|
|
761
|
+
if (
|
|
762
|
+
Number(meta.revision) !== 0 ||
|
|
763
|
+
deliveries ||
|
|
764
|
+
Object.keys(storedBus).length !== 0
|
|
765
|
+
)
|
|
766
|
+
return false;
|
|
767
|
+
return this.db.transaction(() => {
|
|
768
|
+
this.db
|
|
769
|
+
.query(
|
|
770
|
+
"UPDATE delivery_meta SET revision = ?, bus_snapshot = ?, manual_paused = ?, operation_id = ? WHERE project_id = ?",
|
|
771
|
+
)
|
|
772
|
+
.run(
|
|
773
|
+
snapshot.revision,
|
|
774
|
+
JSON.stringify(snapshot.bus),
|
|
775
|
+
JSON.stringify(snapshot.manualPaused),
|
|
776
|
+
operationId,
|
|
777
|
+
this.projectId,
|
|
778
|
+
);
|
|
779
|
+
for (const row of snapshot.deliveries)
|
|
780
|
+
this.db
|
|
781
|
+
.query(
|
|
782
|
+
"INSERT INTO deliveries (id, project_id, peer, state, revision, created_at, updated_at, originals, out_json, reason, attempt, previous_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
|
783
|
+
)
|
|
784
|
+
.run(
|
|
785
|
+
row.id,
|
|
786
|
+
this.projectId,
|
|
787
|
+
row.peer,
|
|
788
|
+
row.state,
|
|
789
|
+
row.revision,
|
|
790
|
+
row.createdAt,
|
|
791
|
+
row.updatedAt,
|
|
792
|
+
JSON.stringify(row.originals),
|
|
793
|
+
JSON.stringify(row.out),
|
|
794
|
+
row.reason ?? null,
|
|
795
|
+
row.attempt ?? null,
|
|
796
|
+
row.previousId ?? null,
|
|
797
|
+
);
|
|
798
|
+
return true;
|
|
799
|
+
})();
|
|
800
|
+
}
|
|
801
|
+
close(): void {
|
|
802
|
+
if (!this.closed) {
|
|
803
|
+
this.harden();
|
|
804
|
+
this.db.close();
|
|
805
|
+
this.closed = true;
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
function actionState(action: JournalResolution): JournalDeliveryState {
|
|
810
|
+
return action === "completed"
|
|
811
|
+
? "completed"
|
|
812
|
+
: action === "discard"
|
|
813
|
+
? "discarded"
|
|
814
|
+
: "failed";
|
|
815
|
+
}
|
|
816
|
+
function parseDelivery(row: Record<string, unknown>): JournalDelivery {
|
|
817
|
+
return validateDelivery({
|
|
818
|
+
id: String(row.id),
|
|
819
|
+
peer: String(row.peer),
|
|
820
|
+
state: String(row.state),
|
|
821
|
+
revision: Number(row.revision),
|
|
822
|
+
createdAt: Number(row.created_at),
|
|
823
|
+
updatedAt: Number(row.updated_at),
|
|
824
|
+
originals: JSON.parse(String(row.originals)),
|
|
825
|
+
out: JSON.parse(String(row.out_json)),
|
|
826
|
+
...(row.reason == null ? {} : { reason: String(row.reason) }),
|
|
827
|
+
...(row.attempt == null ? {} : { attempt: Number(row.attempt) }),
|
|
828
|
+
...(row.previous_id == null ? {} : { previousId: String(row.previous_id) }),
|
|
829
|
+
});
|
|
830
|
+
}
|