@gr8ful/spf 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -9
- package/assets/skill/references/config.md +12 -5
- package/assets/skill/references/observability.md +57 -12
- package/assets/templates/ts-opencode.spf.config.yaml +54 -0
- package/dist/chains/index.js +1 -1
- package/dist/chains/simple_sdlc.d.ts +2 -2
- package/dist/chains/simple_sdlc.js +13 -13
- package/dist/chains/steps.d.ts +2 -2
- package/dist/chains/steps.js +35 -19
- package/dist/cli/commands/abort.d.ts +1 -1
- package/dist/cli/commands/abort.js +30 -3
- package/dist/cli/commands/doctor.js +121 -8
- package/dist/cli/commands/estimate.js +3 -3
- package/dist/cli/commands/events.js +4 -4
- package/dist/cli/commands/fanout.js +93 -21
- package/dist/cli/commands/loop.js +31 -32
- package/dist/cli/commands/migrate.js +8 -1
- package/dist/cli/commands/phases.js +2 -2
- package/dist/cli/commands/sessions.js +2 -2
- package/dist/cli/commands/trace.d.ts +28 -8
- package/dist/cli/commands/trace.js +28 -15
- package/dist/cli/commands/ui.js +15 -5
- package/dist/cli/commands/watch.js +91 -30
- package/dist/cli/index.js +3 -1
- package/dist/cli/interview.d.ts +1 -0
- package/dist/cli/interview.js +95 -5
- package/dist/core/agent_opencode.d.ts +247 -0
- package/dist/core/agent_opencode.js +590 -0
- package/dist/core/agents.d.ts +12 -12
- package/dist/core/agents.js +113 -46
- package/dist/core/console.d.ts +12 -12
- package/dist/core/console.js +25 -25
- package/dist/core/data_types.d.ts +356 -15
- package/dist/core/data_types.js +180 -7
- package/dist/core/fanout.d.ts +1 -1
- package/dist/core/fanout.js +1 -1
- package/dist/core/gates.js +14 -1
- package/dist/core/issues/github_provider.d.ts +66 -2
- package/dist/core/issues/github_provider.js +161 -2
- package/dist/core/issues/jira_provider.d.ts +50 -9
- package/dist/core/issues/jira_provider.js +62 -2
- package/dist/core/paths.d.ts +41 -4
- package/dist/core/paths.js +32 -3
- package/dist/core/quality.d.ts +7 -7
- package/dist/core/quality.js +16 -10
- package/dist/core/refine.js +2 -2
- package/dist/core/runner.d.ts +9 -3
- package/dist/core/runner.js +39 -27
- package/dist/core/session.d.ts +2 -2
- package/dist/core/session.js +39 -18
- package/dist/core/sqlite.d.ts +14 -7
- package/dist/core/sqlite.js +14 -7
- package/dist/core/trace_db.d.ts +118 -0
- package/dist/core/trace_db.js +278 -0
- package/dist/core/tracer.d.ts +64 -34
- package/dist/core/tracer.js +141 -69
- package/dist/core/watch.d.ts +4 -4
- package/dist/core/watch.js +2 -2
- package/dist/ui/server/app.js +10 -10
- package/dist/ui/server/db.d.ts +89 -21
- package/dist/ui/server/db.js +235 -99
- package/dist/ui/server/serve.d.ts +5 -1
- package/dist/ui/server/serve.js +4 -5
- package/package.json +1 -1
- package/web/assets/index-CQ3k1Y1-.css +1 -0
- package/web/assets/index-CU8tom6S.js +21 -0
- package/web/assets/overpass-latin-400-normal-BpeLJ0bs.woff2 +0 -0
- package/web/assets/overpass-latin-600-normal-25RhTNCi.woff2 +0 -0
- package/web/assets/overpass-latin-700-normal-CQX2QTgM.woff2 +0 -0
- package/web/assets/overpass-mono-latin-400-normal-VINZG6Js.woff2 +0 -0
- package/web/assets/overpass-mono-latin-700-normal-D6nRBrbd.woff2 +0 -0
- package/web/index.html +33 -2
- package/web/logo.svg +4 -4
- package/web/assets/index-C7nF068F.css +0 -1
- package/web/assets/index-mzSArcnQ.js +0 -11
- package/web/assets/play-latin-400-normal-GKW-4YV7.woff2 +0 -0
- package/web/assets/play-latin-700-normal-DyPlLDbb.woff2 +0 -0
package/dist/core/session.js
CHANGED
|
@@ -66,24 +66,41 @@ function handleSignal(signal) {
|
|
|
66
66
|
process.exit(code);
|
|
67
67
|
draining = true;
|
|
68
68
|
const runs = [...ACTIVE.values()]; // snapshot: finalize() may mutate ACTIVE mid-drain
|
|
69
|
-
|
|
70
|
-
|
|
69
|
+
// `sessionFinish` is now async (see tracer.ts's header — a D1-backed run's
|
|
70
|
+
// write is a real network call) — a signal handler cannot itself be async,
|
|
71
|
+
// so this fires every run's finish without awaiting it directly. For the
|
|
72
|
+
// LOCAL backend that costs nothing: `LocalTraceDb` performs the write
|
|
73
|
+
// SYNCHRONOUSLY before ever returning the (already-resolved) Promise this
|
|
74
|
+
// discards, so by the time `sessionFinish(...)` returns here the write has
|
|
75
|
+
// already landed — same as before this class existed. Only a run whose
|
|
76
|
+
// `tracer.dbPath` is `null` (d1-backed — see `Tracer.open`) is genuinely
|
|
77
|
+
// still in flight at this point, so ONLY THEN does this add a bounded wait
|
|
78
|
+
// (same `SIGNAL_DRAIN_MS` budget the otel/notify drains already use) before
|
|
79
|
+
// exiting — a killed D1-backed run gets a real chance to land its final
|
|
80
|
+
// write, and every local-only repo keeps exiting with no extra tick.
|
|
81
|
+
const finishes = runs.map((run) => run.tracer.sessionFinish(run.adw_id, false).catch(() => { })); // also closes process rows
|
|
82
|
+
const anyRemote = runs.some((run) => run.tracer.dbPath === null);
|
|
71
83
|
const drains = [];
|
|
84
|
+
if (anyRemote)
|
|
85
|
+
drains.push(Promise.race([Promise.all(finishes), sleep(SIGNAL_DRAIN_MS)]).then(() => { }));
|
|
72
86
|
if (runs.some((run) => run.tracer.otel))
|
|
73
87
|
drains.push(otel.flushAll(SIGNAL_DRAIN_MS));
|
|
74
88
|
if (runs.some((run) => run.notify))
|
|
75
89
|
drains.push(drainNotifiers(SIGNAL_DRAIN_MS));
|
|
76
|
-
// Unconfigured (the default) exits SYNCHRONOUSLY, exactly as it
|
|
77
|
-
// otel/notify existed — no extra tick between the signal and
|
|
78
|
-
// the repos that never opted in to
|
|
90
|
+
// Unconfigured, local-only (the default) exits SYNCHRONOUSLY, exactly as it
|
|
91
|
+
// did before otel/notify/D1 existed — no extra tick between the signal and
|
|
92
|
+
// the exit for the repos that never opted in to any of the three.
|
|
79
93
|
if (drains.length === 0) {
|
|
80
94
|
process.exit(code);
|
|
81
95
|
return;
|
|
82
96
|
}
|
|
83
|
-
// Bounded and never-throwing:
|
|
84
|
-
//
|
|
97
|
+
// Bounded and never-throwing: every drain swallows its own failures and
|
|
98
|
+
// resolves on its own deadline, so this always reaches process.exit().
|
|
85
99
|
void Promise.all(drains).then(() => process.exit(code), () => process.exit(code));
|
|
86
100
|
}
|
|
101
|
+
function sleep(ms) {
|
|
102
|
+
return new Promise((resolve) => setTimeout(resolve, ms).unref());
|
|
103
|
+
}
|
|
87
104
|
function finalizeWhenKilled(run) {
|
|
88
105
|
ACTIVE.set(run.adw_id, run);
|
|
89
106
|
if (installed)
|
|
@@ -114,14 +131,14 @@ function finalizeWhenKilled(run) {
|
|
|
114
131
|
* `releaseOtelExporter`, and harmless for the same reason: that process
|
|
115
132
|
* exits right after anyway.
|
|
116
133
|
*/
|
|
117
|
-
export function finalize(adwId) {
|
|
134
|
+
export async function finalize(adwId) {
|
|
118
135
|
if (!adwId)
|
|
119
136
|
return;
|
|
120
137
|
const run = ACTIVE.get(adwId);
|
|
121
138
|
if (!run)
|
|
122
139
|
return;
|
|
123
140
|
ACTIVE.delete(adwId);
|
|
124
|
-
run.tracer.close();
|
|
141
|
+
await run.tracer.close();
|
|
125
142
|
}
|
|
126
143
|
/** Tests only: which adw_ids the process-wide signal handler currently considers active. */
|
|
127
144
|
export function activeRunIdsForTest() {
|
|
@@ -138,7 +155,7 @@ export function activeRunIdsForTest() {
|
|
|
138
155
|
* longer a `process.argv[1]` basename that means anything. Direct callers
|
|
139
156
|
* that have no chain of their own fall back to `"adw"`.
|
|
140
157
|
*/
|
|
141
|
-
export function ensure(cfg, adwId, cwd, chainName,
|
|
158
|
+
export async function ensure(cfg, adwId, cwd, chainName,
|
|
142
159
|
/** See `RunObserver`'s doc comment (`core/console.ts`). Omitted for every caller except an interactive `cli/commands/run.ts` dispatch — a `spf watch` per-issue run, `spf fanout`'s per-attempt runs, and every test all continue to build a plain, unobserved `Console`. */
|
|
143
160
|
renderHooks) {
|
|
144
161
|
const id = adwId || newId(8);
|
|
@@ -147,16 +164,20 @@ renderHooks) {
|
|
|
147
164
|
// `null` unless `observability.otel` is configured — no environment variable
|
|
148
165
|
// can turn this on (see core/otel.ts's EXPLICIT CONFIG ONLY). Constructed
|
|
149
166
|
// BEFORE the Tracer because the Tracer's write methods are the fan-out
|
|
150
|
-
// seams:
|
|
151
|
-
// registering here (module-level LIVE, exactly like
|
|
152
|
-
// lets the CLI's finally block and the signal
|
|
153
|
-
// threading a handle through every call site.
|
|
167
|
+
// seams: the trace db stays the source of truth, otel is a projection off
|
|
168
|
+
// it, and registering here (module-level LIVE, exactly like
|
|
169
|
+
// resolveNotifier) is what lets the CLI's finally block and the signal
|
|
170
|
+
// handler above drain it without threading a handle through every call site.
|
|
154
171
|
const otelExporter = otel.resolveOtelExporter(cfg, { adwId: id, chainName: chainName || "adw" });
|
|
155
|
-
const tracer =
|
|
172
|
+
const tracer = await Tracer.open(dataPaths.db, path.join(dataPaths.sessions_dir, id, "events.jsonl"), otelExporter);
|
|
173
|
+
// `maxPhaseSeq` is a real trace-db read now (D1: a network call) — resolved
|
|
174
|
+
// BEFORE `new Run(...)` because `Run`'s constructor cannot itself be async.
|
|
175
|
+
const startSeq = await tracer.maxPhaseSeq(id);
|
|
156
176
|
const run = new Run({
|
|
157
177
|
cfg,
|
|
158
178
|
adwId: id,
|
|
159
179
|
tracer,
|
|
180
|
+
startSeq,
|
|
160
181
|
engineer: engineerName(),
|
|
161
182
|
repoRoot: anchor.repo_root,
|
|
162
183
|
sfDir: anchor.spf_dir,
|
|
@@ -167,11 +188,11 @@ renderHooks) {
|
|
|
167
188
|
observer: renderHooks?.observer,
|
|
168
189
|
});
|
|
169
190
|
const scriptPath = process.argv[1] || "adw";
|
|
170
|
-
tracer.sessionStart(id, run.engineer, chainName || "adw");
|
|
191
|
+
await tracer.sessionStart(id, run.engineer, chainName || "adw");
|
|
171
192
|
// This process is the run. Record it before any phase opens, so a run that
|
|
172
193
|
// hangs in its first agent call is still killable by adw_id.
|
|
173
|
-
tracer.processStart(id, "adw", "", process.pid ?? -1, [path.basename(scriptPath), ...process.argv.slice(2)].join(" "));
|
|
194
|
+
await tracer.processStart(id, "adw", "", process.pid ?? -1, [path.basename(scriptPath), ...process.argv.slice(2)].join(" "));
|
|
174
195
|
finalizeWhenKilled(run);
|
|
175
|
-
run.console.sessionStarted(id, run.engineer);
|
|
196
|
+
await run.console.sessionStarted(id, run.engineer);
|
|
176
197
|
return run;
|
|
177
198
|
}
|
package/dist/core/sqlite.d.ts
CHANGED
|
@@ -25,13 +25,20 @@
|
|
|
25
25
|
* come back with a null prototype. Both are normalized below so existing
|
|
26
26
|
* `?? null` / `Object.assign(row, ...)` call sites stay honest.
|
|
27
27
|
* - node:sqlite defaults `enableForeignKeyConstraints: true` — unlike plain
|
|
28
|
-
* SQLite and bun:sqlite, which both default FK enforcement OFF
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
28
|
+
* SQLite and bun:sqlite, which both default FK enforcement OFF — so this
|
|
29
|
+
* is disabled explicitly to restore the behavior the rest of the
|
|
30
|
+
* codebase was written against. `tracer.ts`'s SCHEMA carries no
|
|
31
|
+
* `REFERENCES` clauses at all (removed for SPF #66: rows are not always
|
|
32
|
+
* inserted parent-before-child — e.g. the very first `events` row for a
|
|
33
|
+
* session lands before that session's own `sessions` row commits, and an
|
|
34
|
+
* event recorded outside any phase carries `phase_id: ""`, never NULL —
|
|
35
|
+
* and Cloudflare D1 enforces FKs UNCONDITIONALLY with no way to disable
|
|
36
|
+
* them, so a clause that was already only decorative on local sqlite
|
|
37
|
+
* would crash every D1 session outright). This setting is now a no-op
|
|
38
|
+
* for `tracer.ts`'s own tables specifically, but is kept here as this
|
|
39
|
+
* wrapper's general default — `cli/commands/abort.ts` and `migrate.ts`
|
|
40
|
+
* also open a plain `Database` against the same db file for their own
|
|
41
|
+
* ad-hoc queries, and neither has any reason to want FK enforcement on.
|
|
35
42
|
*/
|
|
36
43
|
export interface DatabaseOptions {
|
|
37
44
|
readonly?: boolean;
|
package/dist/core/sqlite.js
CHANGED
|
@@ -25,13 +25,20 @@
|
|
|
25
25
|
* come back with a null prototype. Both are normalized below so existing
|
|
26
26
|
* `?? null` / `Object.assign(row, ...)` call sites stay honest.
|
|
27
27
|
* - node:sqlite defaults `enableForeignKeyConstraints: true` — unlike plain
|
|
28
|
-
* SQLite and bun:sqlite, which both default FK enforcement OFF
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
28
|
+
* SQLite and bun:sqlite, which both default FK enforcement OFF — so this
|
|
29
|
+
* is disabled explicitly to restore the behavior the rest of the
|
|
30
|
+
* codebase was written against. `tracer.ts`'s SCHEMA carries no
|
|
31
|
+
* `REFERENCES` clauses at all (removed for SPF #66: rows are not always
|
|
32
|
+
* inserted parent-before-child — e.g. the very first `events` row for a
|
|
33
|
+
* session lands before that session's own `sessions` row commits, and an
|
|
34
|
+
* event recorded outside any phase carries `phase_id: ""`, never NULL —
|
|
35
|
+
* and Cloudflare D1 enforces FKs UNCONDITIONALLY with no way to disable
|
|
36
|
+
* them, so a clause that was already only decorative on local sqlite
|
|
37
|
+
* would crash every D1 session outright). This setting is now a no-op
|
|
38
|
+
* for `tracer.ts`'s own tables specifically, but is kept here as this
|
|
39
|
+
* wrapper's general default — `cli/commands/abort.ts` and `migrate.ts`
|
|
40
|
+
* also open a plain `Database` against the same db file for their own
|
|
41
|
+
* ad-hoc queries, and neither has any reason to want FK enforcement on.
|
|
35
42
|
*/
|
|
36
43
|
import { DatabaseSync } from "node:sqlite";
|
|
37
44
|
/**
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TraceDb: the async storage interface the trace database (`Tracer`'s writes,
|
|
3
|
+
* `SfDb`'s reads) is built against — one interface, two backends.
|
|
4
|
+
*
|
|
5
|
+
* WHY ASYNC, WHEN `core/sqlite.ts`'s `Database` IS SYNCHRONOUS: Cloudflare
|
|
6
|
+
* D1's only way to reach a database from a plain Node process (this CLI —
|
|
7
|
+
* not a Cloudflare Worker, which would get a real binding) is its HTTP REST
|
|
8
|
+
* API — a network call, inescapably async. Rather than fake synchrony over
|
|
9
|
+
* that (shelling out to curl, a busy-wait/sync-XHR trick), every caller
|
|
10
|
+
* upstream of storage (`Tracer`'s write methods, `SfDb`'s read methods) is
|
|
11
|
+
* async too — see those modules' own headers for how far that propagates.
|
|
12
|
+
* `LocalTraceDb` below pays that cost for nothing (every call still runs
|
|
13
|
+
* synchronously, immediately; only the return value is wrapped as a settled
|
|
14
|
+
* `Promise`) so the local path stays byte-for-byte the same behavior it
|
|
15
|
+
* always had, just awaited.
|
|
16
|
+
*
|
|
17
|
+
* `query(sql)` mirrors `core/sqlite.ts`'s `Statement` shape exactly, only
|
|
18
|
+
* every method returns a `Promise` — so a caller migrating from `Database`
|
|
19
|
+
* changes `.get(...)` to `await .get(...)` and nothing else.
|
|
20
|
+
*/
|
|
21
|
+
import { type DatabaseOptions } from "./sqlite.ts";
|
|
22
|
+
import type { NormalizedObservabilityDb } from "./data_types.ts";
|
|
23
|
+
export interface AsyncStatement<Row, Params extends unknown[]> {
|
|
24
|
+
get(...params: Params): Promise<Row | null>;
|
|
25
|
+
all(...params: Params): Promise<Row[]>;
|
|
26
|
+
run(...params: Params): Promise<{
|
|
27
|
+
changes: number;
|
|
28
|
+
lastInsertRowid: number | bigint;
|
|
29
|
+
}>;
|
|
30
|
+
}
|
|
31
|
+
export interface TraceDb {
|
|
32
|
+
query<Row = unknown, Params extends unknown[] = unknown[]>(sql: string): AsyncStatement<Row, Params>;
|
|
33
|
+
/** One or more `;`-separated statements — schema DDL and additive migrations. */
|
|
34
|
+
exec(sql: string): Promise<void>;
|
|
35
|
+
close(): Promise<void>;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* A thin async wrapper over the existing synchronous `Database` — every call
|
|
39
|
+
* is dispatched immediately, synchronously, and its result (or thrown error)
|
|
40
|
+
* is handed back as an already-settled `Promise` (via an `async` wrapper, so
|
|
41
|
+
* a synchronous throw becomes a rejection rather than escaping the `Promise`
|
|
42
|
+
* contract). Zero behavior change from `Database` itself; only the signature
|
|
43
|
+
* is async, so this backend and `D1TraceDb` satisfy the exact same `TraceDb`
|
|
44
|
+
* interface.
|
|
45
|
+
*
|
|
46
|
+
* `core/sqlite.ts` is deliberately left untouched — `cli/commands/abort.ts`
|
|
47
|
+
* and `cli/commands/migrate.ts` still open a plain `Database` directly for
|
|
48
|
+
* local-file-specific operations (marking a session aborted, physically
|
|
49
|
+
* relocating the db file) that have no D1 equivalent. See those files' own
|
|
50
|
+
* D1 guards.
|
|
51
|
+
*/
|
|
52
|
+
export declare class LocalTraceDb implements TraceDb {
|
|
53
|
+
private readonly db;
|
|
54
|
+
constructor(dbPath: string, options?: DatabaseOptions);
|
|
55
|
+
query<Row = unknown, Params extends unknown[] = unknown[]>(sql: string): AsyncStatement<Row, Params>;
|
|
56
|
+
exec(sql: string): Promise<void>;
|
|
57
|
+
close(): Promise<void>;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* D1 IS NOT LOCAL WAL SQLITE — DO NOT ASSUME THE SAME LIVE-READ GUARANTEE.
|
|
61
|
+
*
|
|
62
|
+
* `core/tracer.ts`'s header describes the local backend's contract: WAL mode
|
|
63
|
+
* lets `spf ui` read the trace db WHILE an ADW process is still writing to
|
|
64
|
+
* it, in the same instant, because both sides share one file on one
|
|
65
|
+
* filesystem. A D1-backed repo has no such single file to share — `spf ui`
|
|
66
|
+
* and a running chain each speak to D1 over independent HTTP calls, and (per
|
|
67
|
+
* Cloudflare's own docs, https://developers.cloudflare.com/d1/best-practices/read-replication/)
|
|
68
|
+
* a D1 database with read replication enabled offers SEQUENTIAL consistency,
|
|
69
|
+
* not read-your-own-writes by default: a read immediately after a write can
|
|
70
|
+
* land on a replica that has not caught up yet. D1's Sessions API closes that
|
|
71
|
+
* gap with a "bookmark" a caller can pin subsequent reads to, but this
|
|
72
|
+
* adapter does not use it (a bookmark is a client-scoped promise across ONE
|
|
73
|
+
* lightweight connection; a Tracer that mints a fresh request per write and
|
|
74
|
+
* an SfDb serving unrelated browser requests have no session to share it
|
|
75
|
+
* through). This is a deliberate, documented trade-off (SPF #66) — not a bug
|
|
76
|
+
* to fix here: a D1-backed repo's UI may briefly show a slightly-stale trace
|
|
77
|
+
* while a chain is actively writing. Nothing about phase/gate/run OUTCOMES
|
|
78
|
+
* depends on that live read — see tracer.ts's header, "SQLite is the source
|
|
79
|
+
* of truth; nothing downstream of it can affect a phase, a gate, or a run
|
|
80
|
+
* outcome" holds exactly the same way against D1.
|
|
81
|
+
*/
|
|
82
|
+
export declare class D1TraceDb implements TraceDb {
|
|
83
|
+
private readonly accountId;
|
|
84
|
+
private readonly apiToken;
|
|
85
|
+
private readonly databaseId;
|
|
86
|
+
private readonly fetchImpl;
|
|
87
|
+
private readonly baseUrl;
|
|
88
|
+
constructor(config: Extract<NormalizedObservabilityDb, {
|
|
89
|
+
kind: "d1";
|
|
90
|
+
}>,
|
|
91
|
+
/** Injectable so tests never touch the real network — defaults to the global `fetch`. */
|
|
92
|
+
fetchImpl?: typeof fetch);
|
|
93
|
+
query<Row = unknown, Params extends unknown[] = unknown[]>(sql: string): AsyncStatement<Row, Params>;
|
|
94
|
+
/**
|
|
95
|
+
* D1's HTTP `/query` endpoint accepts exactly one statement per `sql`
|
|
96
|
+
* field (see the Workers binding's `prepare()`/`batch()` split — the HTTP
|
|
97
|
+
* endpoint's single-request shape mirrors `prepare()`, not `exec()`);
|
|
98
|
+
* `exec()` here — schema DDL and additive `ALTER TABLE` migrations, always
|
|
99
|
+
* multiple statements — splits the text into individual statements and
|
|
100
|
+
* sends them as one `batch` request instead, which D1 documents as
|
|
101
|
+
* running sequentially and atomically. The split is naive (`;` at
|
|
102
|
+
* statement end, one statement per line-ish chunk) because this only ever
|
|
103
|
+
* runs against `tracer.ts`'s own hand-written `SCHEMA`/`MIGRATIONS`
|
|
104
|
+
* constants — never arbitrary or user-supplied SQL.
|
|
105
|
+
*/
|
|
106
|
+
exec(sql: string): Promise<void>;
|
|
107
|
+
close(): Promise<void>;
|
|
108
|
+
private runOne;
|
|
109
|
+
private post;
|
|
110
|
+
}
|
|
111
|
+
export interface CreateTraceDbOptions {
|
|
112
|
+
/** Local (`sqlite`) only — opens the connection read-only, matching `SfDb`'s reader. */
|
|
113
|
+
readonly?: boolean;
|
|
114
|
+
/** D1 only — injectable for tests. Defaults to the global `fetch`. */
|
|
115
|
+
fetchImpl?: typeof fetch;
|
|
116
|
+
}
|
|
117
|
+
/** The one place `Tracer` and `SfDb` both go to get the right backend for `observability.db`'s resolved kind. */
|
|
118
|
+
export declare function createTraceDb(resolved: NormalizedObservabilityDb, options?: CreateTraceDbOptions): TraceDb;
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TraceDb: the async storage interface the trace database (`Tracer`'s writes,
|
|
3
|
+
* `SfDb`'s reads) is built against — one interface, two backends.
|
|
4
|
+
*
|
|
5
|
+
* WHY ASYNC, WHEN `core/sqlite.ts`'s `Database` IS SYNCHRONOUS: Cloudflare
|
|
6
|
+
* D1's only way to reach a database from a plain Node process (this CLI —
|
|
7
|
+
* not a Cloudflare Worker, which would get a real binding) is its HTTP REST
|
|
8
|
+
* API — a network call, inescapably async. Rather than fake synchrony over
|
|
9
|
+
* that (shelling out to curl, a busy-wait/sync-XHR trick), every caller
|
|
10
|
+
* upstream of storage (`Tracer`'s write methods, `SfDb`'s read methods) is
|
|
11
|
+
* async too — see those modules' own headers for how far that propagates.
|
|
12
|
+
* `LocalTraceDb` below pays that cost for nothing (every call still runs
|
|
13
|
+
* synchronously, immediately; only the return value is wrapped as a settled
|
|
14
|
+
* `Promise`) so the local path stays byte-for-byte the same behavior it
|
|
15
|
+
* always had, just awaited.
|
|
16
|
+
*
|
|
17
|
+
* `query(sql)` mirrors `core/sqlite.ts`'s `Statement` shape exactly, only
|
|
18
|
+
* every method returns a `Promise` — so a caller migrating from `Database`
|
|
19
|
+
* changes `.get(...)` to `await .get(...)` and nothing else.
|
|
20
|
+
*/
|
|
21
|
+
import { mkdirSync } from "node:fs";
|
|
22
|
+
import path from "node:path";
|
|
23
|
+
import { Database } from "./sqlite.js";
|
|
24
|
+
/**
|
|
25
|
+
* Bound on one D1 HTTP request, same `AbortController` + unref'd `setTimeout`
|
|
26
|
+
* pattern `core/otel.ts`'s `flush()` uses for its own fetch calls — mirrored
|
|
27
|
+
* rather than reinvented (see `D1TraceDb.post` below). Set well above otel's
|
|
28
|
+
* own `SEND_TIMEOUT_MS` (2s): telemetry there is lossy and fire-and-forget,
|
|
29
|
+
* so a short timeout just means "drop this batch." A D1 query here is a
|
|
30
|
+
* REAL trace write/read a caller is awaiting — too short a timeout would
|
|
31
|
+
* turn ordinary network jitter into spurious failures — but it must still be
|
|
32
|
+
* BOUNDED, or a hung Cloudflare connection hangs the ADW run indefinitely.
|
|
33
|
+
*/
|
|
34
|
+
const D1_REQUEST_TIMEOUT_MS = 15_000;
|
|
35
|
+
// ── local (sqlite, via core/sqlite.ts's synchronous Database) ──────────────
|
|
36
|
+
/**
|
|
37
|
+
* A thin async wrapper over the existing synchronous `Database` — every call
|
|
38
|
+
* is dispatched immediately, synchronously, and its result (or thrown error)
|
|
39
|
+
* is handed back as an already-settled `Promise` (via an `async` wrapper, so
|
|
40
|
+
* a synchronous throw becomes a rejection rather than escaping the `Promise`
|
|
41
|
+
* contract). Zero behavior change from `Database` itself; only the signature
|
|
42
|
+
* is async, so this backend and `D1TraceDb` satisfy the exact same `TraceDb`
|
|
43
|
+
* interface.
|
|
44
|
+
*
|
|
45
|
+
* `core/sqlite.ts` is deliberately left untouched — `cli/commands/abort.ts`
|
|
46
|
+
* and `cli/commands/migrate.ts` still open a plain `Database` directly for
|
|
47
|
+
* local-file-specific operations (marking a session aborted, physically
|
|
48
|
+
* relocating the db file) that have no D1 equivalent. See those files' own
|
|
49
|
+
* D1 guards.
|
|
50
|
+
*/
|
|
51
|
+
export class LocalTraceDb {
|
|
52
|
+
db;
|
|
53
|
+
constructor(dbPath, options) {
|
|
54
|
+
if (!options?.readonly)
|
|
55
|
+
mkdirSync(path.dirname(dbPath), { recursive: true });
|
|
56
|
+
this.db = new Database(dbPath, options);
|
|
57
|
+
}
|
|
58
|
+
query(sql) {
|
|
59
|
+
const stmt = this.db.query(sql);
|
|
60
|
+
// `async` (not a bare `Promise.resolve(...)` wrapper) so a synchronous
|
|
61
|
+
// throw from the underlying `Database` call — a readonly-connection
|
|
62
|
+
// write, a constraint violation — becomes a REJECTED promise like every
|
|
63
|
+
// other `TraceDb` method, rather than an exception thrown synchronously
|
|
64
|
+
// out of a function whose declared return type is `Promise<...>`. A
|
|
65
|
+
// caller that does `const p = db.query(sql).run(); await p` (not
|
|
66
|
+
// `await db.query(sql).run()` inline) would otherwise never get the
|
|
67
|
+
// chance to catch it.
|
|
68
|
+
return {
|
|
69
|
+
get: async (...params) => stmt.get(...params),
|
|
70
|
+
all: async (...params) => stmt.all(...params),
|
|
71
|
+
run: async (...params) => stmt.run(...params),
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
async exec(sql) {
|
|
75
|
+
this.db.exec(sql);
|
|
76
|
+
}
|
|
77
|
+
async close() {
|
|
78
|
+
this.db.close();
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
// ── remote (Cloudflare D1, via its HTTP REST API) ──────────────────────────
|
|
82
|
+
/**
|
|
83
|
+
* D1 IS NOT LOCAL WAL SQLITE — DO NOT ASSUME THE SAME LIVE-READ GUARANTEE.
|
|
84
|
+
*
|
|
85
|
+
* `core/tracer.ts`'s header describes the local backend's contract: WAL mode
|
|
86
|
+
* lets `spf ui` read the trace db WHILE an ADW process is still writing to
|
|
87
|
+
* it, in the same instant, because both sides share one file on one
|
|
88
|
+
* filesystem. A D1-backed repo has no such single file to share — `spf ui`
|
|
89
|
+
* and a running chain each speak to D1 over independent HTTP calls, and (per
|
|
90
|
+
* Cloudflare's own docs, https://developers.cloudflare.com/d1/best-practices/read-replication/)
|
|
91
|
+
* a D1 database with read replication enabled offers SEQUENTIAL consistency,
|
|
92
|
+
* not read-your-own-writes by default: a read immediately after a write can
|
|
93
|
+
* land on a replica that has not caught up yet. D1's Sessions API closes that
|
|
94
|
+
* gap with a "bookmark" a caller can pin subsequent reads to, but this
|
|
95
|
+
* adapter does not use it (a bookmark is a client-scoped promise across ONE
|
|
96
|
+
* lightweight connection; a Tracer that mints a fresh request per write and
|
|
97
|
+
* an SfDb serving unrelated browser requests have no session to share it
|
|
98
|
+
* through). This is a deliberate, documented trade-off (SPF #66) — not a bug
|
|
99
|
+
* to fix here: a D1-backed repo's UI may briefly show a slightly-stale trace
|
|
100
|
+
* while a chain is actively writing. Nothing about phase/gate/run OUTCOMES
|
|
101
|
+
* depends on that live read — see tracer.ts's header, "SQLite is the source
|
|
102
|
+
* of truth; nothing downstream of it can affect a phase, a gate, or a run
|
|
103
|
+
* outcome" holds exactly the same way against D1.
|
|
104
|
+
*/
|
|
105
|
+
export class D1TraceDb {
|
|
106
|
+
accountId;
|
|
107
|
+
apiToken;
|
|
108
|
+
databaseId;
|
|
109
|
+
fetchImpl;
|
|
110
|
+
baseUrl;
|
|
111
|
+
constructor(config,
|
|
112
|
+
/** Injectable so tests never touch the real network — defaults to the global `fetch`. */
|
|
113
|
+
fetchImpl = fetch) {
|
|
114
|
+
this.databaseId = config.database_id;
|
|
115
|
+
this.accountId = readEnv(config.account_id_env, "D1 account id");
|
|
116
|
+
this.apiToken = readEnv(config.api_token_env, "D1 API token");
|
|
117
|
+
this.fetchImpl = fetchImpl;
|
|
118
|
+
this.baseUrl = `https://api.cloudflare.com/client/v4/accounts/${this.accountId}/d1/database/${this.databaseId}/query`;
|
|
119
|
+
}
|
|
120
|
+
query(sql) {
|
|
121
|
+
return {
|
|
122
|
+
get: async (...params) => {
|
|
123
|
+
const rows = await this.runOne(sql, params);
|
|
124
|
+
return rows.results[0] ?? null;
|
|
125
|
+
},
|
|
126
|
+
all: async (...params) => {
|
|
127
|
+
const rows = await this.runOne(sql, params);
|
|
128
|
+
return rows.results;
|
|
129
|
+
},
|
|
130
|
+
run: async (...params) => {
|
|
131
|
+
const rows = await this.runOne(sql, params);
|
|
132
|
+
return { changes: rows.meta.changes ?? 0, lastInsertRowid: rows.meta.last_row_id ?? 0 };
|
|
133
|
+
},
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* D1's HTTP `/query` endpoint accepts exactly one statement per `sql`
|
|
138
|
+
* field (see the Workers binding's `prepare()`/`batch()` split — the HTTP
|
|
139
|
+
* endpoint's single-request shape mirrors `prepare()`, not `exec()`);
|
|
140
|
+
* `exec()` here — schema DDL and additive `ALTER TABLE` migrations, always
|
|
141
|
+
* multiple statements — splits the text into individual statements and
|
|
142
|
+
* sends them as one `batch` request instead, which D1 documents as
|
|
143
|
+
* running sequentially and atomically. The split is naive (`;` at
|
|
144
|
+
* statement end, one statement per line-ish chunk) because this only ever
|
|
145
|
+
* runs against `tracer.ts`'s own hand-written `SCHEMA`/`MIGRATIONS`
|
|
146
|
+
* constants — never arbitrary or user-supplied SQL.
|
|
147
|
+
*/
|
|
148
|
+
async exec(sql) {
|
|
149
|
+
const statements = splitStatements(sql);
|
|
150
|
+
if (statements.length === 0)
|
|
151
|
+
return;
|
|
152
|
+
const body = { batch: statements.map((s) => ({ sql: s })) };
|
|
153
|
+
await this.post(body);
|
|
154
|
+
}
|
|
155
|
+
close() {
|
|
156
|
+
// Stateless HTTP — no connection held open to release.
|
|
157
|
+
return Promise.resolve();
|
|
158
|
+
}
|
|
159
|
+
async runOne(sql, params) {
|
|
160
|
+
const body = { sql, params: params.length > 0 ? params : undefined };
|
|
161
|
+
const results = await this.post(body);
|
|
162
|
+
const first = results[0];
|
|
163
|
+
if (!first)
|
|
164
|
+
throw new Error(`D1 query returned no result set: ${sql}`);
|
|
165
|
+
return { results: (first.results ?? []), meta: first.meta ?? {} };
|
|
166
|
+
}
|
|
167
|
+
async post(body) {
|
|
168
|
+
// Same shape as `otel.ts`'s `flush()`: an `AbortController` whose timer
|
|
169
|
+
// is unref'd (never the reason this process lingers) and always cleared,
|
|
170
|
+
// win or lose. Unlike otel, a timeout here IS a thrown error — this
|
|
171
|
+
// request is a real trace read/write a caller is awaiting, not a
|
|
172
|
+
// best-effort fire-and-forget send.
|
|
173
|
+
//
|
|
174
|
+
// The timer is NOT cleared the moment `fetch` resolves with a `Response`
|
|
175
|
+
// — resolving only means headers have arrived; the body can still be
|
|
176
|
+
// in flight. `response.text()` is read inside this SAME try, under the
|
|
177
|
+
// SAME `controller.signal`, so a server that sends headers and then
|
|
178
|
+
// stalls the body still gets aborted at `D1_REQUEST_TIMEOUT_MS` (per the
|
|
179
|
+
// fetch spec, aborting a request's signal also aborts an in-progress
|
|
180
|
+
// body read) instead of hanging forever — exactly what this timeout
|
|
181
|
+
// exists to prevent.
|
|
182
|
+
const controller = new AbortController();
|
|
183
|
+
const timer = setTimeout(() => controller.abort(), D1_REQUEST_TIMEOUT_MS);
|
|
184
|
+
timer.unref?.();
|
|
185
|
+
let response;
|
|
186
|
+
let text;
|
|
187
|
+
try {
|
|
188
|
+
response = await this.fetchImpl(this.baseUrl, {
|
|
189
|
+
method: "POST",
|
|
190
|
+
headers: {
|
|
191
|
+
Authorization: `Bearer ${this.apiToken}`,
|
|
192
|
+
"Content-Type": "application/json",
|
|
193
|
+
},
|
|
194
|
+
body: JSON.stringify(body),
|
|
195
|
+
signal: controller.signal,
|
|
196
|
+
});
|
|
197
|
+
text = await response.text();
|
|
198
|
+
}
|
|
199
|
+
catch (error) {
|
|
200
|
+
const timedOut = error.name === "AbortError";
|
|
201
|
+
throw new Error(timedOut
|
|
202
|
+
? `D1 request timed out after ${D1_REQUEST_TIMEOUT_MS}ms`
|
|
203
|
+
: `D1 request failed: ${error.message ?? error}`);
|
|
204
|
+
}
|
|
205
|
+
finally {
|
|
206
|
+
clearTimeout(timer);
|
|
207
|
+
}
|
|
208
|
+
let parsed;
|
|
209
|
+
try {
|
|
210
|
+
parsed = text ? JSON.parse(text) : undefined;
|
|
211
|
+
}
|
|
212
|
+
catch {
|
|
213
|
+
// A non-JSON body (e.g. a Cloudflare edge error page) falls through to
|
|
214
|
+
// the HTTP-status branch below with the raw text as context.
|
|
215
|
+
}
|
|
216
|
+
if (!response.ok) {
|
|
217
|
+
const message = parsed?.errors?.map((e) => e.message).join("; ") || text.slice(0, 500) || response.statusText;
|
|
218
|
+
throw new Error(`D1 HTTP ${response.status}: ${message}`);
|
|
219
|
+
}
|
|
220
|
+
if (!parsed || parsed.success !== true) {
|
|
221
|
+
const message = parsed?.errors?.map((e) => e.message).join("; ") || "D1 reported failure with no error message";
|
|
222
|
+
throw new Error(`D1 query failed: ${message}`);
|
|
223
|
+
}
|
|
224
|
+
// A `success:true` envelope that omits `result` entirely (seen from some
|
|
225
|
+
// D1 edge responses) is still success — treat it as zero statements
|
|
226
|
+
// rather than throwing `TypeError: parsed.result is not iterable`.
|
|
227
|
+
const result = parsed.result ?? [];
|
|
228
|
+
for (const item of result) {
|
|
229
|
+
if (item.success === false) {
|
|
230
|
+
throw new Error(`D1 statement failed: ${parsed.errors?.map((e) => e.message).join("; ") || "unknown error"}`);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
return result;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* Split a `;`-terminated multi-statement string into individual statements,
|
|
238
|
+
* dropping blanks and comment-only lines.
|
|
239
|
+
*
|
|
240
|
+
* `tracer.ts`'s `SCHEMA` carries `--`-style trailing comments on several
|
|
241
|
+
* column definitions (e.g. `archived INTEGER DEFAULT 0 -- ...; never by a
|
|
242
|
+
* run`), and those comments themselves contain `;` — a naive `sql.split(";")`
|
|
243
|
+
* shreds the enclosing `CREATE TABLE` into fragments right in the middle of
|
|
244
|
+
* a comment, producing statements no SQL engine accepts (an unterminated
|
|
245
|
+
* `CREATE TABLE ... (`, an orphaned comment tail). This strips every `--` to
|
|
246
|
+
* end-of-line FIRST, so a `;` inside a comment can no longer be mistaken for
|
|
247
|
+
* a statement terminator, then splits what's left. Still only sound for
|
|
248
|
+
* `tracer.ts`'s own hand-written SCHEMA/MIGRATIONS constants — not general
|
|
249
|
+
* SQL (a `;` or `--` inside a quoted string literal is not accounted for,
|
|
250
|
+
* and neither constant ever uses one).
|
|
251
|
+
*/
|
|
252
|
+
function splitStatements(sql) {
|
|
253
|
+
const withoutLineComments = sql
|
|
254
|
+
.split("\n")
|
|
255
|
+
.map((line) => {
|
|
256
|
+
const commentAt = line.indexOf("--");
|
|
257
|
+
return commentAt === -1 ? line : line.slice(0, commentAt);
|
|
258
|
+
})
|
|
259
|
+
.join("\n");
|
|
260
|
+
return withoutLineComments
|
|
261
|
+
.split(";")
|
|
262
|
+
.map((s) => s.trim())
|
|
263
|
+
.filter((s) => s.length > 0);
|
|
264
|
+
}
|
|
265
|
+
function readEnv(name, what) {
|
|
266
|
+
const value = (process.env[name] ?? "").trim();
|
|
267
|
+
if (!value) {
|
|
268
|
+
throw new Error(`${what} is not set — the observability.db config names env var ${JSON.stringify(name)}, but it is empty or unset`);
|
|
269
|
+
}
|
|
270
|
+
return value;
|
|
271
|
+
}
|
|
272
|
+
/** The one place `Tracer` and `SfDb` both go to get the right backend for `observability.db`'s resolved kind. */
|
|
273
|
+
export function createTraceDb(resolved, options) {
|
|
274
|
+
if (resolved.kind === "sqlite") {
|
|
275
|
+
return new LocalTraceDb(resolved.path, options?.readonly ? { readonly: true } : undefined);
|
|
276
|
+
}
|
|
277
|
+
return new D1TraceDb(resolved, options?.fetchImpl);
|
|
278
|
+
}
|