@gr8ful/spf 0.15.0 → 0.17.0

package/dist/core/trace_db.js

@@ -0,0 +1,278 @@
+/**
+ * TraceDb: the async storage interface the trace database (`Tracer`'s writes,
+ * `SfDb`'s reads) is built against — one interface, two backends.
+ *
+ * WHY ASYNC, WHEN `core/sqlite.ts`'s `Database` IS SYNCHRONOUS: Cloudflare
+ * D1's only way to reach a database from a plain Node process (this CLI —
+ * not a Cloudflare Worker, which would get a real binding) is its HTTP REST
+ * API — a network call, inescapably async. Rather than fake synchrony over
+ * that (shelling out to curl, a busy-wait/sync-XHR trick), every caller
+ * upstream of storage (`Tracer`'s write methods, `SfDb`'s read methods) is
+ * async too — see those modules' own headers for how far that propagates.
+ * `LocalTraceDb` below pays that cost for nothing (every call still runs
+ * synchronously, immediately; only the return value is wrapped as a settled
+ * `Promise`) so the local path stays byte-for-byte the same behavior it
+ * always had, just awaited.
+ *
+ * `query(sql)` mirrors `core/sqlite.ts`'s `Statement` shape exactly, only
+ * every method returns a `Promise` — so a caller migrating from `Database`
+ * changes `.get(...)` to `await .get(...)` and nothing else.
+ */
+import { mkdirSync } from "node:fs";
+import path from "node:path";
+import { Database } from "./sqlite.js";
+/**
+ * Bound on one D1 HTTP request, same `AbortController` + unref'd `setTimeout`
+ * pattern `core/otel.ts`'s `flush()` uses for its own fetch calls — mirrored
+ * rather than reinvented (see `D1TraceDb.post` below). Set well above otel's
+ * own `SEND_TIMEOUT_MS` (2s): telemetry there is lossy and fire-and-forget,
+ * so a short timeout just means "drop this batch." A D1 query here is a
+ * REAL trace write/read a caller is awaiting — too short a timeout would
+ * turn ordinary network jitter into spurious failures — but it must still be
+ * BOUNDED, or a hung Cloudflare connection hangs the ADW run indefinitely.
+ */
+const D1_REQUEST_TIMEOUT_MS = 15_000;
+// ── local (sqlite, via core/sqlite.ts's synchronous Database) ──────────────
+/**
+ * A thin async wrapper over the existing synchronous `Database` — every call
+ * is dispatched immediately, synchronously, and its result (or thrown error)
+ * is handed back as an already-settled `Promise` (via an `async` wrapper, so
+ * a synchronous throw becomes a rejection rather than escaping the `Promise`
+ * contract). Zero behavior change from `Database` itself; only the signature
+ * is async, so this backend and `D1TraceDb` satisfy the exact same `TraceDb`
+ * interface.
+ *
+ * `core/sqlite.ts` is deliberately left untouched — `cli/commands/abort.ts`
+ * and `cli/commands/migrate.ts` still open a plain `Database` directly for
+ * local-file-specific operations (marking a session aborted, physically
+ * relocating the db file) that have no D1 equivalent. See those files' own
+ * D1 guards.
+ */
+export class LocalTraceDb {
+    db;
+    constructor(dbPath, options) {
+        if (!options?.readonly)
+            mkdirSync(path.dirname(dbPath), { recursive: true });
+        this.db = new Database(dbPath, options);
+    }
+    query(sql) {
+        const stmt = this.db.query(sql);
+        // `async` (not a bare `Promise.resolve(...)` wrapper) so a synchronous
+        // throw from the underlying `Database` call — a readonly-connection
+        // write, a constraint violation — becomes a REJECTED promise like every
+        // other `TraceDb` method, rather than an exception thrown synchronously
+        // out of a function whose declared return type is `Promise<...>`. A
+        // caller that does `const p = db.query(sql).run(); await p` (not
+        // `await db.query(sql).run()` inline) would otherwise never get the
+        // chance to catch it.
+        return {
+            get: async (...params) => stmt.get(...params),
+            all: async (...params) => stmt.all(...params),
+            run: async (...params) => stmt.run(...params),
+        };
+    }
+    async exec(sql) {
+        this.db.exec(sql);
+    }
+    async close() {
+        this.db.close();
+    }
+}
+// ── remote (Cloudflare D1, via its HTTP REST API) ──────────────────────────
+/**
+ * D1 IS NOT LOCAL WAL SQLITE — DO NOT ASSUME THE SAME LIVE-READ GUARANTEE.
+ *
+ * `core/tracer.ts`'s header describes the local backend's contract: WAL mode
+ * lets `spf ui` read the trace db WHILE an ADW process is still writing to
+ * it, in the same instant, because both sides share one file on one
+ * filesystem. A D1-backed repo has no such single file to share — `spf ui`
+ * and a running chain each speak to D1 over independent HTTP calls, and (per
+ * Cloudflare's own docs, https://developers.cloudflare.com/d1/best-practices/read-replication/)
+ * a D1 database with read replication enabled offers SEQUENTIAL consistency,
+ * not read-your-own-writes by default: a read immediately after a write can
+ * land on a replica that has not caught up yet. D1's Sessions API closes that
+ * gap with a "bookmark" a caller can pin subsequent reads to, but this
+ * adapter does not use it (a bookmark is a client-scoped promise across ONE
+ * lightweight connection; a Tracer that mints a fresh request per write and
+ * an SfDb serving unrelated browser requests have no session to share it
+ * through). This is a deliberate, documented trade-off (SPF #66) — not a bug
+ * to fix here: a D1-backed repo's UI may briefly show a slightly-stale trace
+ * while a chain is actively writing. Nothing about phase/gate/run OUTCOMES
+ * depends on that live read — see tracer.ts's header, "SQLite is the source
+ * of truth; nothing downstream of it can affect a phase, a gate, or a run
+ * outcome" holds exactly the same way against D1.
+ */
+export class D1TraceDb {
+    accountId;
+    apiToken;
+    databaseId;
+    fetchImpl;
+    baseUrl;
+    constructor(config, 
+    /** Injectable so tests never touch the real network — defaults to the global `fetch`. */
+    fetchImpl = fetch) {
+        this.databaseId = config.database_id;
+        this.accountId = readEnv(config.account_id_env, "D1 account id");
+        this.apiToken = readEnv(config.api_token_env, "D1 API token");
+        this.fetchImpl = fetchImpl;
+        this.baseUrl = `https://api.cloudflare.com/client/v4/accounts/${this.accountId}/d1/database/${this.databaseId}/query`;
+    }
+    query(sql) {
+        return {
+            get: async (...params) => {
+                const rows = await this.runOne(sql, params);
+                return rows.results[0] ?? null;
+            },
+            all: async (...params) => {
+                const rows = await this.runOne(sql, params);
+                return rows.results;
+            },
+            run: async (...params) => {
+                const rows = await this.runOne(sql, params);
+                return { changes: rows.meta.changes ?? 0, lastInsertRowid: rows.meta.last_row_id ?? 0 };
+            },
+        };
+    }
+    /**
+     * D1's HTTP `/query` endpoint accepts exactly one statement per `sql`
+     * field (see the Workers binding's `prepare()`/`batch()` split — the HTTP
+     * endpoint's single-request shape mirrors `prepare()`, not `exec()`);
+     * `exec()` here — schema DDL and additive `ALTER TABLE` migrations, always
+     * multiple statements — splits the text into individual statements and
+     * sends them as one `batch` request instead, which D1 documents as
+     * running sequentially and atomically. The split is naive (`;` at
+     * statement end, one statement per line-ish chunk) because this only ever
+     * runs against `tracer.ts`'s own hand-written `SCHEMA`/`MIGRATIONS`
+     * constants — never arbitrary or user-supplied SQL.
+     */
+    async exec(sql) {
+        const statements = splitStatements(sql);
+        if (statements.length === 0)
+            return;
+        const body = { batch: statements.map((s) => ({ sql: s })) };
+        await this.post(body);
+    }
+    close() {
+        // Stateless HTTP — no connection held open to release.
+        return Promise.resolve();
+    }
+    async runOne(sql, params) {
+        const body = { sql, params: params.length > 0 ? params : undefined };
+        const results = await this.post(body);
+        const first = results[0];
+        if (!first)
+            throw new Error(`D1 query returned no result set: ${sql}`);
+        return { results: (first.results ?? []), meta: first.meta ?? {} };
+    }
+    async post(body) {
+        // Same shape as `otel.ts`'s `flush()`: an `AbortController` whose timer
+        // is unref'd (never the reason this process lingers) and always cleared,
+        // win or lose. Unlike otel, a timeout here IS a thrown error — this
+        // request is a real trace read/write a caller is awaiting, not a
+        // best-effort fire-and-forget send.
+        //
+        // The timer is NOT cleared the moment `fetch` resolves with a `Response`
+        // — resolving only means headers have arrived; the body can still be
+        // in flight. `response.text()` is read inside this SAME try, under the
+        // SAME `controller.signal`, so a server that sends headers and then
+        // stalls the body still gets aborted at `D1_REQUEST_TIMEOUT_MS` (per the
+        // fetch spec, aborting a request's signal also aborts an in-progress
+        // body read) instead of hanging forever — exactly what this timeout
+        // exists to prevent.
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), D1_REQUEST_TIMEOUT_MS);
+        timer.unref?.();
+        let response;
+        let text;
+        try {
+            response = await this.fetchImpl(this.baseUrl, {
+                method: "POST",
+                headers: {
+                    Authorization: `Bearer ${this.apiToken}`,
+                    "Content-Type": "application/json",
+                },
+                body: JSON.stringify(body),
+                signal: controller.signal,
+            });
+            text = await response.text();
+        }
+        catch (error) {
+            const timedOut = error.name === "AbortError";
+            throw new Error(timedOut
+                ? `D1 request timed out after ${D1_REQUEST_TIMEOUT_MS}ms`
+                : `D1 request failed: ${error.message ?? error}`);
+        }
+        finally {
+            clearTimeout(timer);
+        }
+        let parsed;
+        try {
+            parsed = text ? JSON.parse(text) : undefined;
+        }
+        catch {
+            // A non-JSON body (e.g. a Cloudflare edge error page) falls through to
+            // the HTTP-status branch below with the raw text as context.
+        }
+        if (!response.ok) {
+            const message = parsed?.errors?.map((e) => e.message).join("; ") || text.slice(0, 500) || response.statusText;
+            throw new Error(`D1 HTTP ${response.status}: ${message}`);
+        }
+        if (!parsed || parsed.success !== true) {
+            const message = parsed?.errors?.map((e) => e.message).join("; ") || "D1 reported failure with no error message";
+            throw new Error(`D1 query failed: ${message}`);
+        }
+        // A `success:true` envelope that omits `result` entirely (seen from some
+        // D1 edge responses) is still success — treat it as zero statements
+        // rather than throwing `TypeError: parsed.result is not iterable`.
+        const result = parsed.result ?? [];
+        for (const item of result) {
+            if (item.success === false) {
+                throw new Error(`D1 statement failed: ${parsed.errors?.map((e) => e.message).join("; ") || "unknown error"}`);
+            }
+        }
+        return result;
+    }
+}
+/**
+ * Split a `;`-terminated multi-statement string into individual statements,
+ * dropping blanks and comment-only lines.
+ *
+ * `tracer.ts`'s `SCHEMA` carries `--`-style trailing comments on several
+ * column definitions (e.g. `archived INTEGER DEFAULT 0 -- ...; never by a
+ * run`), and those comments themselves contain `;` — a naive `sql.split(";")`
+ * shreds the enclosing `CREATE TABLE` into fragments right in the middle of
+ * a comment, producing statements no SQL engine accepts (an unterminated
+ * `CREATE TABLE ... (`, an orphaned comment tail). This strips every `--` to
+ * end-of-line FIRST, so a `;` inside a comment can no longer be mistaken for
+ * a statement terminator, then splits what's left. Still only sound for
+ * `tracer.ts`'s own hand-written SCHEMA/MIGRATIONS constants — not general
+ * SQL (a `;` or `--` inside a quoted string literal is not accounted for,
+ * and neither constant ever uses one).
+ */
+function splitStatements(sql) {
+    const withoutLineComments = sql
+        .split("\n")
+        .map((line) => {
+        const commentAt = line.indexOf("--");
+        return commentAt === -1 ? line : line.slice(0, commentAt);
+    })
+        .join("\n");
+    return withoutLineComments
+        .split(";")
+        .map((s) => s.trim())
+        .filter((s) => s.length > 0);
+}
+function readEnv(name, what) {
+    const value = (process.env[name] ?? "").trim();
+    if (!value) {
+        throw new Error(`${what} is not set — the observability.db config names env var ${JSON.stringify(name)}, but it is empty or unset`);
+    }
+    return value;
+}
+/** The one place `Tracer` and `SfDb` both go to get the right backend for `observability.db`'s resolved kind. */
+export function createTraceDb(resolved, options) {
+    if (resolved.kind === "sqlite") {
+        return new LocalTraceDb(resolved.path, options?.readonly ? { readonly: true } : undefined);
+    }
+    return new D1TraceDb(resolved, options?.fetchImpl);
+}

package/dist/core/tracer.d.ts

@@ -1,13 +1,28 @@
 /**
- * Tracer: every event lands in JSONL and SQLite AS IT HAPPENS.
+ * Tracer: every event lands in JSONL and the trace db AS IT HAPPENS.
  *
- * Files are the raw record; spf.db is the queryable mirror the UI polls.
- * WAL mode so the UI can read while ADW processes write.
+ * Files are the raw record; the trace db (local sqlite, or remote D1 — see
+ * `core/trace_db.ts`) is the queryable mirror the UI polls. For the LOCAL
+ * backend, WAL mode lets the UI read while ADW processes write, in the same
+ * instant, because both sides share one file on one filesystem.
+ *
+ * THE D1 BACKEND DOES NOT MAKE THAT SAME PROMISE — see `core/trace_db.ts`'s
+ * `D1TraceDb` doc comment for the consistency model it actually has and why
+ * this is a deliberate, documented trade-off (SPF #66) rather than a gap to
+ * close here.
  *
  * No push transport in the CONTROL flow — that is always, still, and only:
- * agents -> sqlite -> web ui. SQLite is the source of truth; nothing
+ * agents -> trace db -> web ui. The trace db is the source of truth; nothing
  * downstream of it can affect a phase, a gate, or a run outcome.
  *
+ * WHY EVERY WRITE METHOD BELOW IS ASYNC: the D1 backend's only way to reach
+ * a database from this plain Node CLI is its HTTP REST API — a network call.
+ * `core/trace_db.ts`'s `TraceDb` interface is async for that reason, and
+ * every write method here is a thin `await this.db...` over it — see that
+ * module's header. For the LOCAL backend this costs nothing but the syntax:
+ * `LocalTraceDb` wraps the exact same synchronous `Database` calls this file
+ * always made, immediately resolved.
+ *
  * The one amendment: when (and only when) `observability.otel` is configured,
  * each write method below ends with a single fan-out line to an optional
  * OtelExporter — a lossy, allowlisted PROJECTION of what was just written,
@@ -21,43 +36,58 @@
  * `processStart`/`processEnd` (pids) — have NO fan-out line on purpose. Do not
  * add one.
  */
-import { Database } from "./sqlite.ts";
-import type { AgentConfig, EventRecord, GateReport, Phase } from "./data_types.ts";
+import type { AgentConfig, EventRecord, GateReport, NormalizedObservabilityDb, Phase } from "./data_types.ts";
 import type { OtelExporter } from "./otel.ts";
+import { type TraceDb } from "./trace_db.ts";
 export declare class Tracer {
-    db: Database;
-    dbPath: string;
+    db: TraceDb;
+    /** Absolute local sqlite path — `null` for a `kind:"d1"` config (`Console.sessionFinished`'s "db" line falls back to `runner.ts`'s `describeObservabilityDb` for a human label in that case). */
+    dbPath: string | null;
     eventsJsonl: string;
     /** `null` unless `observability.otel` is configured — see the header. */
     otel: OtelExporter | null;
-    constructor(dbPath: string, eventsJsonl: string, otel?: OtelExporter | null);
+    private constructor();
+    /**
+     * Open (or create) the trace db, run schema + migrations, and return a
+     * ready-to-use Tracer. The one constructor path — a plain `new Tracer(...)`
+     * cannot do this work itself because opening a D1-backed db means awaited
+     * HTTP calls before the schema exists, and a constructor cannot be async.
+     *
+     * `dbConfig` accepts either a bare local sqlite path (sugar for
+     * `{kind:"sqlite",path}` — every test's shorthand, and what every caller
+     * before SPF #66 passed) or a normalized `observability.db` descriptor
+     * (`paths.resolveDataPaths(...).db` — sqlite or d1).
+     */
+    static open(dbConfig: NormalizedObservabilityDb | string, eventsJsonl: string, otel?: OtelExporter | null, traceDbOptions?: {
+        fetchImpl?: typeof fetch;
+    }): Promise<Tracer>;
     /**
-     * Close the sqlite handle. A one-shot CLI process never needs this — it
+     * Close the trace db handle. A one-shot CLI process never needs this — it
      * exits right after its one Tracer anyway — but `spf watch`'s daemon loop
-     * builds a fresh Tracer (and a fresh `new Database(dbPath)`) per claimed
-     * issue, in-process, for the life of the daemon; without this, every
-     * issue's handle stayed open forever. Called from `session.ts`'s
-     * `finalize()`, once a run's own dispatch has fully settled — see its
-     * comment for why that timing is safe.
+     * builds a fresh Tracer (and a fresh local db connection, or a fresh
+     * stateless D1 client) per claimed issue, in-process, for the life of the
+     * daemon; without this, every issue's local handle stayed open forever.
+     * Called from `session.ts`'s `finalize()`, once a run's own dispatch has
+     * fully settled — see its comment for why that timing is safe. A no-op for
+     * `D1TraceDb` (stateless HTTP — nothing to release).
      */
-    close(): void;
+    close(): Promise<void>;
     /** Additive column migrations, so a db from an older SPF still opens. */
     private migrate;
     /**
-     * The ONE door to the optional otel projection, and the only reason a fan-out
-     * line is safe to put at the end of a synchronous write method: it is a
-     * no-op when unconfigured, and it swallows everything. An exporter bug, a
-     * malformed span, an exhausted queue — none of it may ever surface as a
-     * failed phase, because export is not allowed to dispose of anything. The
+     * The ONE door to the optional otel projection. It is a no-op when
+     * unconfigured, and it swallows everything: an exporter bug, a malformed
+     * span, an exhausted queue — none of it may ever surface as a failed
+     * phase, because export is not allowed to dispose of anything. The
      * exporter's own methods are synchronous enqueues; the network happens later,
      * on an unref'd timer.
      */
     private fanOut;
-    event(record: EventRecord): string;
-    sessionStart(adwId: string, engineer: string, adwName?: string | null): void;
-    sessionRequest(adwId: string, request: string): void;
-    sessionFinish(adwId: string, ok: boolean): void;
-    sessionAddUsage(adwId: string, tokens: number, cost: number): void;
+    event(record: EventRecord): Promise<string>;
+    sessionStart(adwId: string, engineer: string, adwName?: string | null): Promise<void>;
+    sessionRequest(adwId: string, request: string): Promise<void>;
+    sessionFinish(adwId: string, ok: boolean): Promise<void>;
+    sessionAddUsage(adwId: string, tokens: number, cost: number): Promise<void>;
     /**
      * Record a live process for this run.
      *
@@ -66,11 +96,11 @@ export declare class Tracer {
      * belongs to. Writing it here makes the trace the answer to "what is this
      * run running, and how do I stop it".
      */
-    processStart(adwId: string, kind: string, name: string, pid: number, command: string): void;
+    processStart(adwId: string, kind: string, name: string, pid: number, command: string): Promise<void>;
     /** Mark the newest live row for this pid as finished. */
-    processEnd(adwId: string, pid: number): void;
+    processEnd(adwId: string, pid: number): Promise<void>;
     /** Close out every live row for a run — called when the session ends. */
-    processesEndAll(adwId: string): void;
+    processesEndAll(adwId: string): Promise<void>;
     /**
      * Highest seq already recorded for this session; 0 when it is new.
      *
@@ -79,11 +109,11 @@ export declare class Tracer {
      * ordering) and `phase_id` (silently overwriting a row through the
      * phase_upsert conflict clause).
      */
-    maxPhaseSeq(adwId: string): number;
-    phaseUpsert(phase: Phase): void;
-    envelopeRow(phase: Phase, agent: string, outputType: string, payloadJson: string, valid: boolean, attempt: number): void;
+    maxPhaseSeq(adwId: string): Promise<number>;
+    phaseUpsert(phase: Phase): Promise<void>;
+    envelopeRow(phase: Phase, agent: string, outputType: string, payloadJson: string, valid: boolean, attempt: number): Promise<void>;
     /** The report carries both the verdict and the evidence behind it. */
-    gateRow(phase: Phase, gate: string, report: GateReport, attempt: number): void;
+    gateRow(phase: Phase, gate: string, report: GateReport, attempt: number): Promise<void>;
     /**
      * The agent's config row is the source of truth for its label and color.
      *
@@ -91,5 +121,5 @@ export declare class Tracer {
      * wants one number per agent — the latest — and a session that runs the
      * same agent twice overwrites it, exactly like model and session_id.
      */
-    agentSessionRow(adwId: string, agent: AgentConfig, sessionId: string, contextTokens?: number, contextWindow?: number): void;
+    agentSessionRow(adwId: string, agent: AgentConfig, sessionId: string, contextTokens?: number, contextWindow?: number): Promise<void>;
 }