@cortexkit/opencode-magic-context 0.49.6 → 0.49.8

package/src/shared/sqlite-foreground-wait.test.ts

@@ -1,4 +1,4 @@
-import { expect, test } from "bun:test";
+import { expect, spyOn, test } from "bun:test";
 import { rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
@@ -113,6 +113,220 @@ for (const holdMs of [1000, 2500]) {
     }, 30_000);
 }
 
+for (const holdMs of [1000, 2500]) {
+    test(`slow uncontended SQL preserves admission for a ${holdMs}ms autocommit lock`, async () => {
+        const { db, path, close } = fixture();
+        let locker: SqliteWriteLocker | undefined;
+        let served = false;
+        let refusals = 0;
+        const output = {
+            messages: [
+                {
+                    info: { id: "u", sessionID: "slow", role: "user" },
+                    parts: [{ type: "text", text: "raw" }],
+                },
+            ],
+        };
+        const handler = createMessagesTransformHandler({
+            onStorageBusyRefusal: async () => {
+                refusals++;
+            },
+            magicContext: {
+                "experimental.chat.messages.transform": async (_input, result) => {
+                    // Spend three seconds inside successful SQL, not between writes.
+                    // Recursive work keeps this independent of CPU speed and UDF APIs.
+                    const cte =
+                        "WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<1000000)";
+                    const slowWrite = db.prepare(
+                        `${cte} INSERT INTO result SELECT sum(x) FROM n RETURNING value`,
+                    );
+                    const slowRead = db.prepare(`${cte} SELECT sum(x) FROM n`);
+                    const started = performance.now();
+                    do {
+                        slowWrite.all();
+                        slowRead.all();
+                    } while (performance.now() - started < 3000);
+                    locker = await startSqliteWriteLocker(path, holdMs);
+                    db.prepare("INSERT INTO result VALUES ('managed')").run();
+                    (result.messages[0].parts[0] as { text: string }).text = "managed";
+                },
+            },
+        });
+        try {
+            const pass = handler({}, output as Parameters<typeof handler>[1]).then(() => {
+                served = true;
+            });
+            if (holdMs === 1000) {
+                await pass;
+                expect(served).toBe(true);
+                expect(refusals).toBe(0);
+                expect(output.messages[0].parts[0].text).toBe("managed");
+            } else {
+                await expect(pass).rejects.toBeInstanceOf(StorageBusyRefusalError);
+                expect(served).toBe(false);
+                expect(refusals).toBe(1);
+                expect(output.messages[0].parts[0].text).toBe("raw");
+            }
+            expect(
+                db.prepare("SELECT count(*) AS n FROM result WHERE value = 'managed'").get(),
+            ).toEqual({ n: holdMs === 1000 ? 1 : 0 });
+        } finally {
+            if (locker) expect(await locker.exited).toBe(0);
+            close();
+        }
+    }, 30_000);
+}
+
+test("BUSY autocommit probes do not double-apply INSERT OR IGNORE, UPSERT or RETURNING", async () => {
+    const { db, path, close } = fixture();
+    db.exec(
+        "CREATE TABLE counter(id INTEGER PRIMARY KEY, n INTEGER); INSERT INTO counter VALUES (1, 0)",
+    );
+    const forms = [
+        ["INSERT OR IGNORE INTO counter VALUES (2, 1)", "run"],
+        ["INSERT INTO counter VALUES (1, 1) ON CONFLICT(id) DO UPDATE SET n = n + 1", "run"],
+        ["UPDATE counter SET n = n + 1 WHERE id = 1 RETURNING n", "get"],
+        [
+            "INSERT INTO counter VALUES (1, 1) ON CONFLICT(id) DO UPDATE SET n = n + 1 RETURNING n",
+            "all",
+        ],
+    ] as const;
+    try {
+        for (const [sql, method] of forms) {
+            const blocker = new Database(path);
+            blocker.exec("BEGIN IMMEDIATE");
+            db.exec("PRAGMA busy_timeout=0");
+            const before = db.prepare("SELECT * FROM counter ORDER BY id").all();
+            try {
+                expect(() => db.prepare(sql)[method]()).toThrow("locked");
+                expect(db.prepare("SELECT * FROM counter ORDER BY id").all()).toEqual(before);
+                expect(db.inTransaction).toBe(false);
+            } finally {
+                blocker.exec("ROLLBACK");
+                blocker.close();
+            }
+            const locker = await startSqliteWriteLocker(path, 100);
+            try {
+                await withSqliteTransformPass(() => db.prepare(sql)[method]());
+            } finally {
+                expect(await locker.exited).toBe(0);
+            }
+        }
+        expect(db.prepare("SELECT * FROM counter ORDER BY id").all()).toEqual([
+            { id: 1, n: 3 },
+            { id: 2, n: 1 },
+        ]);
+    } finally {
+        close();
+    }
+}, 30_000);
+
+test("many contended autocommit writes share at most two seconds of lock waiting", async () => {
+    const { db, path, close } = fixture();
+    const blocker = new Database(path);
+    blocker.exec("BEGIN IMMEDIATE");
+    const timeouts: number[] = [];
+    const nativeExec = db.exec.bind(db);
+    const exec = spyOn(db, "exec").mockImplementation((sql) => {
+        const timeout = /^PRAGMA busy_timeout=(\d+)$/.exec(sql);
+        if (timeout) timeouts.push(Number(timeout[1]));
+        return nativeExec(sql);
+    });
+    try {
+        await withSqliteTransformPass(() => {
+            // Use real BUSY failures but a controlled accounting clock: each
+            // retry costs 400ms regardless of runner scheduling. The deadline
+            // keeps the real waits short without hiding lease exhaustion.
+            guardSqliteTransformPass({ assert() {}, remainingMs: () => 10 });
+            let now = 0;
+            const clock = spyOn(performance, "now").mockImplementation(() => (now += 400));
+            try {
+                for (let i = 0; i < 8; i++) {
+                    expect(() => db.prepare("INSERT INTO result VALUES ('one')").run()).toThrow(
+                        SqliteAcquisitionBusyError,
+                    );
+                }
+            } finally {
+                clock.mockRestore();
+            }
+            // Five charged retries exhaust 2000ms. All later retries must have
+            // zero timeout even though the deadline still offers another 10ms.
+            expect(timeouts).toEqual([
+                ...Array.from({ length: 5 }, () => [0, 10, 5000]).flat(),
+                ...Array.from({ length: 3 }, () => [0, 0, 5000]).flat(),
+            ]);
+            blocker.exec("ROLLBACK");
+            db.prepare("INSERT INTO result VALUES ('uncontended')").run();
+        });
+        expect(db.prepare("SELECT * FROM result").all()).toEqual([{ value: "uncontended" }]);
+    } finally {
+        exec.mockRestore();
+        if (blocker.inTransaction) blocker.exec("ROLLBACK");
+        blocker.close();
+        close();
+    }
+}, 30_000);
+
+test("an exec script charges its single attempt without a zero-wait probe", () => {
+    const { db, path, close } = fixture();
+    const blocker = new Database(path);
+    const timeouts: number[] = [];
+    const nativeExec = db.exec.bind(db);
+    const exec = spyOn(db, "exec").mockImplementation((sql) => {
+        const timeout = /^PRAGMA busy_timeout=(\d+)$/.exec(sql);
+        if (timeout) timeouts.push(Number(timeout[1]));
+        return nativeExec(sql);
+    });
+    try {
+        withSqliteTransformPass(() => {
+            let now = 0;
+            const clock = spyOn(performance, "now").mockImplementation(() => (now += 2500));
+            try {
+                db.exec(
+                    "INSERT INTO result VALUES ('prefix'); INSERT INTO result VALUES ('suffix')",
+                );
+            } finally {
+                clock.mockRestore();
+            }
+            blocker.exec("BEGIN IMMEDIATE");
+            expect(() => db.prepare("INSERT INTO result VALUES ('blocked')").run()).toThrow(
+                SqliteAcquisitionBusyError,
+            );
+        });
+        expect(timeouts).toEqual([2000, 5000, 0, 0, 5000]);
+        expect(db.prepare("SELECT * FROM result").all()).toEqual([
+            { value: "prefix" },
+            { value: "suffix" },
+        ]);
+    } finally {
+        exec.mockRestore();
+        if (blocker.inTransaction) blocker.exec("ROLLBACK");
+        blocker.close();
+        close();
+    }
+});
+
+test("an explicit transaction keeps its timeout and never probes or retries", () => {
+    const { db, path, close } = fixture();
+    const other = new Database(path);
+    db.exec("PRAGMA busy_timeout=0; BEGIN; SELECT * FROM result");
+    other.exec("INSERT INTO result VALUES ('other')");
+    const exec = spyOn(db, "exec");
+    try {
+        withSqliteTransformPass(() => {
+            expect(() => db.prepare("INSERT INTO result VALUES ('stale')").run()).toThrow("locked");
+        });
+        expect(exec).not.toHaveBeenCalled();
+        expect(db.inTransaction).toBe(true);
+        expect(db.prepare("PRAGMA busy_timeout").get()).toEqual({ timeout: 0 });
+    } finally {
+        exec.mockRestore();
+        if (db.inTransaction) db.exec("ROLLBACK");
+        other.close();
+        close();
+    }
+});
+
 for (const mode of ["BEGIN", "autocommit"]) {
     test(`foreground ${mode} wait is capped by the pass deadline`, () => {
         const { db, path, close } = fixture();
@@ -138,3 +352,41 @@ for (const mode of ["BEGIN", "autocommit"]) {
         }
     });
 }
+
+test("an autocommit exec script never replays a committed prefix after BUSY", () => {
+    const main = fixture();
+    const auxiliary = fixture();
+    const free = fixture();
+    try {
+        main.db.prepare("ATTACH DATABASE ? AS aux").run(auxiliary.path);
+        main.db.prepare("ATTACH DATABASE ? AS free").run(free.path);
+        auxiliary.db.exec("BEGIN IMMEDIATE");
+        withSqliteTransformPass(() => {
+            guardSqliteTransformPass({ assert() {}, remainingMs: () => 100 });
+            expect(() =>
+                main.db.exec(
+                    "INSERT INTO main.result VALUES ('prefix'); INSERT INTO aux.result VALUES ('blocked')",
+                ),
+            ).toThrow(SqliteAcquisitionBusyError);
+            expect(() =>
+                main.db.exec(
+                    "CREATE TABLE applied_once(id); INSERT INTO aux.result VALUES ('blocked')",
+                ),
+            ).toThrow(SqliteAcquisitionBusyError);
+            expect(() =>
+                main.db.exec(
+                    "CREATE TABLE free.applied_once(id); INSERT INTO aux.result VALUES ('blocked')",
+                ),
+            ).toThrow(SqliteAcquisitionBusyError);
+        });
+        expect(main.db.prepare("SELECT * FROM main.result").all()).toEqual([{ value: "prefix" }]);
+        expect(
+            main.db.prepare("SELECT name FROM sqlite_schema WHERE name='applied_once'").all(),
+        ).toHaveLength(1);
+    } finally {
+        auxiliary.db.exec("ROLLBACK");
+        auxiliary.close();
+        main.close();
+        free.close();
+    }
+});

package/src/shared/sqlite-writer-diagnostics.test.ts

@@ -39,6 +39,22 @@ const scenario = `
     hold();
     writer.exec("COMMIT");
 
+    writer.exec("BEGIN IMMEDIATE");
+    now += 101;
+    writer.exec("ROLLBACK");
+
+    writer.exec("CREATE TABLE result(value)");
+    const statementTimes = [now, now + 101];
+    performance.now = () => statementTimes.shift() ?? now;
+    writer.prepare("INSERT INTO result VALUES (1)").run();
+    now += 101;
+
+    let clockReads = 0;
+    performance.now = () => { clockReads++; return now; };
+    writer.exec("BEGIN IMMEDIATE");
+    writer.exec("COMMIT");
+    if (clockReads !== 2) throw new Error("fast unscoped transaction clock reads=" + clockReads);
+
     logger.flushLogger();
     writer.close();
     holder.close();
@@ -106,6 +122,21 @@ describe("sqlite writer diagnostics", () => {
         expect(consoleOutput).not.toContain("sqlite writer");
     });
 
+    test("unscoped rollback names the host, pid and caller at the 100ms threshold", () => {
+        const rolledBack = logLines.find((line) => line.includes("outcome=rolled_back"));
+        expect(rolledBack).toContain("lane=background");
+        expect(rolledBack).toContain("hold_ms=101");
+        expect(rolledBack).toMatch(/pid=\d+ host=opencode/);
+        expect(rolledBack).toContain("caller=");
+    });
+
+    test("native autocommit time is labelled as including acquisition, not lock ownership", () => {
+        const statement = logLines.find((line) => line.includes("site=autocommit"));
+        expect(statement).toContain("elapsed_ms=101");
+        expect(statement).toContain("timing=statement_including_acquisition");
+        expect(statement).not.toContain("hold_ms=");
+    });
+
     test("a writer that waited for the lock reports the wait, not its short hold", () => {
         const waited = logLines.find((line) => line.includes("attempts=2"));
         if (!waited) throw new Error(`no retried acquisition logged:\n${logLines.join("\n")}`);

package/src/shared/sqlite.ts

@@ -39,7 +39,7 @@
  * `run()` → {changes,lastInsertRowid}) is identical and was verified directly.
  */
 
-import { AsyncLocalStorage } from "node:async_hooks";
+import { AsyncLocalStorage, AsyncResource } from "node:async_hooks";
 import { statSync } from "node:fs";
 
 // Type import only — runtime is loaded dynamically below. @types/better-sqlite3
@@ -48,6 +48,16 @@ import { statSync } from "node:fs";
 // node:sqlite at runtime (both expose prepare/run/get/all/exec/close).
 import type BetterSqlite3 from "better-sqlite3";
 
+// Capture the root before creating request-local AsyncLocalStorage scopes.
+// Shared workers and timers must not inherit callbacks that reject stale
+// database writes: those callbacks can retain an obsolete request's inputs.
+const resourceRoot = new AsyncResource("magic-context:root");
+
+/** Create host-owned resources without inheriting request-local state. */
+export function inRootAsyncContext<T>(operation: () => T): T {
+    return resourceRoot.runInAsyncScope(operation);
+}
+
 /**
  * Slow-write reporting is injected rather than imported: this module is also
  * executed directly by Node (the `node:sqlite` smoke in CI, Pi, Desktop), and
@@ -72,14 +82,21 @@ export function registerSlowWriteReporter(reporter: SlowWriteReporter): void {
  */
 type SqliteDiagnosticSink = (message: string) => void;
 let sqliteDiagnosticSink: SqliteDiagnosticSink | undefined;
+let sqliteDiagnosticHost: () => string = () => "unknown";
 
-export function registerSqliteDiagnosticSink(sink: SqliteDiagnosticSink): void {
+export function registerSqliteDiagnosticSink(
+    sink: SqliteDiagnosticSink,
+    host?: () => string,
+): void {
     sqliteDiagnosticSink = sink;
+    if (host) sqliteDiagnosticHost = host;
 }
 
 function reportSqliteDiagnostic(message: string): void {
     try {
-        sqliteDiagnosticSink?.(`[magic-context] ${message}`);
+        sqliteDiagnosticSink?.(
+            `[magic-context] ${message} pid=${process.pid} host=${sqliteDiagnosticHost()}`,
+        );
     } catch {
         // Diagnostics must never change the outcome of the write they describe.
     }
@@ -393,9 +410,18 @@ function installTransactionRouting(db: BetterSqlite3.Database, readonly: boolean
                 // A transaction that ended without passing through exec (for
                 // example an automatic rollback) must not be timed as this one.
                 openWriterTransactions.delete(db);
-                if (transformPassScope.getStore()?.active || backgroundWriterScope.getStore())
+                if (transformPassScope.getStore()?.active || backgroundWriterScope.getStore()) {
                     acquireShort(db, () => nativeExec(sql), sql.trim());
-                else nativeExec(sql);
+                } else {
+                    nativeExec(sql);
+                    openWriterTransactions.set(db, {
+                        site: sql.trim(),
+                        lane: "background",
+                        acquireMs: 0,
+                        attempts: 1,
+                        acquiredAt: performance.now(),
+                    });
+                }
                 return db;
             }
             if (WRITER_TRANSACTION_END.test(sql) && openWriterTransactions.has(db))
@@ -404,7 +430,9 @@ function installTransactionRouting(db: BetterSqlite3.Database, readonly: boolean
                 !/^\s*(?:SELECT|EXPLAIN|PRAGMA|COMMIT|END|ROLLBACK|SAVEPOINT|RELEASE)\b/i.test(sql)
             ) {
                 try {
-                    return withAutocommitTimeout(db, () => nativeExec(sql));
+                    // exec can commit a script prefix before BUSY. Run it once;
+                    // only prepared statements are safe to probe and retry.
+                    return withAutocommitTimeout(db, () => nativeExec(sql), false);
                 } finally {
                     if (mayChangeTagIdentity(sql)) tagIdentityWriteGeneration += 1;
                 }
@@ -571,7 +599,9 @@ function trackSqliteConnection(
 
 const TrackedDatabase = new Proxy(DatabaseImpl, {
     construct(target, args) {
-        const db = Reflect.construct(target, args, target) as BetterSqlite3.Database;
+        const db = inRootAsyncContext(
+            () => Reflect.construct(target, args, target) as BetterSqlite3.Database,
+        );
         return trackSqliteConnection(db, args[0], args[1]);
     },
 }) as typeof BetterSqlite3;
@@ -943,7 +973,7 @@ const SHORT_BUSY_TIMEOUT_MS = 25;
 const FOREGROUND_IN_PASS_BUSY_TIMEOUT_MS = 2_000;
 const FOREGROUND_ACQUISITION_BUDGET_MS = 16_500;
 /** Waiting this long for the writer lock, or holding it this long, is logged. */
-const SLOW_WRITER_LOG_MS = 250;
+const SLOW_WRITER_LOG_MS = 100;
 const WRITER_TRANSACTION_END = /^\s*(?:COMMIT|END|ROLLBACK)(?:\s+TRANSACTION)?\s*;?\s*$/i;
 
 type WriterLane = "foreground" | "background";
@@ -984,32 +1014,67 @@ function foregroundWaitLease() {
         : undefined;
 }
 
-/** Bound implicit writer admission too, including prepared INSERT OR IGNORE and
- * writes with RETURNING. Charge successful execution time conservatively: the
- * native API cannot separate lock waiting from statement execution. Exhaustion
- * switches to zero-wait acquisition, not a refusal when the writer is free. */
-function withAutocommitTimeout<T>(db: Database, operation: () => T): T {
+/** Probe a prepared statement's writer lock without charging uncontended work
+ * against the foreground lock-wait budget. An exec script may commit some of
+ * its statements before failing, so it cannot be retried without duplicating
+ * writes; conservatively charge its single attempt's full execution time. */
+function withAutocommitTimeout<T>(db: Database, operation: () => T, singleStatement = true): T {
     assertTransformWrite();
     const lease = foregroundWaitLease();
-    if (isInTransaction(db) || (!lease && !backgroundWriterScope.getStore())) return operation();
-    const previous = pragmaNumber(db, "busy_timeout");
-    db.exec(
-        `PRAGMA busy_timeout=${lease ? Math.floor(Math.min(lease.remainingWaitMs, lease.guard?.remainingMs() ?? Infinity)) : SHORT_BUSY_TIMEOUT_MS}`,
-    );
+    if (isInTransaction(db)) return operation();
+    const scoped = lease || backgroundWriterScope.getStore();
+    const previous = scoped ? pragmaNumber(db, "busy_timeout") : null;
     const started = performance.now();
+    let completed = false;
+    const execute = () => {
+        const result = operation();
+        completed = true;
+        return result;
+    };
     try {
-        return operation();
+        if (!scoped) return execute();
+        if (singleStatement) {
+            db.exec("PRAGMA busy_timeout=0");
+            try {
+                return execute();
+            } catch (error) {
+                if (!isTransientSqliteError(error) || isInTransaction(db)) throw error;
+            }
+            assertTransformWrite();
+        }
+        const timeout = Math.floor(
+            Math.min(
+                lease?.remainingWaitMs ?? SHORT_BUSY_TIMEOUT_MS,
+                lease?.guard?.remainingMs() ?? Infinity,
+            ),
+        );
+        db.exec(`PRAGMA busy_timeout=${timeout}`);
+        const waitStarted = performance.now();
+        try {
+            return execute();
+        } finally {
+            // Deduct the prepared statement's contended retry or the exec
+            // script's execution from the remaining foreground wait budget;
+            // the native API cannot separate lock waiting from execution.
+            if (lease)
+                lease.remainingWaitMs = Math.max(
+                    0,
+                    lease.remainingWaitMs - (performance.now() - waitStarted),
+                );
+        }
     } catch (error) {
         if (lease && isTransientSqliteError(error))
             throw new SqliteAcquisitionBusyError(error, "autocommit");
         throw error;
     } finally {
-        if (lease)
-            lease.remainingWaitMs = Math.max(
-                0,
-                lease.remainingWaitMs - (performance.now() - started),
-            );
+        const elapsed = performance.now() - started;
         if (previous !== null) db.exec(`PRAGMA busy_timeout=${previous}`);
+        // Native autocommit does not expose the instant its lock was acquired.
+        // Label the upper bound so a waiting statement is not blamed as a holder.
+        if (elapsed >= SLOW_WRITER_LOG_MS)
+            reportSqliteDiagnostic(
+                `sqlite writer site=autocommit lane=${lease ? "foreground" : "background"} elapsed_ms=${Math.round(elapsed)} timing=statement_including_acquisition outcome=${completed ? "completed" : "failed"} caller=${writerCaller()}`,
+            );
     }
 }
 
@@ -1071,20 +1136,18 @@ function acquireShort(db: Database, acquire: () => unknown, site: string): void
  * `acquireShort`, then report how long the writer waited for the lock
  * (`acquire_ms`) and how long it then held it (`hold_ms`) when either crosses
  * the threshold. A long hold blocks every other process's writer, so it is
- * worth seeing even when the lock itself came quickly. The native end statement's
- * `end_ms` is reported separately, so a slow FTS merge or checkpoint at COMMIT
- * is not mistaken for acquisition time.
+ * worth seeing even when the lock itself came quickly. This interval includes
+ * COMMIT: an automatic PASSIVE checkpoint can run after writer-lock release,
+ * so hold_ms is a conservative upper bound, not proof of continuous ownership.
  */
 function endWriterTransaction(db: Database, end: () => unknown, sql: string): unknown {
     let ended = false;
-    const endStarted = performance.now();
     try {
         const result = end();
         ended = true;
         return result;
     } finally {
         const open = openWriterTransactions.get(db);
-        const endMs = performance.now() - endStarted;
         // A COMMIT that failed with the transaction still open has not ended it.
         if (open && !isInTransaction(db)) {
             openWriterTransactions.delete(db);
@@ -1096,13 +1159,32 @@ function endWriterTransaction(db: Database, end: () => unknown, sql: string): un
                       ? "committed"
                       : "failed";
                 reportSqliteDiagnostic(
-                    `sqlite writer site=${open.site} lane=${open.lane} acquire_ms=${Math.round(open.acquireMs)} hold_ms=${Math.round(holdMs)} end_ms=${Math.round(endMs)} attempts=${open.attempts} outcome=${outcome}`,
+                    `sqlite writer site=${open.site} lane=${open.lane} acquire_ms=${Math.round(open.acquireMs)} hold_ms=${Math.round(holdMs)} attempts=${open.attempts} outcome=${outcome} timing=through_commit_upper_bound caller=${writerCaller()}`,
                 );
             }
         }
     }
 }
 
+// Capture a stack only for slow writes. Synchronous COMMIT retains the writer's
+// caller, so fast transactions need neither stack allocation nor source mapping.
+function writerCaller(): string {
+    return (
+        new Error().stack
+            ?.split("\n")
+            .find(
+                (line) =>
+                    line.includes(" at ") &&
+                    !/[/\\]sqlite\.[tj]s[:)]/.test(line) &&
+                    !/\b(?:writerCaller|endWriterTransaction|withAutocommitTimeout|value)\b/.test(
+                        line,
+                    ),
+            )
+            ?.trim()
+            .replace(/\s+/g, "_") ?? "unknown"
+    );
+}
+
 /** Begin an immediate transaction with bounded asynchronous retries on writer contention.
  * Callers must recheck any lease or source snapshot after this function returns.
  * Resolves with the `performance.now()` time at which BEGIN IMMEDIATE succeeded,