@cortexkit/opencode-magic-context 0.44.2 → 0.44.3

package/src/shared/sqlite.ts

@@ -224,7 +224,9 @@ function installTransactionRouting(db: BetterSqlite3.Database, readonly: boolean
         writable: true,
         value: (sql: string) => {
             if (/^\s*BEGIN\s+(?:IMMEDIATE|EXCLUSIVE)(?:\s+TRANSACTION)?\s*;?\s*$/i.test(sql)) {
-                retryAcquisition(() => nativeExec(sql), sql.trim());
+                if (transformPassScope.getStore()?.active || backgroundWriterScope.getStore())
+                    acquireShort(db, () => nativeExec(sql), sql.trim());
+                else nativeExec(sql);
                 return db;
             }
             return nativeExec(sql);
@@ -489,6 +491,13 @@ export type Statement = BetterSqlite3.Statement<unknown[], unknown>;
 
 const privilegeDepth = new WeakMap<Database, number>();
 const transformPassScope = new AsyncLocalStorage<{ active: boolean } | undefined>();
+const admissionScope = new AsyncLocalStorage<boolean>();
+const backgroundWriterScope = new AsyncLocalStorage<boolean>();
+
+/** Opt in only a writer that can safely defer its work after a busy acquisition. */
+export function withSqliteBackgroundWriter<T>(operation: () => T): T {
+    return backgroundWriterScope.run(true, operation);
+}
 
 /** Only an awaited foreground transform may spend the extra acquisition budget.
  * The mutable lease also expires for detached descendants when that pass ends. */
@@ -530,35 +539,123 @@ export class SqliteAcquisitionBusyError extends Error {
     readonly code = "SQLITE_BUSY";
     readonly stage: string;
     constructor(cause: unknown, stage = "BEGIN IMMEDIATE") {
-        super("SQLite writer acquisition remained busy after 3 attempts", { cause });
+        super("SQLite writer acquisition remained busy", { cause });
         this.name = "SqliteAcquisitionBusyError";
         this.stage = stage;
     }
 }
 
-/** Retry only acquisition: no privilege flag, callback or in-memory mutation has run yet. */
-function retryAcquisition(acquire: () => unknown, stage: string): void {
-    // Maintenance writers retry on their next tick. Multiplying their native
-    // busy timeout here would stall every session sharing the host event loop.
-    if (!transformPassScope.getStore()?.active) {
+const SHORT_BUSY_TIMEOUT_MS = 25;
+const FOREGROUND_IN_PASS_BUSY_TIMEOUT_MS = 250;
+const FOREGROUND_ACQUISITION_BUDGET_MS = 16_500;
+
+/** The connection is never handed back to another caller with a shortened timeout. */
+function acquireShort(db: Database, acquire: () => unknown, site: string): void {
+    const started = performance.now();
+    const previous = pragmaNumber(db, "busy_timeout");
+    let acquired = false;
+    try {
+        // Async admission retries after yielding, so each of its BEGIN attempts
+        // stays short. An ordinary BEGIN inside a transform pass cannot yield;
+        // give that single attempt enough time for a brief writer to finish.
+        const timeout =
+            transformPassScope.getStore()?.active &&
+            !admissionScope.getStore() &&
+            !backgroundWriterScope.getStore()
+                ? FOREGROUND_IN_PASS_BUSY_TIMEOUT_MS
+                : SHORT_BUSY_TIMEOUT_MS;
+        db.exec(`PRAGMA busy_timeout=${timeout}`);
         acquire();
-        return;
+        acquired = true;
+    } catch (error) {
+        if (transformPassScope.getStore()?.active && isTransientSqliteError(error))
+            throw new SqliteAcquisitionBusyError(error, site);
+        throw error;
+    } finally {
+        if (previous !== null) db.exec(`PRAGMA busy_timeout=${previous}`);
+        const elapsed = performance.now() - started;
+        if (elapsed >= 250)
+            console.warn(
+                `[${new Date().toISOString()}] [magic-context] sqlite acquisition site=${site} lane=${transformPassScope.getStore()?.active ? "foreground" : "background"} elapsed=${Math.round(elapsed)}ms attempts=1 outcome=${acquired ? "acquired" : "busy"}`,
+            );
     }
-    const delays = [500, 1000];
-    for (let attempt = 0; ; attempt++) {
+}
+
+/** Begin an immediate transaction with bounded asynchronous retries on writer contention.
+ * Callers must recheck any lease or source snapshot after this function returns. */
+export async function beginSqliteWriterAsync(db: Database, site: string): Promise<void> {
+    const started = performance.now();
+    let attempts = 0;
+    for (;;) {
+        attempts++;
         try {
-            acquire();
+            withSqliteBackgroundWriter(() => db.exec("BEGIN IMMEDIATE"));
+            const elapsed = performance.now() - started;
+            if (elapsed >= 250)
+                console.warn(
+                    `[${new Date().toISOString()}] [magic-context] sqlite acquisition site=${site} lane=background elapsed=${Math.round(elapsed)}ms attempts=${attempts} outcome=acquired`,
+                );
             return;
         } catch (error) {
-            // Handles created by this module already retry inside exec. Do not
-            // apply the attempt budget twice; externally supplied handles still
-            // need the acquisition retry performed by withPrivilegedWriter.
-            if (error instanceof SqliteAcquisitionBusyError || !isTransientSqliteError(error))
+            if (!isTransientSqliteError(error)) throw error;
+            const elapsed = performance.now() - started;
+            if (elapsed >= FOREGROUND_ACQUISITION_BUDGET_MS) {
+                console.warn(
+                    `[${new Date().toISOString()}] [magic-context] sqlite acquisition site=${site} lane=background elapsed=${Math.round(elapsed)}ms attempts=${attempts} outcome=busy`,
+                );
                 throw error;
-            if (attempt === delays.length) throw new SqliteAcquisitionBusyError(error, stage);
-            // The database API is synchronous on both hosts. With busy_timeout=5000,
-            // three attempts plus these waits bound one acquisition to 16.5 seconds.
-            Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, delays[attempt]);
+            }
+            await new Promise<void>((resolve) =>
+                setTimeout(
+                    resolve,
+                    Math.min(
+                        attempts === 1 ? 500 : 1000,
+                        FOREGROUND_ACQUISITION_BUDGET_MS - elapsed,
+                    ),
+                ),
+            );
+        }
+    }
+}
+
+/** Retry admission before any turn work, yielding between short lock attempts. */
+export async function withAsyncPrivilegedWriter<T>(db: Database, operation: () => T): Promise<T> {
+    const started = performance.now();
+    let attempts = 0;
+    for (;;) {
+        attempts++;
+        try {
+            const result = admissionScope.run(true, () =>
+                withSqliteTransformPass(() => withPrivilegedWriter(db, operation)),
+            );
+            const elapsed = performance.now() - started;
+            if (elapsed >= 250)
+                console.warn(
+                    `[${new Date().toISOString()}] [magic-context] sqlite acquisition site=privileged_writer lane=foreground elapsed=${Math.round(elapsed)}ms attempts=${attempts} outcome=acquired`,
+                );
+            return result;
+        } catch (error) {
+            // A callback failure can occur after writes; only retry a failed BEGIN.
+            if (!(error instanceof SqliteAcquisitionBusyError)) throw error;
+            const elapsed = performance.now() - started;
+            if (elapsed >= FOREGROUND_ACQUISITION_BUDGET_MS) {
+                if (elapsed >= 250)
+                    console.warn(
+                        `[${new Date().toISOString()}] [magic-context] sqlite acquisition site=privileged_writer lane=foreground elapsed=${Math.round(elapsed)}ms attempts=${attempts} outcome=busy`,
+                    );
+                throw error instanceof SqliteAcquisitionBusyError
+                    ? error
+                    : new SqliteAcquisitionBusyError(error);
+            }
+            await new Promise<void>((resolve) =>
+                setTimeout(
+                    resolve,
+                    Math.min(
+                        attempts === 1 ? 500 : 1000,
+                        FOREGROUND_ACQUISITION_BUDGET_MS - elapsed,
+                    ),
+                ),
+            );
         }
     }
 }
@@ -587,7 +684,13 @@ export function withPrivilegedWriter<T>(db: Database, operation: () => T): T {
     if (nested) {
         db.exec(`SAVEPOINT ${savepoint}`);
     } else {
-        retryAcquisition(() => db.exec("BEGIN IMMEDIATE"), "BEGIN IMMEDIATE");
+        try {
+            db.exec("BEGIN IMMEDIATE");
+        } catch (error) {
+            if (transformPassScope.getStore()?.active && isTransientSqliteError(error))
+                throw new SqliteAcquisitionBusyError(error);
+            throw error;
+        }
     }
     privilegeDepth.set(db, previousDepth + 1);
     try {

package/src/shared/storage-busy.test.ts

@@ -1,70 +1,47 @@
-import { afterAll, afterEach, describe, expect, spyOn, test } from "bun:test";
+import { describe, expect, test } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import {
     Database,
     isTransientSqliteError,
-    withSqliteTransformPass,
-    withPrivilegedWriter as writePrivileged,
+    withAsyncPrivilegedWriter,
+    withPrivilegedWriter,
 } from "./sqlite";
 
-const withPrivilegedWriter = <T>(db: Database, operation: () => T): T =>
-    withSqliteTransformPass(() => writePrivileged(db, operation));
-
-const wait = spyOn(Atomics, "wait");
-afterEach(() => wait.mockClear());
-afterAll(() => wait.mockRestore());
-
-describe("writer acquisition backoff", () => {
-    for (const clearsOn of [2, 3]) {
-        test(`acquisition clears on attempt ${clearsOn} and invokes callback exactly once`, () => {
-            const db = new Database(":memory:");
-            db.exec(
-                "CREATE TABLE context_privilege_state(id INTEGER PRIMARY KEY, enabled INTEGER); CREATE TABLE result(value TEXT)",
-            );
-            const exec = db.exec.bind(db);
-            let attempts = 0;
-            let callbacks = 0;
-            const delays: number[] = [];
-            wait.mockImplementation((_array, _index, _value, timeout) => {
-                delays.push(timeout!);
-                return "timed-out";
-            });
-            const intercepted = spyOn(db, "exec").mockImplementation((sql) => {
-                if (sql === "BEGIN IMMEDIATE" && ++attempts < clearsOn)
-                    throw Object.assign(new Error("locked"), { code: "SQLITE_BUSY" });
-                return exec(sql);
-            });
-            try {
-                const result = withPrivilegedWriter(db, () => {
-                    callbacks++;
-                    db.prepare("INSERT INTO result VALUES (?)").run("managed");
-                    return JSON.stringify(db.prepare("SELECT * FROM result").all());
-                });
-                expect(result).toBe('[{"value":"managed"}]');
-                expect(attempts).toBe(clearsOn);
-                expect(callbacks).toBe(1);
-                expect(delays).toEqual([500, 1000].slice(0, clearsOn - 1));
-            } finally {
-                intercepted.mockRestore();
-                db.close();
-            }
-        });
-    }
-    test("exhausted acquisition never invokes callback", () => {
-        const db = new Database(":memory:");
+describe("writer acquisition", () => {
+    test("exhausted async admission preserves the foreground budget and never invokes the callback", async () => {
+        const dir = mkdtempSync(join(tmpdir(), "mc-writer-budget-"));
+        const path = join(dir, "context.db");
+        const blocker = new Database(path);
+        blocker.exec(
+            "CREATE TABLE context_privilege_state(id INTEGER PRIMARY KEY, enabled INTEGER)",
+        );
+        const db = new Database(path);
+        db.exec("PRAGMA busy_timeout=5000");
+        blocker.exec("BEGIN IMMEDIATE");
         let calls = 0;
-        wait.mockImplementation(() => "timed-out");
-        const exec = spyOn(db, "exec").mockImplementation(() => {
-            throw Object.assign(new Error("original lock"), { code: "SQLITE_LOCKED" });
-        });
+        let ticks = 0;
+        const timer = setInterval(() => ticks++, 50);
+        const started = performance.now();
         try {
-            expect(() => withPrivilegedWriter(db, () => calls++)).toThrow("after 3 attempts");
-            expect(exec).toHaveBeenCalledTimes(3);
+            await expect(withAsyncPrivilegedWriter(db, () => calls++)).rejects.toThrow(
+                "acquisition remained busy",
+            );
+            expect(performance.now() - started).toBeGreaterThanOrEqual(16_500);
+            expect(performance.now() - started).toBeLessThan(17_500);
+            expect(ticks).toBeGreaterThan(100);
             expect(calls).toBe(0);
+            expect(db.prepare("PRAGMA busy_timeout").get()).toEqual({ timeout: 5000 });
         } finally {
-            exec.mockRestore();
+            clearInterval(timer);
+            blocker.exec("ROLLBACK");
+            blocker.close();
             db.close();
+            rmSync(dir, { recursive: true, force: true });
         }
-    });
+    }, 30000);
+
     test("busy callback rolls back without retrying mutations", () => {
         const db = new Database(":memory:");
         db.exec(
@@ -81,11 +58,11 @@ describe("writer acquisition backoff", () => {
             ).toThrow("after acquisition");
             expect(calls).toBe(1);
             expect(db.prepare("SELECT * FROM result").all()).toEqual([]);
-            expect(wait).not.toHaveBeenCalled();
         } finally {
             db.close();
         }
     });
+
     test("recognizes Bun extended and Node numeric contention codes only", () => {
         expect(isTransientSqliteError({ code: "SQLITE_BUSY_SNAPSHOT" })).toBe(true);
         expect(isTransientSqliteError({ code: "ERR_SQLITE_ERROR", errcode: 5 })).toBe(true);

package/src/shared/write-transaction-attribution.test.ts

@@ -6,7 +6,7 @@ import { join, resolve } from "node:path";
 const repoRoot = resolve(import.meta.dir, "../../../../");
 const beginImmediate = /\b(?:[A-Za-z_$][\w$]*\.)?exec\("BEGIN IMMEDIATE"\)/g;
 
-const fencedSources: Array<{ path: string; sites: string[] }> = [
+const fencedSources: Array<{ path: string; sites: string[]; reporterCalls?: string[] }> = [
     {
         path: "packages/plugin/src/features/magic-context/smart-notes/storage.ts",
         sites: ["smart_note_commit"],
@@ -52,8 +52,8 @@ const fencedSources: Array<{ path: string; sites: string[] }> = [
         // The chokepoint cannot import the logging chain (Node executes it
         // directly), so it reports through the injected reporter instead.
         path: "packages/plugin/src/shared/sqlite.ts",
-        sites: ["privileged_writer"],
-        reporterCall: "reportSlowPrivilegedWrite?.(",
+        sites: ["async_acquisition_dynamic_site", "privileged_writer"],
+        reporterCalls: ["console.warn(", "reportSlowPrivilegedWrite?.("],
     },
     {
         path: "packages/plugin/src/features/magic-context/git-commits/sweep-coordinator.ts",
@@ -75,10 +75,6 @@ const fencedSources: Array<{ path: string; sites: string[] }> = [
         path: "packages/plugin/src/hooks/magic-context/compartment-runner-recomp.ts",
         sites: ["historian-publish:recomp"],
     },
-    {
-        path: "packages/plugin/src/hooks/magic-context/compartment-runner-incremental.ts",
-        sites: ["historian-publish"],
-    },
     {
         path: "packages/plugin/src/features/magic-context/dreamer/lease.ts",
         sites: ["lease_dynamic_site"],
@@ -98,6 +94,10 @@ const fencedSources: Array<{ path: string; sites: string[] }> = [
 ];
 
 const nonBeginCoveredSites: Array<{ path: string; site: string }> = [
+    {
+        path: "packages/plugin/src/hooks/magic-context/compartment-runner-incremental.ts",
+        site: "historian-publish",
+    },
     {
         path: "packages/plugin/src/features/magic-context/message-index.ts",
         site: "message_index_clear",
@@ -159,7 +159,7 @@ function sourceFor(relativePath: string): string {
 function assertCoveredBeginImmediate(
     source: string,
     sites: string[],
-    reporterCall = "logSlowWriteTransaction(",
+    reporterCalls: string[] = [],
 ): void {
     const matches = [...source.matchAll(beginImmediate)];
     expect(matches).toHaveLength(sites.length);
@@ -167,8 +167,10 @@ function assertCoveredBeginImmediate(
         const match = matches[index];
         const nextBegin = matches[index + 1]?.index ?? source.length;
         const transactionRegion = source.slice(match.index, nextBegin);
-        expect(transactionRegion).toContain(reporterCall);
-        if (sites[index] === "lease_dynamic_site") {
+        expect(transactionRegion).toContain(reporterCalls[index] ?? "logSlowWriteTransaction(");
+        if (sites[index] === "async_acquisition_dynamic_site") {
+            expect(transactionRegion).toContain("site=${site}");
+        } else if (sites[index] === "lease_dynamic_site") {
             expect(transactionRegion).toContain("logSlowWriteTransaction(site");
         } else {
             expect(transactionRegion).toContain(`"${sites[index]}"`);
@@ -219,7 +221,7 @@ describe("write transaction attribution fences", () => {
             assertCoveredBeginImmediate(
                 sourceFor(fencedSource.path),
                 fencedSource.sites,
-                "reporterCall" in fencedSource ? fencedSource.reporterCall : undefined,
+                fencedSource.reporterCalls,
             );
         }
         for (const coveredSite of nonBeginCoveredSites) {