@gloom-sh/terminal 0.19.1 → 0.19.3

package/src/data/sqlite/database.ts

@@ -1,4 +1,15 @@
 import { Database } from "bun:sqlite";
+import { closeSync, openSync } from "fs";
+import { logFailure } from "../../utils/log-failure";
+import { restrictFileSync } from "../../utils/private-files";
+import { createSqliteAccess, type SqliteAccess } from "./access";
+import {
+  SQLITE_BACKSTOP_AUTOCHECKPOINT_PAGES,
+  SQLITE_DEFAULT_AUTOCHECKPOINT_PAGES,
+  startWalCheckpointer,
+  type WalCheckpointer,
+  type WalCheckpointerOptions,
+} from "./checkpointer";
 import { SQLITE_BUSY_TIMEOUT_MS, withSqliteBusyRetry } from "./retry";
 
 const SCHEMA = `
@@ -60,21 +71,86 @@ const SCHEMA = `
   );
 `;
 
+export interface SqliteDatabaseOptions {
+  /**
+   * The app's own connection, on the thread that answers the user (the
+   * terminal, the desktop app's Bun process): statements do not wait on
+   * another program's lock (see access.ts). Commands and tests wait it out.
+   */
+  interactive?: boolean;
+  /** Tests only: the WAL checkpointer's thread and how long it may go silent. */
+  checkpointer?: WalCheckpointerOptions;
+}
+
 export class SqliteDatabase {
   readonly connection: Database;
+  readonly access: SqliteAccess;
+  private readonly checkpointer: WalCheckpointer | null = null;
 
-  constructor(dbPath: string) {
+  constructor(dbPath: string, options: SqliteDatabaseOptions = {}) {
+    const onDisk = dbPath !== "" && dbPath !== ":memory:" && !dbPath.startsWith("file:");
+    // The database holds the Cloud session: created readable by the user
+    // alone, so the WAL and shared-memory files SQLite makes beside it with
+    // the same mode are too, and an older one made so.
+    if (onDisk) {
+      try {
+        closeSync(openSync(dbPath, "a", 0o600));
+        restrictFileSync(dbPath);
+      } catch (error) {
+        logFailure("data.sqlite", "Restricting the database file to its user failed")(error);
+      }
+    }
     this.connection = new Database(dbPath);
+    // Opening waits for another program's lock: it happens once, and the app
+    // has nothing to show without its data.
     this.connection.exec(`PRAGMA busy_timeout=${SQLITE_BUSY_TIMEOUT_MS}`);
-    withSqliteBusyRetry("enable sqlite wal", () => {
-      this.connection.exec("PRAGMA journal_mode=WAL");
-    });
+    const journalMode = withSqliteBusyRetry("enable sqlite wal", () => (
+      this.connection.query<{ journal_mode: string }, []>("PRAGMA journal_mode=WAL").get()?.journal_mode
+    ));
+    // In WAL mode commits can stop flushing the disk and the database stays
+    // whole; user data is flushed soon after by the checkpointer, or per write
+    // where it does not run (access.ts). Where WAL does not take (a network
+    // share, an in-memory database), every commit flushes.
+    const wal = journalMode?.toLowerCase() === "wal";
+    if (onDisk && wal) {
+      try {
+        restrictFileSync(`${dbPath}-wal`);
+        restrictFileSync(`${dbPath}-shm`);
+      } catch (error) {
+        logFailure("data.sqlite", "Restricting the database's WAL files to their user failed")(error);
+      }
+    }
+    this.connection.exec(wal ? "PRAGMA synchronous=NORMAL" : "PRAGMA synchronous=FULL");
     withSqliteBusyRetry("initialize sqlite schema", () => {
       this.connection.exec(SCHEMA);
     });
+    // The app's statements choose how long to wait for a lock themselves.
+    if (options.interactive) this.connection.exec("PRAGMA busy_timeout=0");
+    // The app's commits leave copying the WAL into the database, and flushing
+    // it, to a thread of its own, and only copy it themselves past a WAL ten
+    // times the usual.
+    let checkpointerRunning = false;
+    if (options.interactive && wal && dbPath !== ":memory:") {
+      this.checkpointer = startWalCheckpointer(dbPath, () => {
+        checkpointerRunning = false;
+        this.connection.exec(`PRAGMA wal_autocheckpoint=${SQLITE_DEFAULT_AUTOCHECKPOINT_PAGES}`);
+      }, options.checkpointer);
+      if (this.checkpointer) {
+        checkpointerRunning = true;
+        this.connection.exec(`PRAGMA wal_autocheckpoint=${SQLITE_BACKSTOP_AUTOCHECKPOINT_PAGES}`);
+      }
+    }
+    const checkpointer = this.checkpointer;
+    this.access = createSqliteAccess(this.connection, {
+      interactive: options.interactive,
+      relaxedCommits: wal,
+      ...(checkpointer ? { onCommit: () => checkpointer.nudge(), flushedElsewhere: () => checkpointerRunning } : {}),
+    });
   }
 
   close(): void {
+    this.access.close();
+    this.checkpointer?.close();
     this.connection.close();
   }
 }

package/src/data/sqlite/retry.ts

@@ -3,6 +3,9 @@ import { recordPerfSample } from "../../utils/perf-marks";
 const SQLITE_BUSY_RETRY_ATTEMPTS = 4;
 const SQLITE_BUSY_RETRY_INITIAL_DELAY_MS = 25;
 const SQLITE_BUSY_RETRY_MAX_DELAY_MS = 250;
+/** Between two tries for a lock another connection holds, while a bounded wait lasts. */
+const SQLITE_LOCK_POLL_INITIAL_DELAY_MS = 2;
+const SQLITE_LOCK_POLL_MAX_DELAY_MS = 25;
 const SQLITE_BUSY_SLEEP_BUFFER = new Int32Array(new SharedArrayBuffer(4));
 
 export const SQLITE_BUSY_TIMEOUT_MS = 2000;
@@ -30,7 +33,7 @@ export function isSqliteBusyError(error: unknown): boolean {
   if (!error || typeof error !== "object") return false;
   const candidate = error as { code?: unknown; errno?: unknown; message?: unknown };
   const code = typeof candidate.code === "string" ? candidate.code.toUpperCase() : "";
-  if (code === "SQLITE_BUSY" || code === "SQLITE_LOCKED") return true;
+  if (code.startsWith("SQLITE_BUSY") || code.startsWith("SQLITE_LOCKED")) return true;
   if (candidate.errno === 5 || candidate.errno === 6) return true;
   const message = typeof candidate.message === "string" ? candidate.message.toLowerCase() : "";
   return message.includes("database is locked")
@@ -43,6 +46,17 @@ function sleepSync(delayMs: number): void {
   Atomics.wait(SQLITE_BUSY_SLEEP_BUFFER, 0, 0, delayMs);
 }
 
+/**
+ * Names a statement that held the thread for a while: which one, how many
+ * tries it took, and whether it waited on another connection's lock (a busy
+ * answer came back) or was simply slow (a large write, the disk).
+ */
+function reportIfBlocking(operationName: string, startedAt: number, attempts: number, lockWait: boolean): void {
+  const elapsedMs = now() - startedAt;
+  if (elapsedMs < SQLITE_BLOCKING_REPORT_MS) return;
+  recordPerfSample("sqlite.blocked", elapsedMs, { operation: operationName, attempts, lockWait });
+}
+
 export function withSqliteBusyRetry<T>(
   operationName: string,
   operation: () => T,
@@ -52,20 +66,18 @@ export function withSqliteBusyRetry<T>(
   const maxDelayMs = Math.max(0, options.maxDelayMs ?? SQLITE_BUSY_RETRY_MAX_DELAY_MS);
   let delayMs = Math.max(0, options.initialDelayMs ?? SQLITE_BUSY_RETRY_INITIAL_DELAY_MS);
   const startedAt = now();
-  const reportIfBlocking = (attempt: number) => {
-    const elapsedMs = now() - startedAt;
-    if (elapsedMs < SQLITE_BLOCKING_REPORT_MS) return;
-    recordPerfSample("sqlite.blocked", elapsedMs, { operation: operationName, attempts: attempt });
-  };
+  let lockWait = false;
 
   for (let attempt = 1; attempt <= attempts; attempt += 1) {
     try {
       const result = operation();
-      reportIfBlocking(attempt);
+      reportIfBlocking(operationName, startedAt, attempt, lockWait);
       return result;
     } catch (error) {
-      if (!isSqliteBusyError(error) || attempt >= attempts) {
-        reportIfBlocking(attempt);
+      const busy = isSqliteBusyError(error);
+      lockWait ||= busy;
+      if (!busy || attempt >= attempts) {
+        reportIfBlocking(operationName, startedAt, attempt, lockWait);
         throw error;
       }
       sleepSync(delayMs);
@@ -88,3 +100,32 @@ export function trySqliteBusyOperation<T>(
     throw error;
   }
 }
+
+/**
+ * Runs `operation` on a connection that does not wait for locks itself (a
+ * busy timeout of 0), trying again while another connection holds the lock
+ * it needs, for `waitMs` at most. Then the busy error is thrown. A wait of 0
+ * is one try.
+ */
+export function withSqliteLockWait<T>(operationName: string, operation: () => T, waitMs: number): T {
+  const startedAt = now();
+  let delayMs = SQLITE_LOCK_POLL_INITIAL_DELAY_MS;
+  let lockWait = false;
+  for (let attempt = 1; ; attempt += 1) {
+    try {
+      const result = operation();
+      reportIfBlocking(operationName, startedAt, attempt, lockWait);
+      return result;
+    } catch (error) {
+      const busy = isSqliteBusyError(error);
+      lockWait ||= busy;
+      const leftMs = waitMs - (now() - startedAt);
+      if (!busy || leftMs <= 0) {
+        reportIfBlocking(operationName, startedAt, attempt, lockWait);
+        throw error;
+      }
+      sleepSync(Math.min(delayMs, leftMs));
+      delayMs = Math.min(SQLITE_LOCK_POLL_MAX_DELAY_MS, delayMs * 2);
+    }
+  }
+}

package/src/data/ticker-store.ts

@@ -1,27 +1,47 @@
 import type { Database } from "bun:sqlite";
+import { createSqliteAccess, type SqliteAccess } from "./sqlite/access";
 import { serializeJson } from "./sqlite/json";
-import { withSqliteBusyRetry } from "./sqlite/retry";
 
 export interface StoredTickerRecord {
   symbol: string;
   metadata: string;
 }
 
+/** A queued write's row (null metadata for a delete), one object per write. */
+interface PendingTicker {
+  metadata: string | null;
+}
+
 export class TickerStore {
-  constructor(private readonly db: Database) {}
+  /** Writes queued behind another program's lock, by symbol, read back until they land. */
+  private readonly pending = new Map<string, PendingTicker>();
+
+  constructor(
+    private readonly db: Database,
+    private readonly access: SqliteAccess = createSqliteAccess(db),
+  ) {}
 
   getAll(): StoredTickerRecord[] {
-    return withSqliteBusyRetry("load tickers", () => (
+    const rows = this.access.run("load tickers", () => (
       this.db
         .query<StoredTickerRecord, []>(
           "SELECT symbol, metadata FROM tickers ORDER BY symbol",
         )
         .all()
     ));
+    if (this.pending.size === 0) return rows;
+    const bySymbol = new Map(rows.map((row) => [row.symbol, row]));
+    for (const [symbol, { metadata }] of this.pending) {
+      if (metadata == null) bySymbol.delete(symbol);
+      else bySymbol.set(symbol, { symbol, metadata });
+    }
+    return [...bySymbol.values()].sort((a, b) => (a.symbol < b.symbol ? -1 : a.symbol > b.symbol ? 1 : 0));
   }
 
   get(symbol: string): string | null {
-    const row = withSqliteBusyRetry("load ticker", () => (
+    const pending = this.pending.get(symbol);
+    if (pending) return pending.metadata;
+    const row = this.access.run("load ticker", () => (
       this.db
         .query<{ metadata: string }, [string]>(
           "SELECT metadata FROM tickers WHERE symbol = ?",
@@ -32,34 +52,48 @@ export class TickerStore {
   }
 
   save(symbol: string, metadata: unknown): void {
-    withSqliteBusyRetry("save ticker", () => {
-      this.db
-        .query(
-          `INSERT OR REPLACE INTO tickers (symbol, metadata, updated_at)
-           VALUES (?, ?, ?)`,
-        )
-        .run(symbol, serializeJson(metadata), Date.now());
-    });
+    this.saveMany([{ symbol, metadata }]);
   }
 
   /** Saves every record in one transaction: one write for a whole import, not one per ticker. */
   saveMany(records: ReadonlyArray<{ symbol: string; metadata: unknown }>): void {
     if (records.length === 0) return;
-    withSqliteBusyRetry("save tickers", () => {
-      const statement = this.db.query(
-        `INSERT OR REPLACE INTO tickers (symbol, metadata, updated_at)
-         VALUES (?, ?, ?)`,
-      );
-      const now = Date.now();
-      this.db.transaction(() => {
-        for (const record of records) statement.run(record.symbol, serializeJson(record.metadata), now);
-      })();
-    });
+    const rows = records.map((record) => ({ symbol: record.symbol, metadata: serializeJson(record.metadata) }));
+    const now = Date.now();
+    this.write(
+      rows.length === 1 ? "save ticker" : "save tickers",
+      rows.map((row) => [row.symbol, { metadata: row.metadata }]),
+      () => {
+        const statement = this.db.query(
+          `INSERT OR REPLACE INTO tickers (symbol, metadata, updated_at)
+           VALUES (?, ?, ?)`,
+        );
+        this.db.transaction(() => {
+          for (const row of rows) statement.run(row.symbol, row.metadata, now);
+        })();
+      },
+    );
   }
 
   delete(symbol: string): void {
-    withSqliteBusyRetry("delete ticker", () => {
+    this.write("delete ticker", [[symbol, { metadata: null }]], () => {
       this.db.query("DELETE FROM tickers WHERE symbol = ?").run(symbol);
     });
   }
+
+  /**
+   * Runs `statement` through the write queue, reading `entries` back until it
+   * lands. A write of one row replaces a queued write of the same row.
+   */
+  private write(operationName: string, entries: Array<[string, PendingTicker]>, statement: () => void): void {
+    for (const [symbol, entry] of entries) this.pending.set(symbol, entry);
+    this.access.write(operationName, statement, {
+      ...(entries.length === 1 ? { key: `tickers\u0000${entries[0]![0]}` } : {}),
+      settled: () => {
+        for (const [symbol, entry] of entries) {
+          if (this.pending.get(symbol) === entry) this.pending.delete(symbol);
+        }
+      },
+    });
+  }
 }