@gloom-sh/terminal 0.19.2 → 0.19.3

package/src/data/sqlite/checkpointer.ts

@@ -1,6 +1,7 @@
 import { resolve } from "path";
 import { debugLog } from "../../utils/debug-log";
 import { logFailure } from "../../utils/log-failure";
+import { recordPerfSample } from "../../utils/perf-marks";
 
 /**
  * Pages SQLite lets the WAL grow to before the connection that commits
@@ -13,13 +14,25 @@ export const SQLITE_BACKSTOP_AUTOCHECKPOINT_PAGES = 10_000;
 const CHECKPOINT_DELAY_MS = 1_000;
 /** How long closing waits for the checkpointer to let go of the file (Windows cannot delete an open one). */
 const CLOSE_WAIT_MS = 1_000;
+/**
+ * How long the thread may go without answering while it owes an answer
+ * (opening its connection, or a checkpoint) before it counts as stuck. A
+ * stuck thread flushes nothing, and the app's commits of user data skip
+ * their flush while it runs, so past this they flush again. The slowest
+ * checkpoint measured on Windows took under a second, and the slowest
+ * flush in stall reports 8 s.
+ */
+const STUCK_AFTER_MS = 30_000;
 
 const checkpointLog = debugLog.createLogger("sqlite");
 
 /**
  * Runs on its own thread with its own connection. A passive checkpoint
  * copies what it can without waiting on a lock and never stops the app's
- * connection from reading or writing meanwhile.
+ * connection from reading or writing meanwhile. It flushes the WAL to disk
+ * before copying it, and the database after: that flush is what makes the
+ * app's writes of user data durable (access.ts), so it never runs with
+ * `synchronous=OFF`.
  */
 const WORKER_SOURCE = `
 import { Database } from "bun:sqlite";
@@ -32,6 +45,7 @@ self.onmessage = (event) => {
     try {
       db = new Database(message.path);
       db.exec("PRAGMA busy_timeout=1000");
+      db.exec("PRAGMA synchronous=FULL");
       postMessage({ type: "ready" });
     } catch (error) {
       postMessage({ type: "failed", message: String(error && error.message || error) });
@@ -43,7 +57,15 @@ self.onmessage = (event) => {
     const startedAt = performance.now();
     try {
       const result = db.query("PRAGMA wal_checkpoint(PASSIVE)").get();
-      postMessage({ type: "checkpointed", ms: performance.now() - startedAt, result });
+      const endedAt = performance.now();
+      // Wall-clock times, so a pause on the app's thread can be lined up with it.
+      postMessage({
+        type: "checkpointed",
+        ms: endedAt - startedAt,
+        startedAt: performance.timeOrigin + startedAt,
+        endedAt: performance.timeOrigin + endedAt,
+        result,
+      });
     } catch (error) {
       postMessage({ type: "failed", message: String(error && error.message || error) });
     }
@@ -65,6 +87,13 @@ self.onmessage = (event) => {
 };
 `;
 
+export interface WalCheckpointerOptions {
+  /** Overrides how long the thread may owe an answer before it counts as stuck. */
+  stuckAfterMs?: number;
+  /** Overrides the thread's code, so a test can run one that never answers. */
+  workerSource?: string;
+}
+
 export interface WalCheckpointer {
   /** Writes were committed: the WAL is copied into the database shortly, off this thread. */
   nudge(): void;
@@ -76,6 +105,8 @@ interface WorkerMessage {
   type: "ready" | "failed" | "checkpointed";
   message?: string;
   ms?: number;
+  startedAt?: number;
+  endedAt?: number;
   result?: { busy?: number; log?: number; checkpointed?: number };
 }
 
@@ -87,15 +118,21 @@ interface WorkerMessage {
  * disk, on the thread that asked for the write. That is the app's only
  * thread, and the copy takes 150 ms to a second on a laptop, more under an
  * antivirus. The app's connection leaves it to this instead (see
- * database.ts), and `onFailure` puts SQLite's own schedule back when the
- * thread cannot run.
+ * database.ts), along with flushing what it commits, and `onFailure` puts
+ * SQLite's own schedule and the app's flushes back when the thread cannot
+ * run.
  *
  * Null where there is no Worker to run it on.
  */
-export function startWalCheckpointer(dbPath: string, onFailure: () => void): WalCheckpointer | null {
+export function startWalCheckpointer(
+  dbPath: string,
+  onFailure: () => void,
+  options: WalCheckpointerOptions = {},
+): WalCheckpointer | null {
   if (typeof Worker === "undefined" || typeof Blob === "undefined" || typeof URL.createObjectURL !== "function") return null;
   let worker: Worker;
-  const source = URL.createObjectURL(new Blob([WORKER_SOURCE], { type: "application/javascript" }));
+  const source = URL.createObjectURL(new Blob([options.workerSource ?? WORKER_SOURCE], { type: "application/javascript" }));
+  const stuckAfterMs = options.stuckAfterMs ?? STUCK_AFTER_MS;
   try {
     worker = new Worker(source);
   } catch (error) {
@@ -110,10 +147,21 @@ export function startWalCheckpointer(dbPath: string, onFailure: () => void): Wal
   let running = false;
   let pending = false;
   let timer: ReturnType<typeof setTimeout> | null = null;
+  /** Messages the thread has yet to answer, and the clock on its next answer. */
+  let owed = 0;
+  let watchdog: ReturnType<typeof setTimeout> | null = null;
+
+  const stopWatching = () => {
+    if (watchdog) clearTimeout(watchdog);
+    watchdog = null;
+  };
 
   const fail = (reason: string) => {
     if (failed || stopped) return;
     failed = true;
+    stopWatching();
+    if (timer) clearTimeout(timer);
+    timer = null;
     checkpointLog.warn("The WAL checkpointer stopped; commits checkpoint themselves", { reason });
     try {
       onFailure();
@@ -141,13 +189,42 @@ export function startWalCheckpointer(dbPath: string, onFailure: () => void): Wal
     }
   };
 
+  /**
+   * Restarts the clock whenever the thread answers and still owes one. A
+   * thread stuck without an error event would otherwise leave user writes
+   * unflushed until the backstop checkpoint or close.
+   */
+  const watch = () => {
+    stopWatching();
+    if (owed === 0 || failed || stopped) return;
+    watchdog = setTimeout(() => {
+      watchdog = null;
+      fail(`no answer in ${stuckAfterMs} ms`);
+      terminate();
+      URL.revokeObjectURL(source);
+    }, stuckAfterMs);
+    (watchdog as { unref?: () => void }).unref?.();
+  };
+
+  const ask = (message: { type: string; path?: string; closed?: SharedArrayBuffer }): boolean => {
+    if (!post(message)) return false;
+    owed += 1;
+    if (owed === 1) watch();
+    return true;
+  };
+
+  const answered = () => {
+    owed = Math.max(0, owed - 1);
+    watch();
+  };
+
   const schedule = () => {
     if (timer || running || failed || stopped) return;
     timer = setTimeout(() => {
       timer = null;
       if (failed || stopped) return;
       pending = false;
-      running = post({ type: "checkpoint" });
+      running = ask({ type: "checkpoint" });
     }, CHECKPOINT_DELAY_MS);
     (timer as { unref?: () => void }).unref?.();
   };
@@ -155,6 +232,7 @@ export function startWalCheckpointer(dbPath: string, onFailure: () => void): Wal
   worker.onmessage = (event: MessageEvent<WorkerMessage>) => {
     const message = event.data;
     if (message.type === "ready") {
+      answered();
       URL.revokeObjectURL(source);
       return;
     }
@@ -163,14 +241,22 @@ export function startWalCheckpointer(dbPath: string, onFailure: () => void): Wal
       return;
     }
     if (message.type === "checkpointed") {
+      answered();
       running = false;
+      // On its own thread, so never the step a stall of the app's names.
+      recordPerfSample("sqlite.checkpoint", message.ms ?? 0, {
+        pages: message.result?.log,
+        copied: message.result?.checkpointed,
+        startedAt: Math.round(message.startedAt ?? 0),
+        endedAt: Math.round(message.endedAt ?? 0),
+      }, { lastStep: false });
       if (pending) schedule();
     }
   };
   worker.onerror = (event) => {
     fail(event.message);
   };
-  if (!post({ type: "open", path: resolve(dbPath), closed })) {
+  if (!ask({ type: "open", path: resolve(dbPath), closed })) {
     terminate();
     URL.revokeObjectURL(source);
     return null;
@@ -186,6 +272,7 @@ export function startWalCheckpointer(dbPath: string, onFailure: () => void): Wal
       if (stopped) return;
       if (timer) clearTimeout(timer);
       timer = null;
+      stopWatching();
       const asked = !failed && post({ type: "close" });
       stopped = true;
       if (!asked) {

package/src/data/sqlite/database.ts

@@ -1,10 +1,14 @@
 import { Database } from "bun:sqlite";
+import { closeSync, openSync } from "fs";
+import { logFailure } from "../../utils/log-failure";
+import { restrictFileSync } from "../../utils/private-files";
 import { createSqliteAccess, type SqliteAccess } from "./access";
 import {
   SQLITE_BACKSTOP_AUTOCHECKPOINT_PAGES,
   SQLITE_DEFAULT_AUTOCHECKPOINT_PAGES,
   startWalCheckpointer,
   type WalCheckpointer,
+  type WalCheckpointerOptions,
 } from "./checkpointer";
 import { SQLITE_BUSY_TIMEOUT_MS, withSqliteBusyRetry } from "./retry";
 
@@ -74,6 +78,8 @@ export interface SqliteDatabaseOptions {
    * another program's lock (see access.ts). Commands and tests wait it out.
    */
   interactive?: boolean;
+  /** Tests only: the WAL checkpointer's thread and how long it may go silent. */
+  checkpointer?: WalCheckpointerOptions;
 }
 
 export class SqliteDatabase {
@@ -82,6 +88,18 @@ export class SqliteDatabase {
   private readonly checkpointer: WalCheckpointer | null = null;
 
   constructor(dbPath: string, options: SqliteDatabaseOptions = {}) {
+    const onDisk = dbPath !== "" && dbPath !== ":memory:" && !dbPath.startsWith("file:");
+    // The database holds the Cloud session: created readable by the user
+    // alone, so the WAL and shared-memory files SQLite makes beside it with
+    // the same mode are too, and an older one made so.
+    if (onDisk) {
+      try {
+        closeSync(openSync(dbPath, "a", 0o600));
+        restrictFileSync(dbPath);
+      } catch (error) {
+        logFailure("data.sqlite", "Restricting the database file to its user failed")(error);
+      }
+    }
     this.connection = new Database(dbPath);
     // Opening waits for another program's lock: it happens once, and the app
     // has nothing to show without its data.
@@ -90,28 +108,43 @@ export class SqliteDatabase {
       this.connection.query<{ journal_mode: string }, []>("PRAGMA journal_mode=WAL").get()?.journal_mode
     ));
     // In WAL mode commits can stop flushing the disk and the database stays
-    // whole; user data still flushes per write (access.ts). Where WAL does not
-    // take (a network share, an in-memory database), every commit flushes.
+    // whole; user data is flushed soon after by the checkpointer, or per write
+    // where it does not run (access.ts). Where WAL does not take (a network
+    // share, an in-memory database), every commit flushes.
     const wal = journalMode?.toLowerCase() === "wal";
+    if (onDisk && wal) {
+      try {
+        restrictFileSync(`${dbPath}-wal`);
+        restrictFileSync(`${dbPath}-shm`);
+      } catch (error) {
+        logFailure("data.sqlite", "Restricting the database's WAL files to their user failed")(error);
+      }
+    }
     this.connection.exec(wal ? "PRAGMA synchronous=NORMAL" : "PRAGMA synchronous=FULL");
     withSqliteBusyRetry("initialize sqlite schema", () => {
       this.connection.exec(SCHEMA);
     });
     // The app's statements choose how long to wait for a lock themselves.
     if (options.interactive) this.connection.exec("PRAGMA busy_timeout=0");
-    // The app's commits leave copying the WAL into the database to a thread
-    // of its own, and only do it themselves past a WAL ten times the usual.
+    // The app's commits leave copying the WAL into the database, and flushing
+    // it, to a thread of its own, and only copy it themselves past a WAL ten
+    // times the usual.
+    let checkpointerRunning = false;
     if (options.interactive && wal && dbPath !== ":memory:") {
       this.checkpointer = startWalCheckpointer(dbPath, () => {
+        checkpointerRunning = false;
         this.connection.exec(`PRAGMA wal_autocheckpoint=${SQLITE_DEFAULT_AUTOCHECKPOINT_PAGES}`);
-      });
-      if (this.checkpointer) this.connection.exec(`PRAGMA wal_autocheckpoint=${SQLITE_BACKSTOP_AUTOCHECKPOINT_PAGES}`);
+      }, options.checkpointer);
+      if (this.checkpointer) {
+        checkpointerRunning = true;
+        this.connection.exec(`PRAGMA wal_autocheckpoint=${SQLITE_BACKSTOP_AUTOCHECKPOINT_PAGES}`);
+      }
     }
     const checkpointer = this.checkpointer;
     this.access = createSqliteAccess(this.connection, {
       interactive: options.interactive,
       relaxedCommits: wal,
-      ...(checkpointer ? { onCommit: () => checkpointer.nudge() } : {}),
+      ...(checkpointer ? { onCommit: () => checkpointer.nudge(), flushedElsewhere: () => checkpointerRunning } : {}),
     });
   }