@volter/supercode-orchestrator 0.5.157 → 0.5.159

package/board/cli.mjs

@@ -26,7 +26,7 @@ import * as slowLog from '@volter/supercode-harness-sdk/slow-log';
 import { basename, dirname, join, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { homedir, tmpdir, userInfo } from 'node:os';
-import { actorOf, heldConfigWatched, heldDocumentsChanged, heldDocumentsWatched, holdBoards, retireHeldBoards, releaseHeldBoards, publishMarked, view, backingPaths, layOut, backingOf, boardExists, boardPath, briefPath, DEFAULT_BOARD, event, json, listBoards, logsDir, now, openBoard, setActor, setPrincipal, principalOf, isCaller, tx } from './store.mjs';
+import { actorOf, CALLER_RUN_ADDRESS, heldConfigWatched, heldDocumentsChanged, heldDocumentsWatched, holdBoards, retireHeldBoards, releaseHeldBoards, publishMarked, view, backingPaths, layOut, backingOf, boardExists, boardPath, briefPath, DEFAULT_BOARD, event, json, listBoards, logsDir, now, openBoard, setActor, setPrincipal, principalOf, isCaller, tx } from './store.mjs';
 import { isLocalWorkspace } from './workspace.mjs';
 import { openZtrackBoard } from './ztrack.mjs';
 import * as engine from './engine.mjs';
@@ -357,7 +357,7 @@ export async function runVerb(argv) {
   await readAgentMains(root);
   const sid = listing && !args.mine ? null : callerSession();
   // what this command records is the calling session's act (its address, as its subscriptions name it)
-  current().recorded = sid && boardExists(root, slug) ? withBoard((b) => b.db.prepare("SELECT json_extract(metadata, '$.supercode.address') AS address FROM task_runs WHERE json_extract(metadata, '$.supercode.session_id') = ? ORDER BY id DESC LIMIT 1").get(sid)?.address) ?? null : null;
+  current().recorded = sid && boardExists(root, slug) ? withBoard((b) => b.db.prepare(CALLER_RUN_ADDRESS).get(sid)?.address) ?? null : null;
   if (sid) setActor(callerAddress());
   setPrincipal(await callerPrincipal(root, sid, { teams: !sid && !READS.has(verb) }));
   // the dispatcher as the owner holds it, said once per call (owner-health.mjs; nothing is read for it)
@@ -1590,22 +1590,18 @@ async function answerVerbs(root, log) {
 
 /** One request to the owner, answered whole: the verb run in the caller's call, its output kept for the answer. A
  *  verb that takes more than a second is a slow line of its own, with the steps that ended while it ran (its own and,
- *  in this one process, any other call's that held it) and what the process used while it ran (`process`): its
- *  processor time beside the line's `ms` says whether the time was computed or waited for, with the major page faults
- *  and the waits (voluntary context switches) and preemptions (involuntary) the kernel counted. */
+ *  in this one process, any other call's that held it) and what the whole process used over the verb's wall time
+ *  (`whole_process`, slow-log.mjs processUse: every thread and every other call in flight, never the verb alone): its
+ *  processor time beside the line's `ms` says whether the time was computed or waited for, and the block reads and
+ *  major page faults say on what. */
 async function answerOne(root, line) {
   const began = performance.now();
-  const before = process.resourceUsage();
+  const before = slowLog.processCounters?.() ?? null;
   let named = null, by = null;
   try { return await answerRequest(root, line, (verb, caller) => { named = verb; by = caller; }); }
   finally {
     const ms = performance.now() - began;
-    if (named && ms > slowLog.SLOW_MS) {
-      const after = process.resourceUsage();
-      const used = { cpu_ms: Math.round((after.userCPUTime + after.systemCPUTime - before.userCPUTime - before.systemCPUTime) / 1000), major_faults: after.majorPageFault - before.majorPageFault,
-        waits: after.voluntaryContextSwitches - before.voluntaryContextSwitches, preempted: after.involuntaryContextSwitches - before.involuntaryContextSwitches };
-      recordSlow({ kind: 'call', name: `owner verb ${named}`, ms, slowest: null, extra: { caller: by, steps: slowLog.stepsWithin?.(began, performance.now()) ?? null, process: used } });
-    }
+    if (named && ms > slowLog.SLOW_MS) recordSlow({ kind: 'call', name: `owner verb ${named}`, ms, slowest: null, extra: { caller: by, steps: slowLog.stepsWithin?.(began, performance.now()) ?? null, whole_process: before ? slowLog.processUse(before) : null } });
   }
 }
 

package/board/legacy-store.mjs

@@ -0,0 +1,70 @@
+// The earlier runtime store beside a board (`ztrack.json.runtime.db`), read for the one thing the board still asks of
+// it after the import: whether the board's store holds its runs (ADR 0003).
+//
+// That store is a WAL file nobody keeps open, so every open of it is a first open: the opener builds the WAL's index
+// under an exclusive lock, and a second process opening in the same moment is refused its first statement with
+// SQLITE_BUSY_RECOVERY ("database is locked"), which SQLite does not wait out. Read-only changes nothing about that.
+// Asked on every board open, two followers opening the board on each pass met there and one of them exited (114
+// times in seven days on one install). So the earlier store's count is read once per process for that store as it
+// stands, and the read is tried again, for a bounded time, when another process is in it.
+//
+// Only the earlier store's half of the answer is kept. The board store's runs are counted on every open, as they
+// always were: its owner keeps it open and writes it through its WAL, so no stamp of its files says its rows are
+// unchanged, and a store that lost runs or was replaced must be refused by the very next open.
+import { statSync } from 'node:fs';
+import { DatabaseSync } from 'node:sqlite';
+
+// ctime as well as mtime: a file rewritten in place with its mtime put back keeps size and mtime, never ctime
+const stamp = (path, { empty = true } = {}) => { try { const s = statSync(path); return !empty && !s.size ? '' : `${s.dev}:${s.ino}:${s.size}:${s.mtimeMs}:${s.ctimeMs}`; } catch { return null; } };
+// The earlier store as it stands: its file, and its WAL while that holds anything (a write to a WAL file changes the
+// WAL, not always the file; an empty WAL is made and removed by readers).
+const standing = (legacy) => `${stamp(legacy)}|${stamp(`${legacy}-wal`, { empty: false }) ?? ''}`;
+const counted = new Map(); // an earlier store's path → { as: the stamps it was counted under, runs }
+
+/** How long a read that keeps meeting another process is tried again before it fails. The meeting this is for (two
+ * first opens of the earlier store) lasts as long as building a WAL index: of 631 collided opens measured in two
+ * processes that did nothing else, the longest took 20 ms and 4 tries. One second is fifty times that and no longer
+ * than a slow line's own bound; a lock held longer is not that meeting, and the opener says so and fails instead of
+ * holding its process on it. */
+export const CONTENTION_BOUND_MS = 1000;
+
+/** How many runs the earlier store holds (0 when it keeps no runtime rows). */
+function runsOf(legacy) {
+  const old = new DatabaseSync(legacy, { readOnly: true });
+  try {
+    if (!old.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'runtime_rows'").get()) return 0;
+    return old.prepare("SELECT COUNT(*) AS n FROM runtime_rows WHERE name = 'task_runs' AND value IS NOT NULL").get().n;
+  } finally { old.close(); }
+}
+
+/** `fn`, tried again after `wait(attempt)` while it meets another process (`contended(error)`), for at most
+ * CONTENTION_BOUND_MS: then it fails as SQLite failed it (same codes), naming the file and the wait. */
+function bounded(path, fn, { contended, wait }) {
+  const began = performance.now();
+  for (let attempt = 0; ; attempt++) {
+    try { return fn(); }
+    catch (error) {
+      if (!contended(error)) throw error;
+      const waited = Math.round(performance.now() - began);
+      if (waited >= CONTENTION_BOUND_MS) {
+        throw Object.assign(new Error(`database is locked: ${path} was held by another process for ${waited} ms over ${attempt + 1} tries (${error.message}, errcode ${error.errcode})`, { cause: error }), { code: error.code, errcode: error.errcode, errstr: error.errstr });
+      }
+      wait(attempt);
+    }
+  }
+}
+
+/**
+ * Whether the board store at `file` holds fewer runs than the earlier store `legacy` beside it: `{ here, runs }` when
+ * it does (it was created without the import), null when it does not or no earlier store is there. `countHere` counts
+ * the board store's runs, on every call. The earlier store's count is read once per process while its file and WAL
+ * stand unchanged. A read that meets another process is tried again within CONTENTION_BOUND_MS (`bounded`).
+ */
+export function unimported(file, legacy, countHere, retry) {
+  if (stamp(legacy) === null) return null;
+  const here = bounded(file, countHere, retry);
+  const as = standing(legacy);
+  let known = counted.get(legacy);
+  if (known?.as !== as) { known = { as, runs: bounded(legacy, () => runsOf(legacy), retry) }; counted.set(legacy, known); }
+  return here < known.runs ? { here, runs: known.runs } : null;
+}

package/board/store.mjs

@@ -143,6 +143,14 @@ export function listBoards(root) {
   return out;
 }
 
+// The run a session's own verb is recorded against, by the session the dispatcher wrote in the run's metadata: asked on
+// every verb a session runs (cli.mjs). The lookup and its index are written from the same two expressions, because
+// SQLite uses an expression index only for a statement that writes the expression the same way.
+const RUN_SESSION = "json_extract(metadata, '$.supercode.session_id')";
+const RUN_ADDRESS = "json_extract(metadata, '$.supercode.address')";
+export const CALLER_RUN_ADDRESS = `SELECT ${RUN_ADDRESS} AS address FROM task_runs WHERE ${RUN_SESSION} = ? ORDER BY id DESC LIMIT 1`;
+const RUN_SESSION_INDEX = 'idx_runs_supercode_session';
+
 /** Hermes's tables on a fresh board, supercode's own table and columns on every one. */
 export function layOut(db, { fresh }) {
   if (fresh) db.exec(SCHEMA);
@@ -154,6 +162,24 @@ export function layOut(db, { fresh }) {
   // each read walked every card's row. Laid out here, after the column it names exists on a store of any age and before
   // any statement is prepared; no statement names it, so a store without it is only slower.
   db.exec('CREATE INDEX IF NOT EXISTS idx_supercode_cards_subtask_of ON supercode_cards(subtask_of)');
+  // The calling session's run (CALLER_RUN_ADDRESS): without this index each lookup read every run's whole metadata,
+  // overflow pages included. `id` keeps a session's runs in the order the lookup asks for, and the address makes the
+  // index answer the lookup without reading the run. No statement names it, so a store without it is only slower, and a
+  // release that does not know it keeps it current like any other index. Its first creation reads `task_runs` once.
+  // A run whose metadata SQLite cannot read as JSON refuses the creation: the board opens without the index, says which
+  // rows to look for, and tries again on its next open. Once the index is there, SQLite refuses a write of such
+  // metadata to a run.
+  if (db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'task_runs'").get()) {
+    try { db.exec(`CREATE INDEX IF NOT EXISTS ${RUN_SESSION_INDEX} ON task_runs(${RUN_SESSION}, id, ${RUN_ADDRESS})`); }
+    catch (error) {
+      if (contention(error)) throw error;
+      const said = current().said;
+      if (!said.has(RUN_SESSION_INDEX)) {
+        said.add(RUN_SESSION_INDEX);
+        current().err(`board: warning: index ${RUN_SESSION_INDEX} on task_runs was not created (${error.message}); the board is open without it and each session's verb reads every run. The runs that refuse it: SELECT id FROM task_runs WHERE metadata IS NOT NULL AND NOT json_valid(metadata, 6)\n`);
+      }
+    }
+  }
   const effectColumns = new Set(db.prepare('PRAGMA table_info(supercode_effects)').all().map((c) => c.name));
   for (const [column, type] of Object.entries(EFFECT_COLUMNS)) if (!effectColumns.has(column)) db.exec(`ALTER TABLE supercode_effects ADD COLUMN ${column} ${type}`);
   // what changed is marked by the store itself, whoever writes it (published.mjs)

package/board/ztrack.mjs

@@ -16,6 +16,7 @@ import { basename, dirname, join, resolve } from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
 import { actorOf, contention, event, json, now, principalOf, storageFault, STORAGE_TRIES } from './store.mjs';
 import { GitDocument } from './git-document.mjs';
+import { unimported } from './legacy-store.mjs';
 import { noteStep, recordSlow } from '@volter/supercode-harness-sdk/slow-log';
 import * as slowLog from '@volter/supercode-harness-sdk/slow-log';
 
@@ -23,8 +24,11 @@ import * as slowLog from '@volter/supercode-harness-sdk/slow-log';
 const timedStep = (step, fn) => { const started = performance.now(); try { return fn(); } finally { noteStep(step, performance.now() - started); } };
 // One statement that takes this long is a step of its own, named by its text: a verb that only reads (`show`) runs in
 // no transaction and has no other step, so a hold inside it was a slow line that named no statement (8.7 s on
-// 2026-10-11, where every statement of `show` measures under 80 ms on a copy of the same store).
-const SLOW_STATEMENT_MS = 100;
+// 2026-10-11, where every statement of `show` measures under 80 ms on a copy of the same store). Every native call on
+// the board's connection is timed this way (ZtrackBoard.statement, the engine's `exec`), the backing's own included:
+// the reads of `board_state` and `board_errors` that come before each of the engine's statements, and a statement's
+// compilation, which reads the schema.
+const SLOW_STATEMENT_MS = 50;
 const timedStatement = (sql, fn) => {
   const started = performance.now();
   try { return fn(); }
@@ -137,13 +141,13 @@ class ZtrackBoard {
       boardErrors: self.errors,
       prepare(sql) {
         return {
-          get: (...args) => self.read(() => timedStatement(sql, () => self.statement(sql).get(...args))),
-          all: (...args) => self.read(() => timedStatement(sql, () => self.statement(sql).all(...args))),
-          iterate: (...args) => self.read(() => timedStatement(sql, () => self.statement(sql).all(...args)))[Symbol.iterator](),
-          run: (...args) => self.active ? self.noting(() => timedStatement(sql, () => self.statement(sql).run(...args))) : self.tx(() => timedStatement(sql, () => self.statement(sql).run(...args))),
+          get: (...args) => self.read(() => self.statement(sql).get(...args)),
+          all: (...args) => self.read(() => self.statement(sql).all(...args)),
+          iterate: (...args) => self.read(() => self.statement(sql).all(...args))[Symbol.iterator](),
+          run: (...args) => self.active ? self.noting(() => self.statement(sql).run(...args)) : self.tx(() => self.statement(sql).run(...args)),
         };
       },
-      exec(sql) { return self.active ? self.noting(() => self.conn.exec(sql)) : self.tx(() => self.conn.exec(sql)); },
+      exec(sql) { const run = () => timedStatement(sql, () => self.conn.exec(sql)); return self.active ? self.noting(run) : self.tx(run); },
       close() { self.close(); },
     };
     // A board's first open in this process recovers what its store accepted and derives what git owes for it (tx), so
@@ -179,7 +183,11 @@ class ZtrackBoard {
 
   statement(sql) {
     let statement = this.statements.get(sql);
-    if (!statement) { statement = this.conn.prepare(sql); this.statements.set(sql, statement); }
+    if (!statement) {
+      const prepared = timedStatement(`prepare: ${sql}`, () => this.conn.prepare(sql));
+      statement = { get: (...args) => timedStatement(sql, () => prepared.get(...args)), all: (...args) => timedStatement(sql, () => prepared.all(...args)), run: (...args) => timedStatement(sql, () => prepared.run(...args)) };
+      this.statements.set(sql, statement);
+    }
     return statement;
   }
   one(sql, ...args) { return this.statement(sql).get(...args); }
@@ -229,18 +237,14 @@ class ZtrackBoard {
   // one such store, created by an opener nobody named, sent the dispatcher on to start a second session on every
   // running card). The board refuses it, so nothing runs on a board without its history.
   refuseUnimported(legacy) {
-    if (!existsSync(legacy)) return;
-    const here = this.conn.prepare('SELECT COUNT(*) AS n FROM main.task_runs').get().n;
-    let runs = 0;
-    const old = new DatabaseSync(legacy, { readOnly: true });
-    try {
-      if (old.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'runtime_rows'").get()) {
-        runs = old.prepare("SELECT COUNT(*) AS n FROM runtime_rows WHERE name = 'task_runs' AND value IS NOT NULL").get().n;
-      }
-    } finally { old.close(); }
-    if (here >= runs) return;
+    // the earlier store's count is read once per process while that store stands unchanged, and a read that meets
+    // another process is tried again for a bounded time (legacy-store.mjs): the earlier store is opened by nobody
+    // else, so two openers at once refuse each other. This store's own runs are counted on every open.
+    const short = unimported(this.file, legacy, () => this.conn.prepare('SELECT COUNT(*) AS n FROM main.task_runs').get().n,
+      { contended: sqliteContention, wait: (attempt) => timedStep('board open: earlier store contention backoff', () => backoff(attempt)) });
+    if (!short) return;
     this.conn.close();
-    throw new Error(`${this.file} holds ${here} runs, but ${legacy} holds ${runs}: it was created without importing them. Move it aside (keep it; writes made to it are not in the earlier store) and run any board command to import.`);
+    throw new Error(`${this.file} holds ${short.here} runs, but ${legacy} holds ${short.runs}: it was created without importing them. Move it aside (keep it; writes made to it are not in the earlier store) and run any board command to import.`);
   }
 
   // A board created from the previous runtime store imports its rows once; the old file stays as it was.
@@ -352,7 +356,8 @@ class ZtrackBoard {
     if (this.active) return this.noting(fn);
     this.observeForRead();
     for (let attempt = 0; ; attempt++) {
-      try { return fn(); } catch (error) { if (!sqliteContention(error)) throw error; backoff(attempt); }
+      // each sleep between tries is a step: its count on a slow line is how many times the read was tried again
+      try { return fn(); } catch (error) { if (!sqliteContention(error)) throw error; timedStep('board read: contention backoff', () => backoff(attempt)); }
     }
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-orchestrator",
-  "version": "0.5.157",
+  "version": "0.5.159",
   "type": "module",
   "description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
   "exports": {
@@ -24,7 +24,7 @@
     "check-types": "node scripts/generate-types.mjs --check"
   },
   "dependencies": {
-    "@volter/supercode-harness-sdk": "^0.3.96",
+    "@volter/supercode-harness-sdk": "^0.3.97",
     "@volter/teams": "0.2.65",
     "@volter/ztrack": "^2.4.23",
     "yaml": "^2.9.0"