@volter/supercode-orchestrator 0.5.159 → 0.5.161

package/board/cli.mjs

@@ -26,7 +26,7 @@ import * as slowLog from '@volter/supercode-harness-sdk/slow-log';
 import { basename, dirname, join, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { homedir, tmpdir, userInfo } from 'node:os';
-import { actorOf, CALLER_RUN_ADDRESS, heldConfigWatched, heldDocumentsChanged, heldDocumentsWatched, holdBoards, retireHeldBoards, releaseHeldBoards, publishMarked, view, backingPaths, layOut, backingOf, boardExists, boardPath, briefPath, DEFAULT_BOARD, event, json, listBoards, logsDir, now, openBoard, setActor, setPrincipal, principalOf, isCaller, tx } from './store.mjs';
+import { actorOf, CALLER_RUN_ADDRESS, SESSION_CURRENT_RUNS, SESSION_RUNS, sessionIndexed, heldConfigWatched, heldDocumentsChanged, heldDocumentsWatched, holdBoards, retireHeldBoards, releaseHeldBoards, publishMarked, view, backingPaths, layOut, backingOf, boardExists, boardPath, briefPath, DEFAULT_BOARD, event, json, listBoards, logsDir, now, openBoard, setActor, setPrincipal, principalOf, isCaller, tx } from './store.mjs';
 import { isLocalWorkspace } from './workspace.mjs';
 import { openZtrackBoard } from './ztrack.mjs';
 import * as engine from './engine.mjs';
@@ -1406,18 +1406,38 @@ function workerRun(db, id, sessionId) {
  * worked card is held to nothing here. The newest run alone named one card, and a session adopted into a second card
  * was refused every write on its first (volter-org t_9a9c2a9c: no comment, heartbeat or closed subtask there, read
  * by the board as no progress).
+ *
+ * Asked on every verb a session runs, so it reads the session's own runs and no others where it can (store.mjs). The
+ * newest run naming the session is the newest whose text holds its id anywhere, an observation of another run's
+ * included, which no index answers: it is looked for only when its answer can add a card (one of the session's own
+ * runs is on a card it does not already work and is still the last worker of), and then from the newest run back to
+ * the session's own newest, which its text names too. A session whose runs are on more cards than OWN_CARDS_ASKED is
+ * not asked card by card first: the newest run is looked for and its card asked alone, which answers the same.
+ * A board without the session's index (store.mjs sessionIndexed) is asked from its cards and by text alone, which
+ * reads no run it cannot read.
  */
+const OWN_CARDS_ASKED = 8;
 function ownCards(db, sessionId) {
   const worked = new Set(), held = new Set();
   const like = `%"session_id":"${sessionId}"%`;
-  for (const row of db.prepare('SELECT t.id, r.metadata FROM tasks t JOIN task_runs r ON r.id = t.current_run_id WHERE r.metadata LIKE ?').all(like)) {
+  const byIndex = sessionIndexed(db);
+  for (const row of db.prepare(byIndex ? SESSION_CURRENT_RUNS : 'SELECT t.id, r.metadata FROM tasks t JOIN task_runs r ON r.id = t.current_run_id WHERE r.metadata LIKE ?').all(...(byIndex ? [sessionId, like] : [like]))) {
     const session = json(row.metadata, {})?.supercode;
     if (session?.session_id !== sessionId) continue;
     held.add(row.id);
     if ((session.slot ?? 'worker') === 'worker') worked.add(row.id);
   }
-  const newest = db.prepare('SELECT task_id FROM task_runs WHERE metadata LIKE ? ORDER BY id DESC LIMIT 1').get(like);
-  if (newest && engine.lastSession(db, newest.task_id)?.session_id === sessionId) { worked.add(newest.task_id); held.add(newest.task_id); }
+  const anywhere = 'SELECT task_id FROM task_runs WHERE metadata LIKE ? ORDER BY id DESC LIMIT 1';
+  const asked = new Map();
+  const lastWorker = (id) => { if (!asked.has(id)) asked.set(id, engine.lastSession(db, id)?.session_id === sessionId); return asked.get(id); };
+  let newest;
+  if (byIndex) {
+    const runs = db.prepare(SESSION_RUNS).all(sessionId);
+    const between = [...new Set(runs.map((run) => run.task_id))].filter((id) => !worked.has(id));
+    if (!between.length || (between.length <= OWN_CARDS_ASKED && !between.some(lastWorker))) return { worked, held };
+    newest = db.prepare('SELECT task_id FROM task_runs WHERE id >= ? AND metadata LIKE ? ORDER BY id DESC LIMIT 1').get(runs.at(-1).id, like) ?? db.prepare(anywhere).get(like);
+  } else newest = db.prepare(anywhere).get(like);
+  if (newest && lastWorker(newest.task_id)) { worked.add(newest.task_id); held.add(newest.task_id); }
   return { worked, held };
 }
 

package/board/dispatch.mjs

@@ -40,7 +40,7 @@ import { homedir, hostname, tmpdir, totalmem } from 'node:os';
 import { basename, dirname, join } from 'node:path';
 import { createHash, randomUUID } from 'node:crypto';
 import { fileURLToPath } from 'node:url';
-import { view, boardCopyOf, boardPath, briefPath, claimer, claimHolds, copyRefusal, event, json, listBoards, logsDir, newClaim, now, openBoard, setActor, storageFault, tx } from './store.mjs';
+import { view, boardCopyOf, boardPath, briefPath, claimer, claimHolds, copyRefusal, event, json, listBoards, logsDir, newClaim, now, openBoard, pendingLaunches, sessionBound, setActor, storageFault, tx } from './store.mjs';
 import { doors, fire, loadWorkflow, resumeOf, runSpec, runWorkingSeconds, requestSessionClose, retireSupersededRuns, refreshCardInputs, refreshRelationAnswers, transitionsFor, scopeOf, statusOf, projectStatus, storedStatus, templateValue, workflowFor } from './workflow.mjs';
 import { checkWorkspace, assertWorkspaceOwner, notePresence, resolveWorkspace } from './workspace.mjs';
 import { capacity, harnessDefaults, holdings } from './capacity.mjs';
@@ -1822,11 +1822,8 @@ function bindLaunch(board, row, session, pane, sid) {
 /** A fast worker can reach its first board door before open's reply. Bind only the daemon's keyed pane. */
 export async function reconcileCallerLaunch(board, sid) {
   fleetRoot = board.root;
-  if (view(board, (db) => db.prepare(`SELECT 1 FROM task_runs r JOIN tasks t ON t.current_run_id=r.id
-    WHERE json_extract(r.metadata, '$.supercode.session_id')=? AND COALESCE(json_extract(r.metadata, '$.supercode.launch_pending'),0)=0`).get(sid))) return;
-  const rows = view(board, (db) => db.prepare(`SELECT r.id, r.task_id, r.metadata FROM task_runs r
-    JOIN tasks t ON t.current_run_id=r.id WHERE r.ended_at IS NULL
-    AND json_extract(r.metadata, '$.supercode.launch_pending') = 1`).all());
+  if (view(board, (db) => db.prepare(sessionBound(db)).get(sid))) return;
+  const rows = view(board, (db) => db.prepare(pendingLaunches(db)).all());
   const machine = localMachine();
   const harness = boardCaller()?.harness ?? (sid === process.env.CODEX_THREAD_ID ? 'codex' : 'claude-code');
   const candidates = rows.filter((row) => {

package/board/store.mjs

@@ -14,7 +14,7 @@ import { fenced } from './fence.mjs';
 import { noteStep } from '@volter/supercode-harness-sdk/slow-log';
 import { redactValue } from '@volter/teams/redact';
 import { randomBytes } from 'node:crypto';
-import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs';
+import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, writeFileSync, writeSync } from 'node:fs';
 import { hostname } from 'node:os';
 import { dirname, join } from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
@@ -148,8 +148,79 @@ export function listBoards(root) {
 // SQLite uses an expression index only for a statement that writes the expression the same way.
 const RUN_SESSION = "json_extract(metadata, '$.supercode.session_id')";
 const RUN_ADDRESS = "json_extract(metadata, '$.supercode.address')";
+const RUN_LAUNCH_PENDING = "json_extract(metadata, '$.supercode.launch_pending')";
 export const CALLER_RUN_ADDRESS = `SELECT ${RUN_ADDRESS} AS address FROM task_runs WHERE ${RUN_SESSION} = ? ORDER BY id DESC LIMIT 1`;
 const RUN_SESSION_INDEX = 'idx_runs_supercode_session';
+const RUN_LAUNCH_PENDING_INDEX = 'idx_runs_supercode_launch_pending';
+// The indexes on a run's metadata. The second holds only the runs whose launch is pending (7 of 1,387 on the store
+// these were measured for), in the runs' own order.
+const RUN_INDEXES = [
+  [RUN_SESSION_INDEX, `(${RUN_SESSION}, id, ${RUN_ADDRESS})`],
+  [RUN_LAUNCH_PENDING_INDEX, `(id) WHERE ${RUN_LAUNCH_PENDING} = 1`],
+];
+
+// What every verb a session runs asks about its caller (cli.mjs ownCards, dispatch.mjs reconcileCallerLaunch). Each
+// starts from the runs an index names (the session's, or the pending launches) and reaches the card through the run's
+// own `task_id`, the card's key: a card's current run is one of its own runs, and the join says both. Starting from the
+// cards instead read every card's row to find the few that have a current run (810 of the 843 pages each of these
+// statements read, whatever the session, on a store of 2,636 cards with 5 of them running). `r` is read from the session's
+// index alone and the run's metadata only as `m`, after the card is known to name the run as current, so a session's
+// earlier runs cost a page each and none of their metadata. The rows come in the cards' own order, as they did.
+//
+// Each of these is asked only of a board that has the index it starts from (sessionIndexed, sessionBound,
+// pendingLaunches). Without the index it would read every run, and a board is without it exactly when one of its runs
+// holds metadata SQLite cannot read as JSON (layOut), which `json_extract` refuses: the statement would fail on a run
+// it had no reason to read. Such a board is asked the earlier statements instead, which start from the cards and read
+// only their current runs. The board is asked which it has at each call, not told at its open: the owner holds a board
+// open for hours, and the index may be made meanwhile by another process's open.
+const ofRun = (expression, run) => expression.replace('(metadata,', `(${run}.metadata,`);
+const CURRENT_RUN_OF = 'JOIN tasks t ON t.id = r.task_id AND t.current_run_id = r.id';
+/** The cards whose current run names the session (`?`), with that run's metadata; `LIKE ?` as the run's text says it. */
+export const SESSION_CURRENT_RUNS = `SELECT t.id, m.metadata FROM task_runs r ${CURRENT_RUN_OF} CROSS JOIN task_runs m ON m.id = r.id
+  WHERE ${ofRun(RUN_SESSION, 'r')} = ? AND m.metadata LIKE ? ORDER BY t.rowid`;
+/** Every run that names the session (`?`) as its own, oldest first, with its card. */
+export const SESSION_RUNS = `SELECT id, task_id FROM task_runs WHERE ${RUN_SESSION} = ? ORDER BY id`;
+const SESSION_BOUND = `SELECT 1 FROM task_runs r ${CURRENT_RUN_OF} CROSS JOIN task_runs m ON m.id = r.id
+  WHERE ${ofRun(RUN_SESSION, 'r')} = ? AND COALESCE(${ofRun(RUN_LAUNCH_PENDING, 'm')},0)=0`;
+// The runs come first (CROSS JOIN): the planner cannot know how few the index holds, and reads every card otherwise.
+const PENDING_LAUNCHES = `SELECT r.id, r.task_id, r.metadata FROM task_runs r CROSS ${CURRENT_RUN_OF}
+  WHERE r.ended_at IS NULL AND ${ofRun(RUN_LAUNCH_PENDING, 'r')} = 1 ORDER BY t.rowid`;
+// the earlier statements, from the cards: they read every card and the current runs alone
+const CURRENT_RUN = 'FROM task_runs r JOIN tasks t ON t.current_run_id=r.id';
+const SESSION_BOUND_BY_CARDS = `SELECT 1 ${CURRENT_RUN} WHERE ${ofRun(RUN_SESSION, 'r')}=? AND COALESCE(${ofRun(RUN_LAUNCH_PENDING, 'r')},0)=0`;
+const PENDING_LAUNCHES_BY_CARDS = `SELECT r.id, r.task_id, r.metadata ${CURRENT_RUN} WHERE r.ended_at IS NULL AND ${ofRun(RUN_LAUNCH_PENDING, 'r')} = 1`;
+const indexed = (db, index) => Boolean(db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?").get(index));
+/** Whether the board has the index SESSION_CURRENT_RUNS and SESSION_RUNS start from. */
+export const sessionIndexed = (db) => indexed(db, RUN_SESSION_INDEX);
+/** The statement that says whether a card's current run names the session (`?`) with its launch already bound. */
+export const sessionBound = (db) => (sessionIndexed(db) ? SESSION_BOUND : SESSION_BOUND_BY_CARDS);
+/** The statement that lists the open current runs whose launch is still pending. */
+export const pendingLaunches = (db) => (indexed(db, RUN_LAUNCH_PENDING_INDEX) ? PENDING_LAUNCHES : PENDING_LAUNCHES_BY_CARDS);
+
+const EVENT_KIND_INDEX = 'idx_events_task_kind';
+// creatorOf's first `created` event and the latest review events are point lookups in this order. The latest event
+// of each kind can group from the index alone; IN-kind lookups still sort, but read only the selected kinds. Keep
+// idx_events_task: statements asking for events since a time need its (task_id, created_at) order.
+// Installed here on every backing's open, including an existing store, before any engine statement is prepared.
+// Its first creation scans task_events once, sorts the keys and writes an entry per event (no payload in the index).
+// SQLite can reuse free pages; this does not vacuum the store. It is synchronous and can hold the owner's loop for
+// seconds, so say so BEFORE starting and report its time afterward. Write the owner's stderr synchronously even
+// inside a caller's call: that call's err is buffered until its answer, after the migration has held the loop.
+function layOutEventIndex(db) {
+  if (!db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'task_events'").get() || indexed(db, EVENT_KIND_INDEX)) return;
+  const report = (text) => { writeSync(2, text); if (current().kind === 'caller') current().err(text); };
+  report(`board: creating index ${EVENT_KIND_INDEX} on task_events: one-time table scan, key sort and index write; this synchronous migration holds the owner event loop until it finishes and can take seconds\n`);
+  const began = performance.now();
+  let made = false;
+  try {
+    db.exec(`CREATE INDEX IF NOT EXISTS ${EVENT_KIND_INDEX} ON task_events(task_id, kind, id)`);
+    made = true;
+  } finally {
+    const ms = performance.now() - began;
+    noteStep(`board open index ${EVENT_KIND_INDEX}`, ms);
+    report(`board: index ${EVENT_KIND_INDEX} ${made ? 'created' : 'creation failed'} after ${Math.round(ms)} ms\n`);
+  }
+}
 
 /** Hermes's tables on a fresh board, supercode's own table and columns on every one. */
 export function layOut(db, { fresh }) {
@@ -162,22 +233,28 @@ export function layOut(db, { fresh }) {
   // each read walked every card's row. Laid out here, after the column it names exists on a store of any age and before
   // any statement is prepared; no statement names it, so a store without it is only slower.
   db.exec('CREATE INDEX IF NOT EXISTS idx_supercode_cards_subtask_of ON supercode_cards(subtask_of)');
-  // The calling session's run (CALLER_RUN_ADDRESS): without this index each lookup read every run's whole metadata,
-  // overflow pages included. `id` keeps a session's runs in the order the lookup asks for, and the address makes the
-  // index answer the lookup without reading the run. No statement names it, so a store without it is only slower, and a
-  // release that does not know it keeps it current like any other index. Its first creation reads `task_runs` once.
-  // A run whose metadata SQLite cannot read as JSON refuses the creation: the board opens without the index, says which
-  // rows to look for, and tries again on its next open. Once the index is there, SQLite refuses a write of such
-  // metadata to a run.
+  layOutEventIndex(db);
+  // The calling session's run (CALLER_RUN_ADDRESS) and its cards (SESSION_CURRENT_RUNS, SESSION_BOUND, SESSION_RUNS):
+  // without the session's index each lookup read every run's whole metadata, overflow pages included. `id` keeps a
+  // session's runs in the order the lookups ask for, and the address makes the index answer the caller's lookup without
+  // reading the run. The pending launches (PENDING_LAUNCHES) have an index of their own, which holds only them. No
+  // statement names either, and a release that does not know one keeps it current like any other index. An index's
+  // first creation reads `task_runs` once.
+  // A run whose metadata SQLite cannot read as JSON refuses the creation: the board opens without the indexes, says so
+  // once with the rows to look for, and tries again on its next open. Without them the caller's lookup reads every run
+  // and fails at the run that refused them once it reaches it, and a session's cards are found the earlier way, from the
+  // cards (sessionIndexed). Once an index is there, SQLite refuses a write of such metadata to a run.
   if (db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'task_runs'").get()) {
-    try { db.exec(`CREATE INDEX IF NOT EXISTS ${RUN_SESSION_INDEX} ON task_runs(${RUN_SESSION}, id, ${RUN_ADDRESS})`); }
-    catch (error) {
-      if (contention(error)) throw error;
-      const said = current().said;
-      if (!said.has(RUN_SESSION_INDEX)) {
-        said.add(RUN_SESSION_INDEX);
-        current().err(`board: warning: index ${RUN_SESSION_INDEX} on task_runs was not created (${error.message}); the board is open without it and each session's verb reads every run. The runs that refuse it: SELECT id FROM task_runs WHERE metadata IS NOT NULL AND NOT json_valid(metadata, 6)\n`);
-      }
+    const refused = [];
+    for (const [index, on] of RUN_INDEXES) {
+      try { db.exec(`CREATE INDEX IF NOT EXISTS ${index} ON task_runs${on}`); }
+      catch (error) { if (contention(error)) throw error; refused.push([index, error.message]); }
+    }
+    const said = current().said;
+    if (refused.length && !said.has(RUN_SESSION_INDEX)) {
+      said.add(RUN_SESSION_INDEX);
+      const names = refused.map(([index]) => index).join(' and ');
+      current().err(`board: warning: ${refused.length > 1 ? 'indexes' : 'index'} ${names} on task_runs ${refused.length > 1 ? 'were' : 'was'} not created (${refused[0][1]}); the board is open without ${refused.length > 1 ? 'them' : 'it'}: each session's verb reads every run for its caller, and finds its cards from the cards. The runs that refuse ${refused.length > 1 ? 'them' : 'it'}: SELECT id FROM task_runs WHERE metadata IS NOT NULL AND NOT json_valid(metadata, 6)\n`);
     }
   }
   const effectColumns = new Set(db.prepare('PRAGMA table_info(supercode_effects)').all().map((c) => c.name));

package/board/ztrack.mjs

@@ -29,10 +29,43 @@ const timedStep = (step, fn) => { const started = performance.now(); try { retur
 // the reads of `board_state` and `board_errors` that come before each of the engine's statements, and a statement's
 // compilation, which reads the schema.
 const SLOW_STATEMENT_MS = 50;
+// A statement that is a step also says what the whole process used while it ran (`used` on its step, named as a slow
+// line's `whole_process` is): processor time, page faults, and context switches given up and taken away. The statement
+// runs on the loop's thread and holds it, so over a statement these are nearly the statement's own. The store is read
+// with pread (no memory map: `PRAGMA mmap_size` is 0), so a page SQLite does not hold and the system has to read is a
+// switch given up, never a page fault and, on macOS, never an `fs_read`. Measured on macOS 26 (Darwin 25.4), a 741 MB
+// store of 181,000 pages evicted from the system's cache before each of 20 runs: the creator's lookup of a card with
+// 1,893 events read 1,116 pages in 80 to 88 ms with 7 to 11 ms of processor time, voluntary_switches 758 every time,
+// major_page_faults 0 and fs_read 0; the same lookup with the file cached took 2.6 ms with voluntary_switches 0. So:
+//   voluntary_switches in the hundreds with little processor time is what waiting for file pages reads as (`ms` over
+//   that count is the wait for each read); few of them with little processor time is a wait that is not page reads.
+// They are the whole process's, so a hold long enough for the loop's watcher to read the held stack (slow-log.mjs
+// watchEventLoop) has that thread's use in them too. Measured over a 2 s sleep on a watched loop, two runs: 24 to 26
+// ms of processor time, 149 to 234 minor page faults, 110 to 140 involuntary switches, no voluntary switch, and 85
+// major page faults in one run against 0 in the next; unwatched, 0.6 ms, 17, 1, 0 and 0. So major page faults under a
+// hundred in a long hold do not say the statement faulted, and the watcher adds no voluntary switch.
+// The counters are read once before every statement (process.resourceUsage, 0.3 µs) and once more only for a step:
+// a fast statement went from 491 to 917 ns (100,000 a round, five rounds, median).
+const statementCounters = () => { try { return process.resourceUsage(); } catch { return null; } };
+const statementUse = (before, after) => {
+  if (!before || !after) return null;
+  const ms = (micros) => Math.round(micros / 1000);
+  return { cpu_user_ms: ms(after.userCPUTime - before.userCPUTime), cpu_system_ms: ms(after.systemCPUTime - before.systemCPUTime),
+    major_page_faults: after.majorPageFault - before.majorPageFault, minor_page_faults: after.minorPageFault - before.minorPageFault, fs_read: after.fsRead - before.fsRead,
+    voluntary_switches: after.voluntaryContextSwitches - before.voluntaryContextSwitches, involuntary_switches: after.involuntaryContextSwitches - before.involuntaryContextSwitches };
+};
+// Called from `finally`: it answers nothing and never throws, so it can replace neither a statement's value nor its error.
+const noteStatement = (sql, started, before) => {
+  try {
+    const ms = performance.now() - started;
+    if (ms >= SLOW_STATEMENT_MS) noteStep(`board statement: ${String(sql).replace(/\s+/g, ' ').trim().slice(0, 120)}`, ms, statementUse(before, statementCounters()));
+  } catch { /* the step is not noted */ }
+};
 const timedStatement = (sql, fn) => {
+  const before = statementCounters();
   const started = performance.now();
   try { return fn(); }
-  finally { const ms = performance.now() - started; if (ms >= SLOW_STATEMENT_MS) noteStep(`board statement: ${String(sql).replace(/\s+/g, ' ').trim().slice(0, 120)}`, ms); }
+  finally { noteStatement(sql, started, before); }
 };
 import { fire, workflowFor } from './workflow.mjs';
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-orchestrator",
-  "version": "0.5.159",
+  "version": "0.5.161",
   "type": "module",
   "description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
   "exports": {
@@ -24,7 +24,7 @@
     "check-types": "node scripts/generate-types.mjs --check"
   },
   "dependencies": {
-    "@volter/supercode-harness-sdk": "^0.3.97",
+    "@volter/supercode-harness-sdk": "^0.3.98",
     "@volter/teams": "0.2.65",
     "@volter/ztrack": "^2.4.23",
     "yaml": "^2.9.0"