@volter/supercode-orchestrator 0.5.159 → 0.5.160

package/board/cli.mjs

@@ -26,7 +26,7 @@ import * as slowLog from '@volter/supercode-harness-sdk/slow-log';
 import { basename, dirname, join, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { homedir, tmpdir, userInfo } from 'node:os';
-import { actorOf, CALLER_RUN_ADDRESS, heldConfigWatched, heldDocumentsChanged, heldDocumentsWatched, holdBoards, retireHeldBoards, releaseHeldBoards, publishMarked, view, backingPaths, layOut, backingOf, boardExists, boardPath, briefPath, DEFAULT_BOARD, event, json, listBoards, logsDir, now, openBoard, setActor, setPrincipal, principalOf, isCaller, tx } from './store.mjs';
+import { actorOf, CALLER_RUN_ADDRESS, SESSION_CURRENT_RUNS, SESSION_RUNS, sessionIndexed, heldConfigWatched, heldDocumentsChanged, heldDocumentsWatched, holdBoards, retireHeldBoards, releaseHeldBoards, publishMarked, view, backingPaths, layOut, backingOf, boardExists, boardPath, briefPath, DEFAULT_BOARD, event, json, listBoards, logsDir, now, openBoard, setActor, setPrincipal, principalOf, isCaller, tx } from './store.mjs';
 import { isLocalWorkspace } from './workspace.mjs';
 import { openZtrackBoard } from './ztrack.mjs';
 import * as engine from './engine.mjs';
@@ -1406,18 +1406,38 @@ function workerRun(db, id, sessionId) {
  * worked card is held to nothing here. The newest run alone named one card, and a session adopted into a second card
  * was refused every write on its first (volter-org t_9a9c2a9c: no comment, heartbeat or closed subtask there, read
  * by the board as no progress).
+ *
+ * Asked on every verb a session runs, so it reads the session's own runs and no others where it can (store.mjs). The
+ * newest run naming the session is the newest whose text holds its id anywhere, an observation of another run's
+ * included, which no index answers: it is looked for only when its answer can add a card (one of the session's own
+ * runs is on a card it does not already work and is still the last worker of), and then from the newest run back to
+ * the session's own newest, which its text names too. A session whose runs are on more cards than OWN_CARDS_ASKED is
+ * not asked card by card first: the newest run is looked for and its card asked alone, which answers the same.
+ * A board without the session's index (store.mjs sessionIndexed) is asked from its cards and by text alone, which
+ * reads no run it cannot read.
  */
+const OWN_CARDS_ASKED = 8;
 function ownCards(db, sessionId) {
   const worked = new Set(), held = new Set();
   const like = `%"session_id":"${sessionId}"%`;
-  for (const row of db.prepare('SELECT t.id, r.metadata FROM tasks t JOIN task_runs r ON r.id = t.current_run_id WHERE r.metadata LIKE ?').all(like)) {
+  const byIndex = sessionIndexed(db);
+  for (const row of db.prepare(byIndex ? SESSION_CURRENT_RUNS : 'SELECT t.id, r.metadata FROM tasks t JOIN task_runs r ON r.id = t.current_run_id WHERE r.metadata LIKE ?').all(...(byIndex ? [sessionId, like] : [like]))) {
     const session = json(row.metadata, {})?.supercode;
     if (session?.session_id !== sessionId) continue;
     held.add(row.id);
     if ((session.slot ?? 'worker') === 'worker') worked.add(row.id);
   }
-  const newest = db.prepare('SELECT task_id FROM task_runs WHERE metadata LIKE ? ORDER BY id DESC LIMIT 1').get(like);
-  if (newest && engine.lastSession(db, newest.task_id)?.session_id === sessionId) { worked.add(newest.task_id); held.add(newest.task_id); }
+  const anywhere = 'SELECT task_id FROM task_runs WHERE metadata LIKE ? ORDER BY id DESC LIMIT 1';
+  const asked = new Map();
+  const lastWorker = (id) => { if (!asked.has(id)) asked.set(id, engine.lastSession(db, id)?.session_id === sessionId); return asked.get(id); };
+  let newest;
+  if (byIndex) {
+    const runs = db.prepare(SESSION_RUNS).all(sessionId);
+    const between = [...new Set(runs.map((run) => run.task_id))].filter((id) => !worked.has(id));
+    if (!between.length || (between.length <= OWN_CARDS_ASKED && !between.some(lastWorker))) return { worked, held };
+    newest = db.prepare('SELECT task_id FROM task_runs WHERE id >= ? AND metadata LIKE ? ORDER BY id DESC LIMIT 1').get(runs.at(-1).id, like) ?? db.prepare(anywhere).get(like);
+  } else newest = db.prepare(anywhere).get(like);
+  if (newest && lastWorker(newest.task_id)) { worked.add(newest.task_id); held.add(newest.task_id); }
   return { worked, held };
 }
 

package/board/dispatch.mjs

@@ -40,7 +40,7 @@ import { homedir, hostname, tmpdir, totalmem } from 'node:os';
 import { basename, dirname, join } from 'node:path';
 import { createHash, randomUUID } from 'node:crypto';
 import { fileURLToPath } from 'node:url';
-import { view, boardCopyOf, boardPath, briefPath, claimer, claimHolds, copyRefusal, event, json, listBoards, logsDir, newClaim, now, openBoard, setActor, storageFault, tx } from './store.mjs';
+import { view, boardCopyOf, boardPath, briefPath, claimer, claimHolds, copyRefusal, event, json, listBoards, logsDir, newClaim, now, openBoard, pendingLaunches, sessionBound, setActor, storageFault, tx } from './store.mjs';
 import { doors, fire, loadWorkflow, resumeOf, runSpec, runWorkingSeconds, requestSessionClose, retireSupersededRuns, refreshCardInputs, refreshRelationAnswers, transitionsFor, scopeOf, statusOf, projectStatus, storedStatus, templateValue, workflowFor } from './workflow.mjs';
 import { checkWorkspace, assertWorkspaceOwner, notePresence, resolveWorkspace } from './workspace.mjs';
 import { capacity, harnessDefaults, holdings } from './capacity.mjs';
@@ -1822,11 +1822,8 @@ function bindLaunch(board, row, session, pane, sid) {
 /** A fast worker can reach its first board door before open's reply. Bind only the daemon's keyed pane. */
 export async function reconcileCallerLaunch(board, sid) {
   fleetRoot = board.root;
-  if (view(board, (db) => db.prepare(`SELECT 1 FROM task_runs r JOIN tasks t ON t.current_run_id=r.id
-    WHERE json_extract(r.metadata, '$.supercode.session_id')=? AND COALESCE(json_extract(r.metadata, '$.supercode.launch_pending'),0)=0`).get(sid))) return;
-  const rows = view(board, (db) => db.prepare(`SELECT r.id, r.task_id, r.metadata FROM task_runs r
-    JOIN tasks t ON t.current_run_id=r.id WHERE r.ended_at IS NULL
-    AND json_extract(r.metadata, '$.supercode.launch_pending') = 1`).all());
+  if (view(board, (db) => db.prepare(sessionBound(db)).get(sid))) return;
+  const rows = view(board, (db) => db.prepare(pendingLaunches(db)).all());
   const machine = localMachine();
   const harness = boardCaller()?.harness ?? (sid === process.env.CODEX_THREAD_ID ? 'codex' : 'claude-code');
   const candidates = rows.filter((row) => {

package/board/store.mjs

@@ -148,8 +148,54 @@ export function listBoards(root) {
 // SQLite uses an expression index only for a statement that writes the expression the same way.
 const RUN_SESSION = "json_extract(metadata, '$.supercode.session_id')";
 const RUN_ADDRESS = "json_extract(metadata, '$.supercode.address')";
+const RUN_LAUNCH_PENDING = "json_extract(metadata, '$.supercode.launch_pending')";
 export const CALLER_RUN_ADDRESS = `SELECT ${RUN_ADDRESS} AS address FROM task_runs WHERE ${RUN_SESSION} = ? ORDER BY id DESC LIMIT 1`;
 const RUN_SESSION_INDEX = 'idx_runs_supercode_session';
+const RUN_LAUNCH_PENDING_INDEX = 'idx_runs_supercode_launch_pending';
+// The indexes on a run's metadata. The second holds only the runs whose launch is pending (7 of 1,387 on the store
+// these were measured for), in the runs' own order.
+const RUN_INDEXES = [
+  [RUN_SESSION_INDEX, `(${RUN_SESSION}, id, ${RUN_ADDRESS})`],
+  [RUN_LAUNCH_PENDING_INDEX, `(id) WHERE ${RUN_LAUNCH_PENDING} = 1`],
+];
+
+// What every verb a session runs asks about its caller (cli.mjs ownCards, dispatch.mjs reconcileCallerLaunch). Each
+// starts from the runs an index names (the session's, or the pending launches) and reaches the card through the run's
+// own `task_id`, the card's key: a card's current run is one of its own runs, and the join says both. Starting from the
+// cards instead read every card's row to find the few that have a current run (810 of the 843 pages each of these
+// statements read, whatever the session, on a store of 2,636 cards with 5 of them running). `r` is read from the session's
+// index alone and the run's metadata only as `m`, after the card is known to name the run as current, so a session's
+// earlier runs cost a page each and none of their metadata. The rows come in the cards' own order, as they did.
+//
+// Each of these is asked only of a board that has the index it starts from (sessionIndexed, sessionBound,
+// pendingLaunches). Without the index it would read every run, and a board is without it exactly when one of its runs
+// holds metadata SQLite cannot read as JSON (layOut), which `json_extract` refuses: the statement would fail on a run
+// it had no reason to read. Such a board is asked the earlier statements instead, which start from the cards and read
+// only their current runs. The board is asked which it has at each call, not told at its open: the owner holds a board
+// open for hours, and the index may be made meanwhile by another process's open.
+const ofRun = (expression, run) => expression.replace('(metadata,', `(${run}.metadata,`);
+const CURRENT_RUN_OF = 'JOIN tasks t ON t.id = r.task_id AND t.current_run_id = r.id';
+/** The cards whose current run names the session (`?`), with that run's metadata; `LIKE ?` as the run's text says it. */
+export const SESSION_CURRENT_RUNS = `SELECT t.id, m.metadata FROM task_runs r ${CURRENT_RUN_OF} CROSS JOIN task_runs m ON m.id = r.id
+  WHERE ${ofRun(RUN_SESSION, 'r')} = ? AND m.metadata LIKE ? ORDER BY t.rowid`;
+/** Every run that names the session (`?`) as its own, oldest first, with its card. */
+export const SESSION_RUNS = `SELECT id, task_id FROM task_runs WHERE ${RUN_SESSION} = ? ORDER BY id`;
+const SESSION_BOUND = `SELECT 1 FROM task_runs r ${CURRENT_RUN_OF} CROSS JOIN task_runs m ON m.id = r.id
+  WHERE ${ofRun(RUN_SESSION, 'r')} = ? AND COALESCE(${ofRun(RUN_LAUNCH_PENDING, 'm')},0)=0`;
+// The runs come first (CROSS JOIN): the planner cannot know how few the index holds, and reads every card otherwise.
+const PENDING_LAUNCHES = `SELECT r.id, r.task_id, r.metadata FROM task_runs r CROSS ${CURRENT_RUN_OF}
+  WHERE r.ended_at IS NULL AND ${ofRun(RUN_LAUNCH_PENDING, 'r')} = 1 ORDER BY t.rowid`;
+// the earlier statements, from the cards: they read every card and the current runs alone
+const CURRENT_RUN = 'FROM task_runs r JOIN tasks t ON t.current_run_id=r.id';
+const SESSION_BOUND_BY_CARDS = `SELECT 1 ${CURRENT_RUN} WHERE ${ofRun(RUN_SESSION, 'r')}=? AND COALESCE(${ofRun(RUN_LAUNCH_PENDING, 'r')},0)=0`;
+const PENDING_LAUNCHES_BY_CARDS = `SELECT r.id, r.task_id, r.metadata ${CURRENT_RUN} WHERE r.ended_at IS NULL AND ${ofRun(RUN_LAUNCH_PENDING, 'r')} = 1`;
+const indexed = (db, index) => Boolean(db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?").get(index));
+/** Whether the board has the index SESSION_CURRENT_RUNS and SESSION_RUNS start from. */
+export const sessionIndexed = (db) => indexed(db, RUN_SESSION_INDEX);
+/** The statement that says whether a card's current run names the session (`?`) with its launch already bound. */
+export const sessionBound = (db) => (sessionIndexed(db) ? SESSION_BOUND : SESSION_BOUND_BY_CARDS);
+/** The statement that lists the open current runs whose launch is still pending. */
+export const pendingLaunches = (db) => (indexed(db, RUN_LAUNCH_PENDING_INDEX) ? PENDING_LAUNCHES : PENDING_LAUNCHES_BY_CARDS);
 
 /** Hermes's tables on a fresh board, supercode's own table and columns on every one. */
 export function layOut(db, { fresh }) {
@@ -162,22 +208,27 @@ export function layOut(db, { fresh }) {
   // each read walked every card's row. Laid out here, after the column it names exists on a store of any age and before
   // any statement is prepared; no statement names it, so a store without it is only slower.
   db.exec('CREATE INDEX IF NOT EXISTS idx_supercode_cards_subtask_of ON supercode_cards(subtask_of)');
-  // The calling session's run (CALLER_RUN_ADDRESS): without this index each lookup read every run's whole metadata,
-  // overflow pages included. `id` keeps a session's runs in the order the lookup asks for, and the address makes the
-  // index answer the lookup without reading the run. No statement names it, so a store without it is only slower, and a
-  // release that does not know it keeps it current like any other index. Its first creation reads `task_runs` once.
-  // A run whose metadata SQLite cannot read as JSON refuses the creation: the board opens without the index, says which
-  // rows to look for, and tries again on its next open. Once the index is there, SQLite refuses a write of such
-  // metadata to a run.
+  // The calling session's run (CALLER_RUN_ADDRESS) and its cards (SESSION_CURRENT_RUNS, SESSION_BOUND, SESSION_RUNS):
+  // without the session's index each lookup read every run's whole metadata, overflow pages included. `id` keeps a
+  // session's runs in the order the lookups ask for, and the address makes the index answer the caller's lookup without
+  // reading the run. The pending launches (PENDING_LAUNCHES) have an index of their own, which holds only them. No
+  // statement names either, and a release that does not know one keeps it current like any other index. An index's
+  // first creation reads `task_runs` once.
+  // A run whose metadata SQLite cannot read as JSON refuses the creation: the board opens without the indexes, says so
+  // once with the rows to look for, and tries again on its next open. Without them the caller's lookup reads every run
+  // and fails at the run that refused them once it reaches it, and a session's cards are found the earlier way, from the
+  // cards (sessionIndexed). Once an index is there, SQLite refuses a write of such metadata to a run.
   if (db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'task_runs'").get()) {
-    try { db.exec(`CREATE INDEX IF NOT EXISTS ${RUN_SESSION_INDEX} ON task_runs(${RUN_SESSION}, id, ${RUN_ADDRESS})`); }
-    catch (error) {
-      if (contention(error)) throw error;
-      const said = current().said;
-      if (!said.has(RUN_SESSION_INDEX)) {
-        said.add(RUN_SESSION_INDEX);
-        current().err(`board: warning: index ${RUN_SESSION_INDEX} on task_runs was not created (${error.message}); the board is open without it and each session's verb reads every run. The runs that refuse it: SELECT id FROM task_runs WHERE metadata IS NOT NULL AND NOT json_valid(metadata, 6)\n`);
-      }
+    const refused = [];
+    for (const [index, on] of RUN_INDEXES) {
+      try { db.exec(`CREATE INDEX IF NOT EXISTS ${index} ON task_runs${on}`); }
+      catch (error) { if (contention(error)) throw error; refused.push([index, error.message]); }
+    }
+    const said = current().said;
+    if (refused.length && !said.has(RUN_SESSION_INDEX)) {
+      said.add(RUN_SESSION_INDEX);
+      const names = refused.map(([index]) => index).join(' and ');
+      current().err(`board: warning: ${refused.length > 1 ? 'indexes' : 'index'} ${names} on task_runs ${refused.length > 1 ? 'were' : 'was'} not created (${refused[0][1]}); the board is open without ${refused.length > 1 ? 'them' : 'it'}: each session's verb reads every run for its caller, and finds its cards from the cards. The runs that refuse ${refused.length > 1 ? 'them' : 'it'}: SELECT id FROM task_runs WHERE metadata IS NOT NULL AND NOT json_valid(metadata, 6)\n`);
     }
   }
   const effectColumns = new Set(db.prepare('PRAGMA table_info(supercode_effects)').all().map((c) => c.name));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-orchestrator",
-  "version": "0.5.159",
+  "version": "0.5.160",
   "type": "module",
   "description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
   "exports": {