@volter/supercode-orchestrator 0.5.160 → 0.5.162

package/board/record-view.mjs

@@ -1,31 +1,17 @@
 // Resident local consumer of the daemon's committed record rows. Disconnects
 // preserve the last applied cursor, but held data is stale until caught up.
 export class RecordView {
-  constructor(open) {
+  constructor(open, { progressMs = 10_000 } = {}) {
     this.open = open; this.records = {}; this.positions = new Map();
     this.cursor = null; this.stream = null; this.waiters = new Set();
+    this.pending = [];
     this.ready = false; this.heardAt = 0;
+    this.progressMs = progressMs; this.progressTimer = null;
   }
   async read(timeoutMs) {
     const deadline = Date.now() + timeoutMs;
     if (this.stream && Date.now() - this.heardAt > 45_000) this.stream.close('its record stream went quiet');
-    if (!this.stream && !this.opening) {
-      this.opening = this.open({ after: this.cursor }).then((stream) => {
-        this.stream = stream;
-        this.ready = false;
-        this.heardAt = Date.now();
-        stream.on('data', (frame) => {
-          try { this.apply(frame); }
-          catch (error) { this.fail(error); stream.close(error.message); }
-        });
-        stream.on('close', (reason) => {
-          if (this.stream !== stream) return;
-          this.stream = null; this.ready = false;
-          this.fail(new Error(reason ?? 'the local record stream closed'));
-        });
-      }).finally(() => { this.opening = null; });
-    }
-    if (this.opening) await this.opening;
+    if (!this.stream) await this.connect();
     if (this.ready) return this.records;
     if (!this.stream) throw this.failure ?? new Error('the local record stream ended before its snapshot');
     return new Promise((resolve, reject) => {
@@ -37,6 +23,44 @@ export class RecordView {
       this.waiters.add(waiter);
     });
   }
+  connect() {
+    if (this.stream) return Promise.resolve();
+    return this.opening ??= Promise.resolve().then(() => this.open({ after: this.cursor })).then((stream) => {
+      this.stream = stream; this.ready = false; this.heardAt = Date.now();
+      stream.on('data', (frame) => {
+        if (this.stream !== stream) return;
+        try { this.apply(frame); }
+        catch (error) { this.fail(error); stream.close(error.message); }
+      });
+      stream.on('receive-progress', ({ bytes, type }) => {
+        if (this.stream !== stream || bytes <= 0 || (type !== 'snapshot' && type !== 'changes')) return;
+        this.heardAt = Date.now(); this.watchProgress();
+      });
+      stream.on('close', (reason) => {
+        if (this.stream !== stream) return;
+        clearTimeout(this.progressTimer); this.progressTimer = null;
+        this.stream = null; this.ready = false; this.pending = [];
+        this.fail(new Error(reason ?? 'the local record stream closed'));
+      });
+      this.watchProgress();
+    }).finally(() => { this.opening = null; });
+  }
+  // Beats prove the door is alive, not that its snapshot or unfinished page is advancing. Record frames and bytes
+  // arriving inside them renew this deadline. Expiry reopens at the applied cursor, never its held prefix.
+  watchProgress() {
+    clearTimeout(this.progressTimer); this.progressTimer = null;
+    if (this.ready || !this.stream) return;
+    const stream = this.stream;
+    this.progressTimer = setTimeout(() => {
+      if (this.stream !== stream) return;
+      this.progressTimer = null; this.stream = null; this.ready = false; this.pending = [];
+      const error = new Error('the local record stream stopped making snapshot or cursor catch-up progress');
+      this.fail(error);
+      try { stream.close(error.message); } catch { /* the failed stream is already detached */ }
+      this.connect().catch((error) => this.fail(error));
+    }, this.progressMs);
+    this.progressTimer.unref?.();
+  }
   fail(error) {
     this.failure = error;
     for (const waiter of this.waiters) { clearTimeout(waiter.timer); waiter.reject(error); }
@@ -46,13 +70,21 @@ export class RecordView {
   apply(frame) {
     if (frame.type === 'beat') { this.heardAt = Date.now(); return; }
     if (frame.type === 'snapshot') {
-      this.self = frame.machine; this.records = frame.records; this.positions.clear();
+      this.self = frame.machine; this.records = frame.records; this.positions.clear(); this.pending = [];
       for (const [machine, held] of Object.entries(this.records)) this.positions.set(machine, new Map((held.record?.sessions ?? []).map((row, at) => [row.address, at])));
       this.cursor = frame.cursor; this.ready = true;
     } else if (frame.type === 'changes') {
       if (!this.cursor || frame.cursor?.id !== this.cursor.id) throw new Error('record row changes name a different database; a new snapshot is required');
+      // Row-sized transport frames belong to one logical page. Neither the applied state nor its resumable cursor
+      // advances until that page's final frame arrives; a disconnect discards the unfinished page.
+      if (frame.partial) {
+        this.ready = false; this.pending.push(...(frame.changes ?? [])); this.heardAt = Date.now();
+        if (frame.changes?.length || !this.progressTimer) this.watchProgress();
+        return;
+      }
+      const changes = this.pending.concat(frame.changes ?? []); this.pending = [];
       const moved = new Set();
-      for (const change of frame.changes ?? []) {
+      for (const change of changes) {
         if (change.sequence <= this.cursor.sequence || !this.own(change.key)) continue;
         const { machine, section, key, position } = change.key;
         if (change.topic === 'record-head') {
@@ -85,6 +117,7 @@ export class RecordView {
       this.cursor = frame.cursor; this.ready = !frame.more;
     } else throw new Error(`unrecognized record-view frame ${frame.type}`);
     this.heardAt = Date.now();
+    this.watchProgress();
     if (this.ready) {
       for (const waiter of this.waiters) { clearTimeout(waiter.timer); waiter.resolve(this.records); }
       this.waiters.clear();

package/board/store.mjs

@@ -14,7 +14,7 @@ import { fenced } from './fence.mjs';
 import { noteStep } from '@volter/supercode-harness-sdk/slow-log';
 import { redactValue } from '@volter/teams/redact';
 import { randomBytes } from 'node:crypto';
-import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, writeFileSync } from 'node:fs';
+import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, writeFileSync, writeSync } from 'node:fs';
 import { hostname } from 'node:os';
 import { dirname, join } from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
@@ -197,6 +197,31 @@ export const sessionBound = (db) => (sessionIndexed(db) ? SESSION_BOUND : SESSIO
 /** The statement that lists the open current runs whose launch is still pending. */
 export const pendingLaunches = (db) => (indexed(db, RUN_LAUNCH_PENDING_INDEX) ? PENDING_LAUNCHES : PENDING_LAUNCHES_BY_CARDS);
 
+const EVENT_KIND_INDEX = 'idx_events_task_kind';
+// creatorOf's first `created` event and the latest review events are point lookups in this order. The latest event
+// of each kind can group from the index alone; IN-kind lookups still sort, but read only the selected kinds. Keep
+// idx_events_task: statements asking for events since a time need its (task_id, created_at) order.
+// Installed here on every backing's open, including an existing store, before any engine statement is prepared.
+// Its first creation scans task_events once, sorts the keys and writes an entry per event (no payload in the index).
+// SQLite can reuse free pages; this does not vacuum the store. It is synchronous and can hold the owner's loop for
+// seconds, so say so BEFORE starting and report its time afterward. Write the owner's stderr synchronously even
+// inside a caller's call: that call's err is buffered until its answer, after the migration has held the loop.
+function layOutEventIndex(db) {
+  if (!db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'task_events'").get() || indexed(db, EVENT_KIND_INDEX)) return;
+  const report = (text) => { writeSync(2, text); if (current().kind === 'caller') current().err(text); };
+  report(`board: creating index ${EVENT_KIND_INDEX} on task_events: one-time table scan, key sort and index write; this synchronous migration holds the owner event loop until it finishes and can take seconds\n`);
+  const began = performance.now();
+  let made = false;
+  try {
+    db.exec(`CREATE INDEX IF NOT EXISTS ${EVENT_KIND_INDEX} ON task_events(task_id, kind, id)`);
+    made = true;
+  } finally {
+    const ms = performance.now() - began;
+    noteStep(`board open index ${EVENT_KIND_INDEX}`, ms);
+    report(`board: index ${EVENT_KIND_INDEX} ${made ? 'created' : 'creation failed'} after ${Math.round(ms)} ms\n`);
+  }
+}
+
 /** Hermes's tables on a fresh board, supercode's own table and columns on every one. */
 export function layOut(db, { fresh }) {
   if (fresh) db.exec(SCHEMA);
@@ -208,6 +233,7 @@ export function layOut(db, { fresh }) {
   // each read walked every card's row. Laid out here, after the column it names exists on a store of any age and before
   // any statement is prepared; no statement names it, so a store without it is only slower.
   db.exec('CREATE INDEX IF NOT EXISTS idx_supercode_cards_subtask_of ON supercode_cards(subtask_of)');
+  layOutEventIndex(db);
   // The calling session's run (CALLER_RUN_ADDRESS) and its cards (SESSION_CURRENT_RUNS, SESSION_BOUND, SESSION_RUNS):
   // without the session's index each lookup read every run's whole metadata, overflow pages included. `id` keeps a
   // session's runs in the order the lookups ask for, and the address makes the index answer the caller's lookup without

package/board/ztrack.mjs

@@ -29,10 +29,43 @@ const timedStep = (step, fn) => { const started = performance.now(); try { retur
 // the reads of `board_state` and `board_errors` that come before each of the engine's statements, and a statement's
 // compilation, which reads the schema.
 const SLOW_STATEMENT_MS = 50;
+// A statement that is a step also says what the whole process used while it ran (`used` on its step, named as a slow
+// line's `whole_process` is): processor time, page faults, and context switches given up and taken away. The statement
+// runs on the loop's thread and holds it, so over a statement these are nearly the statement's own. The store is read
+// with pread (no memory map: `PRAGMA mmap_size` is 0), so a page SQLite does not hold and the system has to read is a
+// switch given up, never a page fault and, on macOS, never an `fs_read`. Measured on macOS 26 (Darwin 25.4), a 741 MB
+// store of 181,000 pages evicted from the system's cache before each of 20 runs: the creator's lookup of a card with
+// 1,893 events read 1,116 pages in 80 to 88 ms with 7 to 11 ms of processor time, voluntary_switches 758 every time,
+// major_page_faults 0 and fs_read 0; the same lookup with the file cached took 2.6 ms with voluntary_switches 0. So:
+//   voluntary_switches in the hundreds with little processor time is what waiting for file pages reads as (`ms` over
+//   that count is the wait for each read); few of them with little processor time is a wait that is not page reads.
+// They are the whole process's, so a hold long enough for the loop's watcher to read the held stack (slow-log.mjs
+// watchEventLoop) has that thread's use in them too. Measured over a 2 s sleep on a watched loop, two runs: 24 to 26
+// ms of processor time, 149 to 234 minor page faults, 110 to 140 involuntary switches, no voluntary switch, and 85
+// major page faults in one run against 0 in the next; unwatched, 0.6 ms, 17, 1, 0 and 0. So major page faults under a
+// hundred in a long hold do not say the statement faulted, and the watcher adds no voluntary switch.
+// The counters are read once before every statement (process.resourceUsage, 0.3 µs) and once more only for a step:
+// a fast statement went from 491 to 917 ns (100,000 a round, five rounds, median).
+const statementCounters = () => { try { return process.resourceUsage(); } catch { return null; } };
+const statementUse = (before, after) => {
+  if (!before || !after) return null;
+  const ms = (micros) => Math.round(micros / 1000);
+  return { cpu_user_ms: ms(after.userCPUTime - before.userCPUTime), cpu_system_ms: ms(after.systemCPUTime - before.systemCPUTime),
+    major_page_faults: after.majorPageFault - before.majorPageFault, minor_page_faults: after.minorPageFault - before.minorPageFault, fs_read: after.fsRead - before.fsRead,
+    voluntary_switches: after.voluntaryContextSwitches - before.voluntaryContextSwitches, involuntary_switches: after.involuntaryContextSwitches - before.involuntaryContextSwitches };
+};
+// Called from `finally`: it answers nothing and never throws, so it can replace neither a statement's value nor its error.
+const noteStatement = (sql, started, before) => {
+  try {
+    const ms = performance.now() - started;
+    if (ms >= SLOW_STATEMENT_MS) noteStep(`board statement: ${String(sql).replace(/\s+/g, ' ').trim().slice(0, 120)}`, ms, statementUse(before, statementCounters()));
+  } catch { /* the step is not noted */ }
+};
 const timedStatement = (sql, fn) => {
+  const before = statementCounters();
   const started = performance.now();
   try { return fn(); }
-  finally { const ms = performance.now() - started; if (ms >= SLOW_STATEMENT_MS) noteStep(`board statement: ${String(sql).replace(/\s+/g, ' ').trim().slice(0, 120)}`, ms); }
+  finally { noteStatement(sql, started, before); }
 };
 import { fire, workflowFor } from './workflow.mjs';
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-orchestrator",
-  "version": "0.5.160",
+  "version": "0.5.162",
   "type": "module",
   "description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
   "exports": {
@@ -24,8 +24,8 @@
     "check-types": "node scripts/generate-types.mjs --check"
   },
   "dependencies": {
-    "@volter/supercode-harness-sdk": "^0.3.97",
-    "@volter/teams": "0.2.65",
+    "@volter/supercode-harness-sdk": "^0.3.98",
+    "@volter/teams": "0.2.66",
     "@volter/ztrack": "^2.4.23",
     "yaml": "^2.9.0"
   },