@volter/supercode-orchestrator 0.5.10 → 0.5.12

package/bin/orchestrator.mjs

@@ -1,7 +1,10 @@
 #!/usr/bin/env node
 // Board commands do not load daemon transports, platform adapters or the MCP server.
 if (process.argv[2] === 'workflow') {
-  const [{runWorkflow},{closeWorld}]=await Promise.all([import('../board/cli.mjs'),import('../orchestration.mjs')]);
+  const [{runWorkflow},{closeWorld},{timeCommand}]=await Promise.all([import('../board/cli.mjs'),import('../orchestration.mjs'),import('@volter/supercode-harness-sdk/slow-log')]);
+  // A board command is timed into the slow log; `serve` runs until stopped and is not one.
+  const verb=process.argv.slice(3).find((arg,index,all)=>!arg.startsWith('-')&&!(index>0&&/^--(root|board|cwd|context|at)$/.test(all[index-1])));
+  if(verb!=='serve')timeCommand(`workflow ${verb??''}`.trim(),process.argv.slice(3));
   let code=0;
   try {code=await runWorkflow(process.argv.slice(3));}
   catch(error) {process.stderr.write(`supercode workflow: ${error.message??error}\n`);code=1;}

package/board/event-stream.mjs

@@ -1,6 +1,6 @@
 // Public native workflow observation. Filesystem custody authorizes this door;
 // Teams separately authorizes the Org event and its installed subscribers.
-import { watch, existsSync, readFileSync, realpathSync } from 'node:fs';
+import { watch, existsSync, readFileSync, realpathSync, statSync } from 'node:fs';
 import { dirname, basename, join, resolve } from 'node:path';
 import { createHash } from 'node:crypto';
 import { listBoards, openBoard, backingPaths } from './store.mjs';
@@ -9,28 +9,39 @@ import { recordCardPublications } from './published.mjs';
 const identity = root => createHash('sha256').update(realpathSync(root)).digest('hex');
 const cursorOf = (home, boards) => Buffer.from(JSON.stringify({version:1,home,boards})).toString('base64url');
 const resync = reason => Object.assign(new Error(reason), {code:'resync_required'});
+const RESETTLE_MS = 30_000;
 
 /** Snapshot once, then replay only committed publications on backing notifications. */
 export async function* followBoardPublications({root,after,signal}) {
   const home=identity(root), positions={}, watchers=new Map();
   let dirty=true, wake=null, failure=null, inventory=null;
-  const notify=()=>{dirty=true;wake?.();};
+  // The stores' files as this follower last left them. Its own pass writes (the publication transaction) and closes
+  // the board, which rewrites the WAL and raises a watch event of its own; an event that finds every store file as the
+  // pass left it is that echo, not a change, and does not start another pass. Without this the follower woke itself
+  // every half second and burned a core with nothing to publish (t_0328d520).
+  let stores=[], settled=null;
+  const stamp=()=>stores.map(path=>{try{const s=statSync(path);return `${s.size}:${s.mtimeMs}`;}catch{return '-';}}).join('|');
+  const notify=(name=null)=>{
+    if(name!==null&&settled!==null&&stores.some(path=>basename(path)===name)&&stamp()===settled)return;
+    dirty=true;wake?.();
+  };
   const directory=(path,accept)=>{
     if(!existsSync(path))return;
     if(watchers.has(path)){watchers.get(path).accept.push(accept);return;}
     const entry={accept:[accept],handle:null};
-    entry.handle=watch(path,(_kind,name)=>{if(name==null||entry.accept.some(fn=>fn(String(name))))notify();});
+    entry.handle=watch(path,(_kind,name)=>{if(name==null)notify();else if(entry.accept.some(fn=>fn(String(name))))notify(String(name));});
     entry.handle.on('error',error=>{failure=error;notify();});watchers.set(path,entry);
   };
   const file=path=>directory(dirname(path),name=>name===basename(path));
   const register=slugs=>{
     for(const entry of watchers.values())entry.accept=[];
+    stores=[join(root,'kanban.db'),join(root,'kanban.db-wal')];
     directory(root,name=>['kanban.db','kanban.db-wal','kanban'].includes(name));
     directory(join(root,'kanban'),name=>['boards','ztrack.json','board'].includes(name));
     directory(join(root,'kanban','boards'),()=>true);
     for(const slug of slugs){
       const paths=backingPaths(root,slug);
-      file(paths.sqlite);file(`${paths.sqlite}-wal`);file(paths.ztrack);
+      file(paths.sqlite);file(`${paths.sqlite}-wal`);file(paths.ztrack);stores.push(paths.sqlite,`${paths.sqlite}-wal`);
       directory(paths.files,name=>name.endsWith('.json'));
       try {const cfg=JSON.parse(readFileSync(paths.ztrack,'utf8'));const document=resolve(dirname(paths.ztrack),cfg.document);file(document);file(`${document}.archive.md`);}catch{/* other backing */}
     }
@@ -83,7 +94,12 @@ export async function* followBoardPublications({root,after,signal}) {
       }
       if(initial){initial=false;yield {type:'snapshot_end',cursor:cursorOf(home,positions)};dirty=true;}
       else if(checkpointPending){checkpointPending=false;yield {type:'checkpoint',cursor:cursorOf(home,positions)};}
-      if(!dirty&&!signal?.aborted)await new Promise(done=>{wake=done;if(dirty||signal?.aborted)done();});
+      settled=stamp();
+      // A write that lands between the pass and the stamp above has its event taken for this pass's echo: a pass
+      // every RESETTLE_MS anyway bounds how long that write can wait.
+      let timer=null;
+      if(!dirty&&!signal?.aborted)await new Promise(done=>{wake=done;timer=setTimeout(()=>{dirty=true;done();},RESETTLE_MS);timer.unref?.();if(dirty||signal?.aborted)done();});
+      clearTimeout(timer);
       wake=null;
     }
   }finally{signal?.removeEventListener('abort',aborted);for(const {handle} of watchers.values())handle.close();}

package/board/git-document.mjs

@@ -33,12 +33,24 @@ export class GitDocument {
 
   get usable() { return this.root !== null; }
   rel(path) { return relative(this.root, path); }
-  branch() { return git(this.root, ['rev-parse', '--abbrev-ref', 'HEAD']).trim(); }
+  /** The checkout's branch, or null on no branch (a detached HEAD has nowhere to push and nothing to pull). */
+  branch() {
+    const name = git(this.root, ['rev-parse', '--abbrev-ref', 'HEAD']).trim();
+    return name && name !== 'HEAD' ? name : null;
+  }
+
+  /** Said once per process: a checkout on no branch keeps its commits here (every write tried to push `HEAD:HEAD`
+   *  three times, with a fetch and a merge between, and failed: seconds of network on each write, t_23e47b73). */
+  detached() {
+    if (!this.warnedDetached) process.stderr.write(`board: the board documents' checkout ${this.root} is on no branch; writes are committed there and not pushed until it is on one\n`);
+    this.warnedDetached = true;
+  }
 
   // Before a write: take what was pushed, so a merged pull request's edit is the document the write starts from.
   pull() {
     if (!this.usable) return;
     const branch = this.branch();
+    if (!branch) return this.detached();
     if (quiet(this.root, ['fetch', '--quiet', 'origin', branch]) === null) return; // offline: the next write tries again
     const behind = Number(git(this.root, ['rev-list', '--count', `HEAD..origin/${branch}`]).trim());
     if (behind === 0) return;
@@ -59,6 +71,7 @@ export class GitDocument {
     const who = doer || 'board';
     indexed(this.root, ['-c', `user.name=${who}`, '-c', 'user.email=board@supercode.invalid', 'commit', '--quiet', '--author', `${who} <board@supercode.invalid>`, '-m', message, '--', ...files]);
     const branch = this.branch();
+    if (!branch) return this.detached();
     for (let attempt = 0; attempt < 3; attempt++) {
       if (quiet(this.root, ['push', '--quiet', 'origin', `HEAD:${branch}`]) !== null) return;
       if (quiet(this.root, ['fetch', '--quiet', 'origin', branch]) === null) throw new Error('the board documents were committed here but could not be pushed: the remote did not answer');

package/board/published.mjs

@@ -34,12 +34,37 @@ export function taskStatusOf(lane, completedAt) {
   return 'todo';
 }
 
-export function cardSnapshot(db,row) {
-  const own=db.prepare('SELECT * FROM supercode_cards WHERE task_id=?').get(row.id)??{};
-  const comments=db.prepare('SELECT id,author,body,created_at FROM task_comments WHERE task_id=? ORDER BY id').all(row.id).map(comment=>({...comment,created_at:at(comment.created_at)}));
+const REVIEW_KINDS="('review_requested','approved','review_approved','changes_requested','escalated')";
+/** A card's rows read one card at a time. */
+const perCard=db=>({
+  own:id=>db.prepare('SELECT * FROM supercode_cards WHERE task_id=?').get(id),
+  comments:id=>db.prepare('SELECT id,author,body,created_at FROM task_comments WHERE task_id=? ORDER BY id').all(id),
+  reviews:id=>db.prepare(`SELECT id,kind,payload,created_at FROM task_events WHERE task_id=? AND kind IN ${REVIEW_KINDS} ORDER BY id`).all(id),
+  attempts:id=>db.prepare('SELECT id,profile,step_key,status,started_at,ended_at,outcome,summary,metadata,error FROM task_runs WHERE task_id=? ORDER BY id').all(id),
+  parents:id=>db.prepare('SELECT parent_id FROM task_links WHERE child_id=? ORDER BY parent_id').all(id).map(link=>link.parent_id),
+  children:id=>db.prepare('SELECT child_id FROM task_links WHERE parent_id=? ORDER BY child_id').all(id).map(link=>link.child_id),
+});
+/** Every card's rows read in one query per table: what a write that snapshots the whole board reads (one query per card
+ *  and table was most of each write's transaction, 470 ms for this board's 1,553 cards, t_23e47b73). */
+function wholeBoard(db){
+  const grouped=(sql,pick)=>{const by=new Map();for(const row of db.prepare(sql).all()){const list=by.get(row.task_id);const value=pick(row);if(list)list.push(value);else by.set(row.task_id,[value]);}return id=>by.get(id)??[];};
+  const own=new Map(db.prepare('SELECT * FROM supercode_cards').all().map(row=>[row.task_id,row]));
+  return {
+    own:id=>own.get(id),
+    comments:grouped('SELECT task_id,id,author,body,created_at FROM task_comments ORDER BY id',({id,author,body,created_at})=>({id,author,body,created_at})),
+    reviews:grouped(`SELECT task_id,id,kind,payload,created_at FROM task_events WHERE kind IN ${REVIEW_KINDS} ORDER BY id`,({id,kind,payload,created_at})=>({id,kind,payload,created_at})),
+    attempts:grouped('SELECT task_id,id,profile,step_key,status,started_at,ended_at,outcome,summary,metadata,error FROM task_runs ORDER BY id',({task_id:_,...run})=>run),
+    parents:grouped('SELECT child_id AS task_id,parent_id FROM task_links ORDER BY parent_id',link=>link.parent_id),
+    children:grouped('SELECT parent_id AS task_id,child_id FROM task_links ORDER BY child_id',link=>link.child_id),
+  };
+}
+
+export function cardSnapshot(db,row,rows=perCard(db)) {
+  const own=rows.own(row.id)??{};
+  const comments=rows.comments(row.id).map(comment=>({...comment,created_at:at(comment.created_at)}));
   const verdicts={review_requested:'requested',approved:'approved',review_approved:'approved',changes_requested:'changes_requested',escalated:'escalated'};
-  const reviews=db.prepare("SELECT id,kind,payload,created_at FROM task_events WHERE task_id=? AND kind IN ('review_requested','approved','review_approved','changes_requested','escalated') ORDER BY id").all(row.id).map(event=>{const payload=parse(event.payload,{});return {id:event.id,verdict:verdicts[event.kind],by:payload.reviewer??payload.by??payload.profile,reason:payload.reason,at:at(event.created_at)};});
-  const attempts=db.prepare('SELECT id,profile,step_key,status,started_at,ended_at,outcome,summary,metadata,error FROM task_runs WHERE task_id=? ORDER BY id').all(row.id).map(run=>{
+  const reviews=rows.reviews(row.id).map(event=>{const payload=parse(event.payload,{});return {id:event.id,verdict:verdicts[event.kind],by:payload.reviewer??payload.by??payload.profile,reason:payload.reason,at:at(event.created_at)};});
+  const attempts=rows.attempts(row.id).map(run=>{
     const native=parse(run.metadata,{})?.supercode??null;
     const session=native?{machine:native.machine,harness:native.harness,session_id:native.session_id,address:native.address,pane:native.pane,surface:native.surface,state:native.state,closed_at:native.closed_at,agent:native.agent_id??null,counted:native.counted??null,profile_differences:native.agent_profile_mismatch?.differences??null}:null;
     const metadata=parse(run.metadata,{});delete metadata.supercode;
@@ -51,16 +76,17 @@ export function cardSnapshot(db,row) {
     tags:parse(own.tags,null)??(row.tenant?[row.tenant]:[]),sponsor:own.sponsor??null,sponsor_evidence:parse(own.sponsor_evidence),requested_by_unverified:parse(own.requested_by_unverified),machine:own.machine??null,subtask_of:own.subtask_of??null,archived:!!own.archived,
     blocked_by:parse(own.blocked_by,[]),paused:parse(own.paused),every:own.every??null,
     workspace:{kind:row.workspace_kind,path:row.workspace_path,branch:row.branch_name,root:own.workspace_root,anchor:own.workspace_anchor},
-    parents:db.prepare('SELECT parent_id FROM task_links WHERE child_id=? ORDER BY parent_id').all(row.id).map(link=>link.parent_id),
-    reviews,children:db.prepare('SELECT child_id FROM task_links WHERE parent_id=? ORDER BY child_id').all(row.id).map(link=>link.child_id),comments,attempts};
+    parents:rows.parents(row.id),
+    reviews,children:rows.children(row.id),comments,attempts};
 }
 
 /** Runs inside the backing's native transaction, including files and ztrack projections. */
 export function recordCardPublications(db,actor) {
-  const rows=db.prepare('SELECT * FROM tasks ORDER BY id').all(),present=new Set();let changed=0;
+  const rows=db.prepare('SELECT * FROM tasks ORDER BY id').all(),present=new Set(),board=wholeBoard(db);let changed=0;
+  const heads=new Map(db.prepare('SELECT resource_id,digest,last_event_id FROM supercode_publication_heads').all().map(head=>[head.resource_id,head]));
   for(const row of rows){
-    present.add(row.id);const snapshot=JSON.stringify(cardSnapshot(db,row)),digest=hash(snapshot);
-    const previous=db.prepare('SELECT digest,last_event_id FROM supercode_publication_heads WHERE resource_id=?').get(row.id);
+    present.add(row.id);const snapshot=JSON.stringify(cardSnapshot(db,row,board)),digest=hash(snapshot);
+    const previous=heads.get(row.id);
     if(previous?.digest===digest)continue;
     const last=db.prepare('SELECT id,kind FROM task_events WHERE task_id=? ORDER BY id DESC LIMIT 1').get(row.id);
     const kind=`task.${previous?(last?.id!==previous.last_event_id?kinds[last?.kind]??'changed':'changed'):'created'}`;

package/board/ztrack.mjs

@@ -14,6 +14,10 @@ import { basename, dirname, resolve } from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
 import { actorOf, event, json, now, principalOf, setActor } from './store.mjs';
 import { GitDocument } from './git-document.mjs';
+import { noteStep, recordSlow } from '@volter/supercode-harness-sdk/slow-log';
+
+/** Run `fn` as a named step of the running command (the slow log names its slowest step). */
+const timedStep = (step, fn) => { const started = performance.now(); try { return fn(); } finally { noteStep(step, performance.now() - started); } };
 import { fire, statusOf, storedStatus, workflowFor } from './workflow.mjs';
 import { completionPhase, inspectCompletionEvidence, withCompletionEvidence } from './ci.mjs';
 
@@ -333,14 +337,17 @@ class ZtrackBoard {
   // ------------------------------------------------------------------ writes
 
   tx(fn) {
-    // A pull request merged on the remote (a page edited in Git Notes) is in the document before this write reads it.
-    try { this.git?.pull(); } catch (error) { process.stderr.write(`board: ${error.message}\n`); }
+    // A write inside a write is part of it: the pull below happened once, for the outermost one.
     if (this.active) return fn(this.db);
+    // A pull request merged on the remote (a page edited in Git Notes) is in the document before this write reads it.
+    try { timedStep('board git pull', () => this.git?.pull()); } catch (error) { process.stderr.write(`board: ${error.message}\n`); }
+    let txStarted = 0;
     for (let attempt = 0; ; attempt++) {
       const hermes = this.seed ? null : this.prepareHermes();
       this.dropOverlay();
       let outcome;
       try {
+        txStarted = performance.now();
         this.conn.exec('BEGIN');
         this.active = true;
         this.conn.exec('DELETE FROM temp.board_dirty; DELETE FROM temp.board_touched');
@@ -364,11 +371,16 @@ class ZtrackBoard {
           this.put('pending_hashes', Object.fromEntries(files.map((f) => [f.role, { before: f.before, after: f.after }])));
         }
         this.conn.exec('COMMIT');
+        // A long transaction holds every other writer of this board (the dispatcher's tick included), so it is a line
+        // of its own in the slow log, from whichever process ran it.
+        const held = performance.now() - txStarted;
+        noteStep('board transaction', held);
+        recordSlow({ kind: 'call', name: 'board transaction', ms: held, slowest: null, extra: { board: this.path } });
         outcome = { result, files, intent };
       } catch (error) {
         try { this.conn.exec('ROLLBACK'); } catch { /* not begun */ }
         this.active = false; this.seen = null; this.loadErrors();
-        if (error === RETRY || sqliteContention(error)) { backoff(attempt); continue; }
+        if (error === RETRY || sqliteContention(error)) { backoff(attempt); noteStep('board lock wait', performance.now() - txStarted); continue; }
         throw error;
       } finally { this.active = false; }
       this.seed = null;
@@ -376,7 +388,7 @@ class ZtrackBoard {
         this.publish(outcome);
         // Committed as the session (or person) whose act this write was, and pushed.
         const byManager = this.manager !== null && (actorOf() === this.manager || principalOf() === this.manager);
-        if (!byManager) try { this.git?.commit(actorOf(), `board: ${outcome.files.map((file) => basename(file.path)).join(', ')} written for ${actorOf() ?? 'the board'}`); }
+        if (!byManager) try { timedStep('board git commit and push', () => this.git?.commit(actorOf(), `board: ${outcome.files.map((file) => basename(file.path)).join(', ')} written for ${actorOf() ?? 'the board'}`)); }
         catch (error) { process.stderr.write(`board: ${error.message}\n`); }
       }
       this.exportFollowing();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-orchestrator",
-  "version": "0.5.10",
+  "version": "0.5.12",
   "type": "module",
   "description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
   "exports": {
@@ -25,7 +25,7 @@
   },
   "dependencies": {
     "@volter/supercode-harness-sdk": "^0.3.41",
-    "@volter/teams": "0.2.6",
+    "@volter/teams": "0.2.7",
     "@volter/ztrack": "^2.4.23",
     "yaml": "^2.9.0"
   },