@volter/supercode-orchestrator 0.5.228 → 0.5.230

package/board/event-stream.mjs

@@ -4,7 +4,8 @@ import { watch, existsSync, readFileSync, realpathSync, statSync } from 'node:fs
 import { dirname, basename, join, resolve } from 'node:path';
 import { createHash } from 'node:crypto';
 import { once } from 'node:events';
-import { listBoards, openBoard, backingPaths } from './store.mjs';
+import { listBoards, backingPaths } from './store.mjs';
+import { openPublicationReader } from './publication-reader.mjs';
 
 const identity = root => createHash('sha256').update(realpathSync(root)).digest('hex');
 const cursorOf = (home, boards) => Buffer.from(JSON.stringify({version:1,home,boards})).toString('base64url');
@@ -24,7 +25,11 @@ export async function* followBoardPublications({root,after,signal}) {
   const settled=new Map();
   const statOf=path=>{try{const s=statSync(path);return `${s.size}:${s.mtimeMs}`;}catch{return '-';}};
   const notify=(name=null)=>{
-    const named=name===null?[]:stores.filter(path=>basename(path)===name);
+    // A store in rollback-journal mode is written in place, and on macOS FSEvents names no change to the store file
+    // itself, only its journal coming and going (a board's sqlite store, kept in `delete` journal mode; VO-3761, where
+    // every start waited out RESETTLE_MS): a journal's notice is its store's, read against the store's own stamp.
+    const store=name!==null&&name.endsWith('-journal')?name.slice(0,-'-journal'.length):name;
+    const named=store===null?[]:stores.filter(path=>basename(path)===store);
     // A store file no pass stamps (the root's kanban.db where only named boards exist) reads '-' until one does.
     if(named.length&&named.every(path=>statOf(path)===(settled.get(path)??'-')))return;
     dirty=true;wake?.();
@@ -40,12 +45,12 @@ export async function* followBoardPublications({root,after,signal}) {
   const register=slugs=>{
     for(const entry of watchers.values())entry.accept=[];
     stores=[join(root,'kanban.db'),join(root,'kanban.db-wal')];
-    directory(root,name=>['kanban.db','kanban.db-wal','kanban'].includes(name));
+    directory(root,name=>['kanban.db','kanban.db-wal','kanban.db-journal','kanban'].includes(name));
     directory(join(root,'kanban'),name=>['boards','ztrack.json','board'].includes(name));
     directory(join(root,'kanban','boards'),()=>true);
     for(const slug of slugs){
       const paths=backingPaths(root,slug);
-      file(paths.sqlite);file(`${paths.sqlite}-wal`);file(paths.ztrack);stores.push(paths.sqlite,`${paths.sqlite}-wal`);
+      file(paths.sqlite);file(`${paths.sqlite}-wal`);file(`${paths.sqlite}-journal`);file(paths.ztrack);stores.push(paths.sqlite,`${paths.sqlite}-wal`);
       directory(paths.files,name=>name.endsWith('.json'));
       try {const cfg=JSON.parse(readFileSync(paths.ztrack,'utf8'));const document=resolve(dirname(paths.ztrack),cfg.document);file(document);const own=basename(document);directory(dirname(document),name=>name===`${own}.archive.md`||(name.startsWith(`${own}.archive-`)&&name.endsWith('.md')));}catch{/* other backing */}
     }
@@ -65,7 +70,7 @@ export async function* followBoardPublications({root,after,signal}) {
       if(inventory!==null&&current!==inventory)throw resync('Workflow board inventory changed');
       inventory=current;
       for(const slug of slugs){
-        const board=openBoard(root,slug);
+        const board=openPublicationReader(root,slug);
         try {
           // The follower only reads the board's retained log: what a writer changed, it published in its own transaction,
           // and what a native writer marked, the board's owner publishes on its wake (published.mjs). Read in one view,

package/board/fence.mjs

@@ -9,6 +9,7 @@
 import { renameSync, readFileSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { boardHome } from '@volter/teams/board-owner';
+import { assertRestingRows } from './resting-state.mjs';
 import { assertGenerationWriter } from './generation.mjs';
 
 let mine = null;
@@ -34,6 +35,7 @@ export function takeEpoch(root, boards) {
 /** At a write transaction's start: in the owner, store its epoch unless a newer one is stored, and exit when it is. */
 export function fenced(db) {
   assertGenerationWriter();
+  assertRestingRows(db);
   if (mine === null) return;
   db.exec(STATE);
   const r = db.prepare(`INSERT INTO supercode_board_state (key, value) VALUES ('owner_epoch', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value

package/board/files.mjs

@@ -1,4 +1,5 @@
 import { fillReadableIds } from './readable-id.mjs';
+import { assertBacklogQualified, assertPauseRecord, assertRestingRows } from './resting-state.mjs';
 // The `files` backing of a board (docs/architecture/orchestrator.md §2.9, "Backings"): the same tables as a
 // Hermes `kanban.db`, kept as a directory of JSON files instead of one SQLite file. `<id>.json` holds everything of
 // one card (its `tasks` row, its `supercode_cards` row, the links to its parents, its comments, events, runs,
@@ -16,7 +17,7 @@ import { CARD_NUMBERS, unnumberTasks } from './store.mjs';
 import { createHash } from 'node:crypto';
 import { join } from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
-import { LIST_REVISION, listRevision } from './listing.mjs';
+import { LIST_CONNECTION, LIST_REVISION, READ_BOUNDARY, listRevision } from './listing.mjs';
 
 const STATE = '_board.json';
 
@@ -61,6 +62,8 @@ class FilesBoard {
     const self = this;
     // the connection the engine holds: SQLite's statement API over whatever snapshot is current
     this.db = {
+      get [LIST_CONNECTION]() { return self.mem; },
+      [READ_BOUNDARY]: fn => self.read(fn),
       [LIST_REVISION]: () => self.read(() => listRevision(self.mem)),
       prepare(sql) {
         return {
@@ -93,66 +96,90 @@ class FilesBoard {
     if (this.mem && stamp === this.stamp) return;
     const marked = this.mem && this.mem.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'supercode_publication_dirty'").get()
       ? this.mem.prepare('SELECT DISTINCT task_id FROM supercode_publication_dirty').all().map((r) => r.task_id) : [];
-    this.mem?.close();
     this.statements.clear();
     const mem = new DatabaseSync(':memory:');
-    this.stateText = existsSync(join(this.dir, STATE)) ? readFileSync(join(this.dir, STATE), 'utf8') : null;
-    const state = this.stateText === null ? null : JSON.parse(this.stateText);
-    for (const sql of state?.schema ?? []) mem.exec(sql);
-    this.init(mem, { fresh: !state?.schema?.length });
-    // the schema as the directory keeps it: a layout this load added (a table, a column, the card numbers) is written to
-    // every card's file by the next write (persist)
-    this.keptSchema = state?.schema ?? [];
-    const columns = (table) => new Set(mem.prepare(`PRAGMA table_info(${quote(table)})`).all().map((c) => c.name));
-    const insert = (table, rows) => {
-      const have = columns(table);
-      for (const row of rows) {
-        const keys = Object.keys(row).filter((k) => have.has(k));
-        mem.prepare(`INSERT INTO ${quote(table)} (${keys.map(quote).join(', ')}) VALUES (${keys.map(() => '?').join(', ')})`).run(...keys.map((k) => row[k]));
+    const previousMem = this.mem, previousKeys = this.keys;
+    // Parse and qualify a replacement before retiring the committed projection.
+    // A malformed or partially written file must not leave closed prepared readers.
+    try {
+      const stateText = existsSync(join(this.dir, STATE)) ? readFileSync(join(this.dir, STATE), 'utf8') : null;
+      const state = stateText === null ? null : JSON.parse(stateText);
+      for (const sql of state?.schema ?? []) mem.exec(sql);
+      this.init(mem, { fresh: !state?.schema?.length });
+      // the schema as the directory keeps it: a layout this load added (a table, a column, the card numbers) is written to
+      // every card's file by the next write (persist)
+      const keptSchema = state?.schema ?? [];
+      const columns = (table) => new Set(mem.prepare(`PRAGMA table_info(${quote(table)})`).all().map((c) => c.name));
+      const insert = (table, rows) => {
+        const have = columns(table);
+        for (const row of rows) {
+          const keys = Object.keys(row).filter((k) => have.has(k));
+          mem.prepare(`INSERT INTO ${quote(table)} (${keys.map(quote).join(', ')}) VALUES (${keys.map(() => '?').join(', ')})`).run(...keys.map((k) => row[k]));
+        }
+      };
+      const previous = this.texts;
+      const texts = new Map();
+      const cards = [];
+      for (const name of readdirSync(this.dir)) {
+        if (!name.endsWith('.json') || name.startsWith('_')) continue;
+        const text = readFileSync(join(this.dir, name), 'utf8');
+        const card = JSON.parse(text);
+        texts.set(card.id, text);
+        cards.push(card);
       }
-    };
-    const previous = this.texts;
-    this.texts = new Map();
-    const cards = [];
-    for (const name of readdirSync(this.dir)) {
-      if (!name.endsWith('.json') || name.startsWith('_')) continue;
-      const text = readFileSync(join(this.dir, name), 'utf8');
-      const card = JSON.parse(text);
-      this.texts.set(card.id, text);
-      cards.push(card);
-    }
-    // The id counters first: a number handed out before stays handed out.
-    for (const { name, seq } of state?.sequence ?? []) {
-      if (mem.prepare('UPDATE sqlite_sequence SET seq = MAX(seq, ?) WHERE name = ?').run(seq, name).changes === 0) mem.prepare('INSERT INTO sqlite_sequence (name, seq) VALUES (?, ?)').run(name, seq);
-    }
-    // Inserting a card's row numbers it when it has no number yet (store.mjs layOutNumbers): every number a card file
-    // holds loads first, so each card keeps its own and a new one is past them all, and then the cards load in the
-    // order they were made.
-    for (const card of cards) if (card.tables?.[CARD_NUMBERS]) insert(CARD_NUMBERS, card.tables[CARD_NUMBERS]);
-    const madeAt = (card) => Number(card.tables?.tasks?.[0]?.created_at ?? 0);
-    cards.sort((a, b) => madeAt(a) - madeAt(b) || (String(a.id) < String(b.id) ? -1 : String(a.id) > String(b.id) ? 1 : 0));
-    for (const card of cards) for (const [table, rows] of Object.entries(card.tables ?? {})) if (table !== CARD_NUMBERS) insert(table, rows);
-    for (const [table, rows] of Object.entries(state?.tables ?? {})) insert(table, rows);
-    // A task whose file names no number keeps none: only cards are numbered.
-    unnumberTasks(mem, new Set(cards.filter((card) => card.tables?.[CARD_NUMBERS]?.length).map((card) => card.id)),
-      (state?.sequence ?? []).find((row) => row.name === CARD_NUMBERS)?.seq ?? 0);
-    // Loading inserts every card, which marks every card changed (published.mjs). The first load of this projection keeps
-    // those marks: its next write compares each card with its published head once (a named limit: each projection opened
-    // compares every card once, as each load already reads every card's file). A reload keeps the marks not yet
-    // published and adds only the cards whose files changed since (a hand edit, a card another writer added or removed);
-    // this process's own writes are already the texts it knows (persist), so they are not marked again.
-    if (previous && mem.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'supercode_publication_dirty'").get()) {
-      mem.exec('DELETE FROM supercode_publication_dirty');
-      const mark = mem.prepare('INSERT INTO supercode_publication_dirty (task_id) VALUES (?)');
-      const changed = [...new Set([...previous.keys(), ...this.texts.keys()])].filter((id) => previous.get(id) !== this.texts.get(id));
-      for (const id of new Set([...marked, ...changed])) mark.run(id);
+      // The id counters first: a number handed out before stays handed out.
+      for (const { name, seq } of state?.sequence ?? []) {
+        if (mem.prepare('UPDATE sqlite_sequence SET seq = MAX(seq, ?) WHERE name = ?').run(seq, name).changes === 0) mem.prepare('INSERT INTO sqlite_sequence (name, seq) VALUES (?, ?)').run(name, seq);
+      }
+      // Inserting a card's row numbers it when it has no number yet (store.mjs layOutNumbers): every number a card file
+      // holds loads first, so each card keeps its own and a new one is past them all, and then the cards load in the
+      // order they were made.
+      for (const card of cards) if (card.tables?.[CARD_NUMBERS]) insert(CARD_NUMBERS, card.tables[CARD_NUMBERS]);
+      const madeAt = (card) => Number(card.tables?.tasks?.[0]?.created_at ?? 0);
+      cards.sort((a, b) => madeAt(a) - madeAt(b) || (String(a.id) < String(b.id) ? -1 : String(a.id) > String(b.id) ? 1 : 0));
+      for (const card of cards) for (const [table, rows] of Object.entries(card.tables ?? {})) if (table !== CARD_NUMBERS) insert(table, rows);
+      for (const [table, rows] of Object.entries(state?.tables ?? {})) insert(table, rows);
+      // A files import can carry execution history as well as card flags. Qualify each
+      // backlog against that history before publishing the imported connection to readers.
+      assertRestingRows(mem, this.dir);
+      for (const row of mem.prepare("SELECT task_id,paused FROM supercode_cards WHERE paused IS NOT NULL AND paused != 'null'").all()) {
+        assertPauseRecord(typeof row.paused === 'string' ? JSON.parse(row.paused) : row.paused, { id: row.task_id, source: 'files' });
+      }
+      for (const row of mem.prepare("SELECT task_id,backlog FROM supercode_cards WHERE backlog IS NOT NULL AND backlog != 'null'").all()) {
+        const backlog = typeof row.backlog === 'string' ? JSON.parse(row.backlog) : row.backlog;
+        assertBacklogQualified(mem, row.task_id, { backlog, source: 'files' });
+      }
+      // A task whose file names no number keeps none: only cards are numbered.
+      unnumberTasks(mem, new Set(cards.filter((card) => card.tables?.[CARD_NUMBERS]?.length).map((card) => card.id)),
+        (state?.sequence ?? []).find((row) => row.name === CARD_NUMBERS)?.seq ?? 0);
+      // Loading inserts every card, which marks every card changed (published.mjs). The first load of this projection keeps
+      // those marks: its next write compares each card with its published head once (a named limit: each projection opened
+      // compares every card once, as each load already reads every card's file). A reload keeps the marks not yet
+      // published and adds only the cards whose files changed since (a hand edit, a card another writer added or removed);
+      // this process's own writes are already the texts it knows (persist), so they are not marked again.
+      if (previous && mem.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'supercode_publication_dirty'").get()) {
+        mem.exec('DELETE FROM supercode_publication_dirty');
+        const mark = mem.prepare('INSERT INTO supercode_publication_dirty (task_id) VALUES (?)');
+        const changed = [...new Set([...previous.keys(), ...texts.keys()])].filter((id) => previous.get(id) !== texts.get(id));
+        for (const id of new Set([...marked, ...changed])) mark.run(id);
+      }
+      // a card whose file is gone has no row to fire its delete trigger: its live head is marked, so it is published removed
+      if (mem.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'supercode_publication_heads'").get()) markRemovedHeads(mem);
+      this.mem = mem;
+      this.watch();
+      fillReadableIds(mem);
+      this.stateText = stateText;
+      this.keptSchema = keptSchema;
+      this.texts = texts;
+      this.stamp = stamp;
+      previousMem?.close();
+    } catch (error) {
+      this.mem = previousMem;
+      this.keys = previousKeys;
+      mem.close();
+      this.statements.clear();
+      throw error;
     }
-    // a card whose file is gone has no row to fire its delete trigger: its live head is marked, so it is published removed
-    if (mem.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'supercode_publication_heads'").get()) markRemovedHeads(mem);
-    this.mem = mem;
-    this.watch();
-    fillReadableIds(mem);
-    this.stamp = stamp;
   }
 
   /** Each table that belongs to a card, with the column naming the card. */
@@ -242,11 +269,15 @@ class FilesBoard {
   }
 
   read(fn) {
-    // a held board knows its files until an event says otherwise; one opened per use reads them on each query
-    const known = this.held && !this.changed && !this.unwatched && this.mem && this.stamp !== null;
-    // the mark clears only once the files were read: a read that fails leaves it, and the next read tries again
-    if (!this.inTx && !known) { if (this.stampOf() !== this.stamp) this.load(); if (this.held) this.changed = false; }
-    return fn();
+    if (this.readDepth) return fn();
+    this.readDepth = 1;
+    try {
+      // a held board knows its files until an event says otherwise; one opened per use reads them on each query
+      const known = this.held && !this.changed && !this.unwatched && this.mem && this.stamp !== null;
+      // the mark clears only once the files were read: a read that fails leaves it, and the next read tries again
+      if (!this.inTx && !known) { if (!this.mem || this.stampOf() !== this.stamp) this.load(); if (this.held) this.changed = false; }
+      return fn();
+    } finally { this.readDepth = 0; }
   }
 
   /** A write outside a transaction commits on its own, as SQLite's autocommit does; VACUUM and PRAGMA only read here. */