@volter/supercode-orchestrator 0.4.82 → 0.4.84

package/board/cli.mjs

@@ -9,7 +9,6 @@ import { spawnSync } from 'node:child_process';
 import { basename, join, resolve } from 'node:path';
 import { tmpdir } from 'node:os';
 import { read } from '../orchestration.mjs';
-import { holdDispatcherLock } from '../hermes-locks.mjs';
 import { backingPaths, layOut, backingOf, boardExists, boardPath, DEFAULT_BOARD, event, json, listBoards, logsDir, now, openBoard, setActor, tx } from './store.mjs';
 import { openZtrackBoard } from './ztrack.mjs';
 import * as engine from './engine.mjs';
@@ -1139,11 +1138,10 @@ export function summary(r) {
 
 /**
  * The dispatcher alone, for a home whose board is dispatched without the rest of the orchestrator (the lanes are
- * re-read every tick, so a profile's change takes effect without a restart). It holds the board's one-dispatcher
- * lock, as the daemon and Hermes's gateways do, so one dispatcher serves a board.
+ * re-read every tick, so a profile's change takes effect without a restart). Card claims
+ * are version-checked by the backing; the dispatcher takes no file lock.
  */
 async function serve(root, args = {}) {
-  const lock = await holdDispatcherLock(root);
   // Hermes's daemon flags: --interval seconds, --max starts per tick, --failure-limit, --pidfile, --verbose
   const override = Number(args.interval) > 0 ? { interval: Number(args.interval) } : {};
   // --max and --failure-limit are params of every tick, read by the workflow as its own
@@ -1205,7 +1203,6 @@ async function serve(root, args = {}) {
       if (moved) { log(`orchestrator ${moved} is installed (this serve runs ${running}); stopping, for its keeper to start the new one`); break; }
     }
   } finally {
-    lock.release();
     if (args.pidfile) { try { unlinkSync(args.pidfile); } catch { /* gone */ } }
   }
   log(stopped ? 'stopped (a signal)' : 'stopped');

package/board/dispatch.mjs

@@ -22,7 +22,6 @@ import { basename, dirname, join } from 'node:path';
 import { randomUUID } from 'node:crypto';
 import { fileURLToPath } from 'node:url';
 import { boardPath, claimer, event, json, listBoards, logsDir, now, openBoard, setActor, tx } from './store.mjs';
-import { holdBoardTickLock } from '../hermes-locks.mjs';
 import { doors, fire, loadWorkflow, resumeOf, runSpec, transitionsFor, scopeOf, statusOf, templateValue, workflowFor } from './workflow.mjs';
 import { evaluate, holds, render } from './expr.mjs';
 import {
@@ -1206,7 +1205,7 @@ export async function plan({ root, lanes, settings, slugs = null, params = {} })
 /**
  * One tick over every board of the home at `root`. `lanes` from `lanesOf`, `settings` from `boardSettings`.
  * `only` limits starting to one card (a create that answers whether its card started). Each board is ticked under its
- * per-tick lock (Hermes's `.dispatch.lock`), skipped when another dispatcher holds it. Answers one report per board.
+ * version-checked card claims; no dispatcher file lock. Answers one report per board.
  */
 export async function tick({ root, lanes, settings, only = null, slugs = null, dispatch = true, params = {} }) {
   // a verb's flags (dispatch --max, --failure-limit) are params of this tick, read by the workflow as its own
@@ -1230,8 +1229,6 @@ export async function tick({ root, lanes, settings, only = null, slugs = null, d
     const path = boardPath(root, slug);
     const marker = `${path}.corrupt-seen`;
     if (existsSync(marker) && readFileSync(marker, 'utf8').trim() === fingerprint(path)) { report.error = 'skipped: the board file was corrupt and has not changed since (`supercode workflow repair`)'; out.push(report); continue; }
-    let lock = null;
-    try { lock = await holdBoardTickLock(path); } catch (error) { report.skipped_locked = error.message; out.push(report); continue; }
     let board;
     try {
       board = openBoard(root, slug);
@@ -1268,7 +1265,6 @@ export async function tick({ root, lanes, settings, only = null, slugs = null, d
     } finally {
       await Promise.all(report.answers);
       board?.close();
-      lock.release();
     }
     out.push(report);
   }

package/board/files.mjs

@@ -5,23 +5,20 @@
 // counters. A card's history is a file a person or a diff reads, and nothing but supercode's code touches it.
 //
 // The engine speaks SQL to every backing, so this one answers it from an in-memory SQLite database loaded from the
-// directory: a write transaction takes the directory's lock (`_board.lock`), reloads what another process wrote,
-// runs, and writes back the files of the cards it touched (temporary triggers name them), `_board.json` last. A read
+// directory: a write transaction reads the directory's content hash, reloads what another process wrote,
+// runs, retries if that hash changed, and writes the cards it touched (temporary triggers name them), `_board.json` last. A read
 // outside a transaction reloads first when `_board.json` changed, as a SQLite reader sees the last commit.
-import { closeSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync, writeSync } from 'node:fs';
-import { hostname } from 'node:os';
+import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from 'node:fs';
+import { createHash } from 'node:crypto';
 import { join } from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
 
 const STATE = '_board.json';
-const LOCK = '_board.lock';
-const LOCK_WAIT_MS = 10_000; // as the sqlite backing's busy_timeout
 
 /** Whether `dir` holds a files board. */
 export const isFilesBoard = (dir) => existsSync(join(dir, STATE));
 
 const sleep = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
-const alive = (pid) => { try { process.kill(pid, 0); return true; } catch (error) { return error.code === 'EPERM'; } };
 const quote = (name) => `"${String(name).replaceAll('"', '""')}"`;
 
 function writeAtomic(file, text) {
@@ -66,23 +63,28 @@ class FilesBoard {
       exec: (sql) => self.write(sql, () => self.mem.exec(sql)),
       close: () => self.close(),
     };
-    this.locked(() => this.load());
+    this.load();
   }
 
   // ---------------------------------------------------------------- snapshot
 
   stampOf() {
-    try { const s = statSync(join(this.dir, STATE)); return `${s.ino}:${s.mtimeMs}:${s.size}`; } catch { return 'none'; }
+    const hash = createHash('sha256');
+    for (const name of readdirSync(this.dir).filter(name => name.endsWith('.json')).sort()) {
+      try { hash.update(name).update('\0').update(readFileSync(join(this.dir, name))).update('\0'); }
+      catch (error) { if (error.code !== 'ENOENT') throw error; return this.stampOf(); }
+    }
+    return hash.digest('hex');
   }
 
-  /** (Re)load the directory into a fresh in-memory database; the caller holds the lock. */
+  /** (Re)load the directory into a fresh in-memory database; writes compare the content hash. */
   load() {
     const stamp = this.stampOf();
     if (this.mem && stamp === this.stamp) return;
     this.mem?.close();
     this.statements.clear();
     const mem = new DatabaseSync(':memory:');
-    this.stateText = stamp === 'none' ? null : readFileSync(join(this.dir, STATE), 'utf8');
+    this.stateText = existsSync(join(this.dir, STATE)) ? readFileSync(join(this.dir, STATE), 'utf8') : null;
     const state = this.stateText === null ? null : JSON.parse(this.stateText);
     for (const sql of state?.schema ?? []) mem.exec(sql);
     this.init(mem, { fresh: !state?.schema?.length });
@@ -143,7 +145,7 @@ class FilesBoard {
     return this.mem.prepare("SELECT sql FROM sqlite_master WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%' ORDER BY CASE type WHEN 'table' THEN 0 ELSE 1 END, rowid").all().map((r) => r.sql);
   }
 
-  /** Write the files of the cards the transaction touched, then `_board.json`; the caller holds the lock. */
+  /** Write the files of the cards the transaction touched, then `_board.json`; writes compare the content hash. */
   persist(schemaBefore) {
     const schema = this.schema();
     const reshaped = schema.join('\n') !== schemaBefore.join('\n');
@@ -157,6 +159,7 @@ class FilesBoard {
     }
     const state = { format: 'supercode-board-files', format_version: 1, schema, tables: boardTables, sequence: this.mem.prepare('SELECT name, seq FROM sqlite_sequence ORDER BY name').all() };
     const stateText = `${JSON.stringify(state, null, 2)}\n`;
+    const writes = [];
     let wrote = false;
     for (const id of touched) {
       const tables = {};
@@ -166,49 +169,28 @@ class FilesBoard {
       }
       const file = join(this.dir, `${String(id).replace(/[^A-Za-z0-9_.-]/g, '_')}.json`);
       if (!Object.keys(tables).length) {
-        if (this.texts.has(id)) { try { unlinkSync(file); } catch { /* already gone */ } this.texts.delete(id); wrote = true; }
+        if (this.texts.has(id)) { writes.push({ file, text: null }); wrote = true; }
         continue;
       }
       const text = `${JSON.stringify({ id, tables }, null, 2)}\n`;
       if (this.texts.get(id) === text) continue;
-      writeAtomic(file, text);
-      this.texts.set(id, text);
+      writes.push({ file, text });
       wrote = true;
     }
     if (wrote || stateText !== this.stateText) {
-      writeAtomic(join(this.dir, STATE), stateText);
-      this.stateText = stateText;
+      writes.push({ file: join(this.dir, STATE), text: stateText });
     }
-    this.mem.exec('DELETE FROM _touched');
-    this.stamp = this.stampOf();
-  }
-
-  // ---------------------------------------------------------------- lock and transactions
-
-  locked(fn) {
-    const file = join(this.dir, LOCK);
-    const holder = `${hostname()}:${process.pid}`;
-    const until = Date.now() + LOCK_WAIT_MS;
-    for (;;) {
-      try {
-        const fd = openSync(file, 'wx');
-        writeSync(fd, holder);
-        closeSync(fd);
-        break;
-      } catch (error) {
-        if (error.code !== 'EEXIST') throw error;
-        let held = '';
-        try { held = readFileSync(file, 'utf8'); } catch { continue; }
-        const [host, pid] = held.split(':');
-        // a holder on this host that is gone left its lock behind
-        if (host === hostname() && Number(pid) && !alive(Number(pid))) { try { unlinkSync(file); } catch { /* taken by another */ } continue; }
-        if (Date.now() > until) throw new Error(`the board ${this.dir} is locked by ${held || 'another writer'}`);
-        sleep(20);
-      }
+    if (this.stampOf() !== this.stamp) return false;
+    for (const { file, text } of writes) {
+      if (text === null) { try { unlinkSync(file); } catch (error) { if (error.code !== 'ENOENT') throw error; } }
+      else writeAtomic(file, text);
     }
-    try { return fn(); } finally { try { unlinkSync(file); } catch { /* gone */ } }
+    this.stamp = null; // next read reloads the committed file hashes and card texts
+    return true;
   }
 
+  // ---------------------------------------------------------------- transactions
+
   statement(sql) {
     let s = this.statements.get(sql);
     if (!s) { s = this.mem.prepare(sql); this.statements.set(sql, s); }
@@ -216,7 +198,7 @@ class FilesBoard {
   }
 
   read(fn) {
-    if (!this.inTx && this.stampOf() !== this.stamp) this.locked(() => this.load());
+    if (!this.inTx && this.stampOf() !== this.stamp) this.load();
     return fn();
   }
 
@@ -228,7 +210,8 @@ class FilesBoard {
 
   tx(fn, { force = false } = {}) {
     if (this.inTx) return fn(this.db);
-    return this.locked(() => {
+    for (;;) {
+      this.stamp = null;
       this.load();
       const before = this.schema();
       this.mem.exec('BEGIN');
@@ -245,13 +228,13 @@ class FilesBoard {
       }
       try {
         if (force) this.stateText = null;
-        this.persist(before);
+        if (!this.persist(before)) { sleep(10); continue; }
       } catch (error) {
         this.stamp = null; // what memory holds may not be on disk: the next reader reloads from the files
         throw error;
       }
       return out;
-    });
+    }
   }
 
   close() {

package/board/ztrack.mjs

@@ -2,7 +2,7 @@
 // ADR 0001. All markdown interpretation belongs to ztrack's public codec and kanban preset.
 import { createRequire } from 'node:module';
 import { createHash, randomBytes } from 'node:crypto';
-import { closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
+import { existsSync, mkdirSync, readFileSync, renameSync, statSync, writeFileSync } from 'node:fs';
 import { dirname, resolve } from 'node:path';
 import { spawnSync } from 'node:child_process';
 import { DatabaseSync } from 'node:sqlite';
@@ -34,7 +34,6 @@ class ZtrackBoard {
   constructor({ document, seed, marker, init, root, slug, compatPath }) {
     this.path = resolve(dirname(marker), document);
     this.archive = `${this.path}.archive.md`;
-    this.lockPath = `${marker}.lock`;
     this.seed = seed;
     this.root = root; this.slug = slug; this.init = init; this.compatPath = compatPath;
     this.codec = require('@volter/ztrack/board-document');
@@ -51,8 +50,8 @@ class ZtrackBoard {
     };
     this.tx(() => {});
   }
-  get(key, fallback = null) { return json(this.runtime.prepare('SELECT value FROM state WHERE key = ?').get(key)?.value, fallback); }
-  put(key, value) { this.runtime.prepare('INSERT INTO state VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, JSON.stringify(value)); }
+  get(key, fallback = null) { return this.computing && this.pendingValues.has(key) ? this.pendingValues.get(key) : json(this.runtime.prepare('SELECT value FROM state WHERE key = ?').get(key)?.value, fallback); }
+  put(key, value) { if (this.computing) { this.pendingValues.set(key, value); return; } this.runtime.prepare('INSERT INTO state VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value').run(key, JSON.stringify(value)); }
   stamp() {
     const fileStamp = (path) => { try { const s = statSync(path); return `${s.ino}:${s.size}:${s.mtimeMs}`; } catch { return 'missing'; } };
     return [fileStamp(this.path), fileStamp(this.archive), fileStamp(this.compatPath), this.get('version', '')].join('|');
@@ -72,47 +71,59 @@ class ZtrackBoard {
       if (current === file.before) atomic(file.path, file.text);
     }
     this.put('snapshot', pending.snapshot);
+    if (pending.intent) this.put(`applied:${pending.intent}`, true);
+    this.put('version', randomBytes(8).toString('hex'));
     this.runtime.prepare("DELETE FROM state WHERE key = 'pending'").run();
   }
   tx(fn) {
     if (this.active) return fn(this.db);
-    let lock;
-    for (let n = 0; ; n++) {
-      try { lock = openSync(this.lockPath, 'wx', 0o600); writeFileSync(lock, String(process.pid)); break; }
-      catch (error) {
-        if (error.code !== 'EEXIST' || n >= 100) throw error;
-        const pid = Number(read(this.lockPath));
-        if (pid) { try { process.kill(pid, 0); } catch (e) { if (e.code === 'ESRCH') { unlinkSync(this.lockPath); continue; } } }
-        Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 100);
-      }
-    }
-    try { return this.transaction(fn); } finally { closeSync(lock); unlinkSync(this.lockPath); }
+    return this.transaction(fn);
   }
+
   transaction(fn) {
-    for (let attempt = 0; attempt < 8; attempt++) {
-      this.runtime.exec('BEGIN IMMEDIATE'); this.active = true;
+    for (;;) {
+      try { this.runtime.exec('BEGIN IMMEDIATE'); }
+      catch (error) { if (!/busy|locked/i.test(error.message)) throw error; continue; }
+      this.active = true;
       let committed = false;
       try {
         this.recover();
+        const version = this.get('version');
+        this.runtime.exec('COMMIT');
+        // Parsing, rules and rendering run against a snapshot, without a runtime writer transaction.
+        this.pendingValues = new Map(); this.computing = true;
         this.load();
         if (this.seed) this.seedIntoMemory();
         else this.lift();
         const result = fn(this.db);
         const documents = this.documents();
         const files = [{ path: this.path, before: hash(this.text), text: documents.open }, { path: this.archive, before: hash(this.archiveText), text: documents.archived }];
-        if (files.some((f) => hash(read(f.path)) !== f.before)) { this.runtime.exec('ROLLBACK'); continue; }
         const snapshot = this.snapshot();
+        this.computing = false;
+        this.runtime.exec('BEGIN IMMEDIATE');
+        this.recover();
+        if (this.get('version') !== version || files.some((f) => hash(read(f.path)) !== f.before)) { this.runtime.exec('ROLLBACK'); continue; }
+        for (const [key, value] of this.pendingValues) this.put(key, value);
         // A durable intent precedes filesystem writes; recovery completes exactly this version.
-        this.put('pending', { files, snapshot });
+        const intent = randomBytes(16).toString('hex');
+        this.put('pending', { intent, files, snapshot });
         this.runtime.exec('COMMIT'); committed = true;
+        // A concurrent writer may recover this intent and advance the runtime. Re-read
+        // its identity before touching documents, retrying from any newer version.
+        this.runtime.exec('BEGIN IMMEDIATE');
+        if (this.get('pending')?.intent !== intent) {
+          const applied = this.get(`applied:${intent}`, false);
+          this.runtime.exec('ROLLBACK');
+          if (applied) { this.loadedStamp = null; this.seed = null; return result; }
+          continue;
+        }
         // Detect an editor racing the journal commit before replacing either document.
         if (files.some((f) => hash(read(f.path)) !== f.before)) {
-          this.runtime.exec('BEGIN IMMEDIATE'); this.runtime.prepare("DELETE FROM state WHERE key = 'pending'").run(); this.runtime.exec('COMMIT');
+          this.runtime.prepare("DELETE FROM state WHERE key = 'pending'").run(); this.runtime.exec('COMMIT');
           continue;
         }
         for (const f of files) if (hash(f.text) !== f.before) atomic(f.path, f.text);
-        this.runtime.exec('BEGIN IMMEDIATE');
-        this.put('snapshot', snapshot); this.runtime.prepare("DELETE FROM state WHERE key = 'pending'").run();
+        this.put('snapshot', snapshot); this.put(`applied:${intent}`, true); this.runtime.prepare("DELETE FROM state WHERE key = 'pending'").run();
         this.export();
         this.put('version', randomBytes(8).toString('hex'));
         this.runtime.exec('COMMIT');
@@ -121,11 +132,11 @@ class ZtrackBoard {
         return result;
       } catch (error) {
         try { this.runtime.exec('ROLLBACK'); } catch { /* intent already durable */ }
+        if (/busy|locked|Hermes wrote/i.test(error.message)) continue;
         if (committed) error.message += ' (the runtime journal retains any unfinished document write)';
         throw error;
-      } finally { this.active = false; }
+      } finally { this.active = false; this.computing = false; this.pendingValues = null; }
     }
-    throw new Error('board document kept changing during eight version-checked retries; no edit was overwritten');
   }
   seedIntoMemory() {
     const source = this.seed;

package/board.mjs

@@ -2,14 +2,12 @@
 // docs/plans/hermes-compat.md row 14). The board and its dispatcher are supercode's own (`board/`): Hermes Kanban's
 // format and lifecycle, each profile's lane decided by its `worker` in the IR. Where a Hermes gateway would run
 // its dispatcher, the daemon runs this one the same way: when the board's workflow says so (its `dispatch.serve`),
-// holding the board's one-dispatcher lock (`kanban/.dispatcher.lock`, the
-// lock Hermes's gateways take, so the two never dispatch one board together), one tick every `dispatch.tick` seconds
+// taking no file locks, one tick every `dispatch.tick` seconds
 // of the workflow (at least 1), the first after five seconds. Each tick runs as
 // `workflow dispatch` in a child process: a tick waits on supercode (the fleet, launches, mail), and the daemon's
 // own loop never does.
 import { spawn } from 'node:child_process';
 import { fileURLToPath } from 'node:url';
-import { holdDispatcherLock } from './hermes-locks.mjs';
 import { summary } from './board/cli.mjs';
 import { notifyChats } from './board/dispatch.mjs';
 import { workflowEntry } from './board/workflow.mjs';
@@ -38,9 +36,6 @@ export async function startBoard({ root, orchestration = null, log = () => {}, a
   let entry;
   try { entry = workflowEntry(orchestration); } catch (error) { log('warn', `board: not dispatched: ${error.message}`); return null; }
   if (!holds(entry.workflow.dispatch.serve ?? null, { params: entry.params })) { log('info', 'board: not dispatched (the board workflow\'s dispatch.serve)'); return null; }
-  let lock;
-  try { lock = await holdDispatcherLock(root); }
-  catch (error) { log('info', `board: not dispatched: ${error.message}`); return null; }
   const seconds = Math.max(Number(evaluate(entry.workflow.dispatch.tick, { params: entry.params })) || 60, 1);
   let running = false;
   let stopped = false;
@@ -78,6 +73,6 @@ export async function startBoard({ root, orchestration = null, log = () => {}, a
   log('info', `board: dispatching every ${seconds}s, each card as its profile's worker says`);
   return {
     tick,
-    stop() { stopped = true; clearTimeout(first); clearInterval(timer); lock.release(); },
+    stop() { stopped = true; clearTimeout(first); clearInterval(timer); },
   };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@volter/supercode-orchestrator",
-  "version": "0.4.82",
+  "version": "0.4.84",
   "type": "module",
   "description": "The orchestrator runtime over the Volter Harness ontology: one typed operational model whose folder is its serialization, read and written through the harness orchestration doors (docs/ORCHESTRATOR-IR.md)",
   "exports": {