brainchain-worker 0.12.2 → 0.13.0

package/dist/state.js

@@ -1,5 +1,6 @@
-import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
-import { homedir } from "node:os";
+import { randomBytes } from "node:crypto";
+import { closeSync, fsyncSync, mkdirSync, openSync, readFileSync, readlinkSync, renameSync, rmSync, statSync, utimesSync, writeFileSync } from "node:fs";
+import { homedir, hostname } from "node:os";
 import { dirname, join } from "node:path";
 export const stateDir = () => join(homedir(), ".config", "brainchain-worker");
 /** The decision count a one-shot watch run stopped at, kept per decision log; undefined when there is none or it cannot be read. */
@@ -15,8 +16,10 @@ export function readWatched(log, dir = stateDir()) {
 export function writeWatched(log, seen, dir = stateDir()) {
     mkdirSync(dir, { recursive: true, mode: 0o700 });
     const file = join(dir, `watch.${log.toLowerCase()}.json`);
-    writeFileSync(`${file}.tmp`, JSON.stringify({ seen: seen.toString() }), { mode: 0o600 });
-    renameSync(`${file}.tmp`, file);
+    // Each write its own temp file: runs from cron overlap, and nothing else keeps them apart.
+    const temp = `${file}.${process.pid}.${randomBytes(4).toString("hex")}.tmp`;
+    writeFileSync(temp, JSON.stringify({ seen: seen.toString() }), { mode: 0o600 });
+    renameSync(temp, file);
 }
 const empty = (board) => ({ board: board.toLowerCase(), tasks: {}, done: [] });
 export function memoryStore(board = "") {
@@ -101,6 +104,26 @@ export function fileStore(file, board, warn = console.warn) {
         },
     };
 }
+/** How often a run touches its lock. */
+const LOCK_BEAT_MS = 5_000;
+/** With no touch for this long a lock was left behind, whatever process now has its pid. */
+const LOCK_STALE_MS = 15_000;
+/** How long a lock from another PID namespace is watched for a touch, a little over one beat. */
+const LOCK_WATCH_MS = 7_000;
+/** A takeover takes a few system calls, so one marker older than this was left by a crash. */
+const TAKEOVER_STALE_MS = 5_000;
+/**
+ * Where a pid means something. Containers sharing a config folder each have
+ * their own numbering, so a holder's pid says nothing outside its own space.
+ */
+function pidSpace() {
+    try {
+        return `${readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim()}/${readlinkSync("/proc/self/ns/pid")}`;
+    }
+    catch {
+        return hostname();
+    }
+}
 function alive(pid) {
     // A container restarts its process with the same pid, so our own pid in the file is a stale lock.
     if (!Number.isInteger(pid) || pid <= 0 || pid === process.pid)
@@ -113,28 +136,132 @@ function alive(pid) {
         return error.code === "EPERM";
     }
 }
-const holder = (file) => (existsSync(file) ? Number(readFileSync(file, "utf8").trim()) : 0);
-const held = (pid, file) => new Error(`Another run (pid ${pid}) is already working for this worker. Stop it before starting another; ${file} holds its lock.`);
-/** Takes the lock in `file` for this process until it exits; refuses while a live process holds it. */
-export function lock(file) {
+const sleep = (ms) => void Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+function readIfThere(file) {
+    try {
+        return readFileSync(file, "utf8");
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return undefined;
+        throw error;
+    }
+}
+function touchedAt(file) {
+    try {
+        return statSync(file).mtimeMs;
+    }
+    catch {
+        return undefined;
+    }
+}
+/** Who a lock's text names. A bare pid is a lock from 0.12 or earlier; text that is neither is a lock still being written. */
+function holderOf(text) {
+    try {
+        const value = JSON.parse(text);
+        if (typeof value === "number")
+            return { pid: value };
+        if (isObject(value) && typeof value.pid === "number" && typeof value.space === "string")
+            return { pid: value.pid, space: value.space };
+    }
+    catch {
+        // Handled below.
+    }
+    return { pid: 0, space: "" };
+}
+/** Whether the run that wrote a lock is still going. */
+function isHeld(file, { pid, space }, watchMs) {
+    if (space === undefined)
+        return alive(pid);
+    const touched = touchedAt(file);
+    if (touched === undefined || Date.now() - touched >= LOCK_STALE_MS)
+        return false;
+    if (pid === 0)
+        return true;
+    if (space === pidSpace())
+        return alive(pid);
+    // A recent touch can be a run that just died, and a restarted container is one more pid namespace: a live run touches its lock again within a beat.
+    sleep(watchMs);
+    return touchedAt(file) !== touched;
+}
+const held = (pid, space, file) => new Error(`Another run (pid ${pid}${space === undefined || space === pidSpace() ? "" : " in another container or PID namespace"}) is already working for this worker. ` +
+    `Stop it before starting another; ${file} holds its lock.`);
+/**
+ * Replaces a lock left behind. The marker lets one start at a time judge and
+ * replace it, since removing the file another start has just written would give
+ * the lock to both.
+ */
+function takeOver(file, mine, judged) {
+    const marker = `${file}.takeover`;
+    try {
+        writeFileSync(marker, "", { flag: "wx", mode: 0o600 });
+    }
+    catch (error) {
+        if (error.code !== "EEXIST")
+            throw error;
+        const at = touchedAt(marker);
+        if (at !== undefined && Date.now() - at >= TAKEOVER_STALE_MS)
+            rmSync(marker, { force: true });
+        sleep(20);
+        return false;
+    }
+    try {
+        if (readIfThere(file) !== judged)
+            return false;
+        const temp = `${file}.${process.pid}.${randomBytes(4).toString("hex")}.tmp`;
+        writeFileSync(temp, mine, { mode: 0o600 });
+        renameSync(temp, file);
+        return true;
+    }
+    finally {
+        rmSync(marker, { force: true });
+    }
+}
+/**
+ * Takes the lock in `file` for this process until it exits; refuses while another
+ * run holds it, in this PID namespace or another sharing the folder. The run
+ * touches the lock every few seconds, and `lost` is called if another took it
+ * over, as can happen to a run frozen for longer than a lock lasts.
+ */
+export function lock(file, lost = () => process.exit(1), watchMs = LOCK_WATCH_MS) {
     mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
+    const mine = JSON.stringify({ pid: process.pid, space: pidSpace(), run: randomBytes(4).toString("hex") });
     for (;;) {
         try {
-            writeFileSync(file, String(process.pid), { flag: "wx", mode: 0o600 });
+            writeFileSync(file, mine, { flag: "wx", mode: 0o600 });
             break;
         }
         catch (error) {
             if (error.code !== "EEXIST")
                 throw error;
         }
-        // The holder may have released it since the write failed; then the next try takes it.
-        const pid = holder(file);
-        if (alive(pid))
-            throw held(pid, file);
-        rmSync(file, { force: true });
+        const text = readIfThere(file);
+        // The holder released it since the write failed, so the next try takes it.
+        if (text === undefined)
+            continue;
+        const holder = holderOf(text);
+        if (isHeld(file, holder, watchMs))
+            throw held(holder.pid, holder.space, file);
+        if (takeOver(file, mine, text))
+            break;
     }
+    const beat = setInterval(() => {
+        try {
+            if (readIfThere(file) !== mine) {
+                clearInterval(beat);
+                return lost();
+            }
+            const now = new Date();
+            utimesSync(file, now, now);
+        }
+        catch {
+            // A disk that fails for a moment is no reason to drop a task; the next beat tries again.
+        }
+    }, LOCK_BEAT_MS);
+    beat.unref();
     const release = () => {
-        if (existsSync(file) && readFileSync(file, "utf8") === String(process.pid))
+        clearInterval(beat);
+        if (readIfThere(file) === mine)
             rmSync(file, { force: true });
     };
     process.once("exit", release);
@@ -158,10 +285,11 @@ function migrate(old, file, board) {
     if (parse(text)?.board !== board.toLowerCase())
         return false;
     // A run from before the move may still be working on this network's tasks from the old file.
-    const pid = holder(`${old}.lock`);
+    const left = readIfThere(`${old}.lock`);
+    const pid = left === undefined ? 0 : holderOf(left).pid;
     if (alive(pid))
-        throw held(pid, `${old}.lock`);
-    if (existsSync(`${file}.json`))
+        throw held(pid, undefined, `${old}.lock`);
+    if (readIfThere(`${file}.json`) !== undefined)
         return false;
     // Removed rather than moved: this run already holds the new lock.
     rmSync(`${old}.lock`, { force: true });
@@ -176,7 +304,10 @@ function migrate(old, file, board) {
 export function openStore(address, network, log = console.log, dir = stateDir()) {
     const old = join(dir, address);
     const file = `${old}.${network.id}`;
-    lock(`${file}.lock`);
+    lock(`${file}.lock`, () => {
+        log(`another run took over ${file}.lock, so this one stops`);
+        process.exit(1);
+    });
     if (migrate(old, file, network.contracts.taskBoard))
         log(`moved ${old}.json to ${file}.json`);
     return fileStore(`${file}.json`, network.contracts.taskBoard, (message) => log(`warning: ${message}`));

package/dist/watch.js

@@ -81,6 +81,7 @@ export function createWatchChain(network, account) {
         decisions: () => client.readContract({ address: network.contracts.log, abi: logAbi, functionName: "count" }),
         ...(gate
             ? {
+                watcher: () => client.readContract({ address: gate, abi: gateAbi, functionName: "watcher" }),
                 async veto(changeId, why) {
                     const watcher = await client.readContract({ address: gate, abi: gateAbi, functionName: "watcher" });
                     if (watcher.toLowerCase() !== account.address.toLowerCase())
@@ -93,12 +94,29 @@ export function createWatchChain(network, account) {
             : {}),
     };
 }
+/**
+ * Refuses a --veto run whose key is not the gate's watcher, which otherwise
+ * shows only when a veto is due. A read that fails does not stop the watch: a
+ * round outlasts a bad RPC, and each veto reads the watcher again.
+ */
+export async function requireWatcher(chain, log) {
+    const watcher = await chain.watcher?.().catch((error) => {
+        log(`could not read the gate's watcher (${reason(error)}); it is read again before any veto`);
+        return undefined;
+    });
+    if (watcher !== undefined && watcher.toLowerCase() !== chain.account.toLowerCase()) {
+        throw new Error(`This key cannot veto: the gate's watcher is ${watcher}, not ${chain.account}. Set BRAINCHAIN_WORKER_KEY to the watcher's key, or run watch without --veto.`);
+    }
+}
 /** The service's prefix on a report; the worker signs the prefix and the report's JSON text. */
 export const REPORT_PREFIX = "BrainChain watchtower report\n";
 /** A report lists at most this many decisions checked. */
 export const MAX_CHECKED = 200;
 /** The proof a record was fetched: the hash of its bytes followed by the worker's address, which nothing on chain gives away. */
 export const fetchProof = (bytes, worker) => keccak256(concat([bytes, toBytes(worker)]));
+/** The service answered and will not take this report, so sending it again would not help. */
+export class ReportRefused extends Error {
+}
 /**
  * Sends a round's re-verified decisions to the service, signed by the worker,
  * so a staked worker's watchtower counts toward its mark in hire questions.
@@ -123,6 +141,8 @@ export function createReporter(network, account, fetcher = fetch) {
             signal: AbortSignal.timeout(20_000),
         });
         const body = (await response.json().catch(() => ({})));
+        if (response.status >= 400 && response.status < 500)
+            throw new ReportRefused(body.error ?? `${response.status} from ${network.service}`);
         if (!response.ok)
             throw new Error(body.error ?? `${response.status} from ${network.service}`);
         return body.mark
@@ -141,7 +161,10 @@ export function createWatch(options) {
     let nextBlock = options.fromBlock ?? network.fromBlock;
     let seenDecision = options.seenDecision;
     const judged = new Set();
+    /** Vetoes due and not yet landed; each is tried every round until it lands or its change closes. */
+    const owed = new Map();
     let findings = 0;
+    let incomplete = false;
     async function changes() {
         const head = await chain.head();
         if (head < nextBlock)
@@ -153,36 +176,49 @@ export function createWatch(options) {
                 const key = change.changeId.toString();
                 if (judged.has(key))
                     continue;
-                judged.add(key);
                 const { name, broken } = judge(change.data);
+                // A failed read leaves the change unjudged and the block range unread, so the next round comes back to it.
+                const open = await chain.open(change.changeId);
+                judged.add(key);
+                const why = broken.join("; ");
+                if (!open) {
+                    log(`change ${change.changeId}: ${name} on ${change.target}${broken.length > 0 ? ` broke the constitution (${why})` : ""}; closed, it has run or was vetoed`);
+                    continue;
+                }
                 const runsAt = new Date(Number(change.eta) * 1000).toISOString().replace("T", " ").slice(0, 16);
                 if (broken.length === 0) {
                     log(`change ${change.changeId}: ${name} on ${change.target}, within the constitution, runs after ${runsAt} UTC`);
                     continue;
                 }
                 findings += 1;
-                const why = broken.join("; ");
                 log(`change ${change.changeId}: ${name} on ${change.target} breaks the constitution (${why}), runs after ${runsAt} UTC`);
-                if (!options.veto)
-                    continue;
-                if (!(await chain.open(change.changeId))) {
-                    log(`change ${change.changeId} is closed already, nothing to veto`);
-                    continue;
+                if (options.veto)
+                    owed.set(key, { id: change.changeId, why });
+            }
+        }
+        nextBlock = head + 1n;
+    }
+    async function vetoes() {
+        for (const [key, { id, why }] of owed) {
+            try {
+                if (!(await chain.open(id))) {
+                    owed.delete(key);
+                    log(`change ${id} is closed already, nothing to veto`);
                 }
-                if (!chain.veto) {
+                else if (!chain.veto) {
+                    owed.delete(key);
                     log(`cannot veto: ${network.name} has no guardian gate this worker can reach`);
-                    continue;
-                }
-                try {
-                    const hash = await chain.veto(change.changeId, `Breaks the constitution: ${why}`);
-                    log(`vetoed change ${change.changeId} through the guardian gate: ${hash}`);
                 }
-                catch (error) {
-                    log(`veto of change ${change.changeId} failed: ${reason(error)}`);
+                else {
+                    const hash = await chain.veto(id, `Breaks the constitution: ${why}`);
+                    owed.delete(key);
+                    log(`vetoed change ${id} through the guardian gate: ${hash}`);
                 }
             }
+            catch (error) {
+                log(`veto of change ${id} failed, tried again next round: ${reason(error)}`);
+            }
         }
-        nextBlock = head + 1n;
     }
     async function decisions() {
         if (!verify)
@@ -200,6 +236,7 @@ export function createWatch(options) {
         }
         const checked = [];
         const found = [];
+        let reached = seenDecision;
         for (let id = seenDecision + 1n; id <= count; id++) {
             try {
                 const { findings: wrong, record } = await verify(id);
@@ -213,27 +250,48 @@ export function createWatch(options) {
                     found.push({ id, checks: wrong });
                     log(`decision ${id} does not verify: ${wrong.join("; ")}`);
                 }
+                reached = id;
             }
             catch (error) {
-                log(`decision ${id} could not be checked: ${reason(error)}`);
+                // The place stays here, so this decision and the ones after it are checked next time.
+                log(`decision ${id} could not be checked, so it is tried again next round: ${reason(error)}`.replace(/\s*\n\s*/g, " "));
+                incomplete = true;
+                break;
             }
         }
-        seenDecision = count;
-        options.saw?.(count);
-        if (!options.report || (checked.length === 0 && found.length === 0))
-            return;
+        // Before the place is saved, so a save that fails does not lose the round's work.
+        if (options.report && (checked.length > 0 || found.length > 0)) {
+            try {
+                log(await options.report(checked, found));
+            }
+            catch (error) {
+                log(`report not taken: ${reason(error)}`);
+                // Only an answer from the service settles a report; these decisions are checked and sent again otherwise.
+                if (!(error instanceof ReportRefused)) {
+                    incomplete = true;
+                    reached = seenDecision;
+                }
+            }
+        }
+        seenDecision = reached;
         try {
-            log(await options.report(checked, found));
+            options.saw?.(reached);
         }
         catch (error) {
-            log(`report not taken: ${reason(error)}`);
+            log(`warning: could not save how far this got, so the next run checks these decisions again: ${reason(error)}`);
         }
     }
     return {
         async round() {
-            await changes();
+            incomplete = false;
+            try {
+                await changes();
+            }
+            finally {
+                await vetoes();
+            }
             await decisions();
-            return { findings };
+            return { findings, incomplete };
         },
     };
 }

package/dist/work.js

@@ -1,4 +1,5 @@
 import { spawn, spawnSync } from "node:child_process";
+import { readFileSync } from "node:fs";
 import { toBytes } from "viem";
 import { fetchBrief, PermanentError, reason, RECEIPT_MS } from "./chain.js";
 import { memoryStore } from "./state.js";
@@ -10,6 +11,8 @@ const OUTPUT_CAP = MAX_RESULT_BYTES * 2;
 const MARGIN_SEC = 300n;
 const RETRY_SEC = [60n, 240n, 960n];
 const DEFAULT_TIMEOUT_MS = 20 * 60_000;
+/** How long the pipes stay open after a command exits, for what it had already written. */
+const DRAIN_MS = 2_000;
 const ANSI = /\x1b(?:\[[0-?]*[ -\/]*[@-~]|\][^\x07\x1b]*(?:\x07|\x1b\\)|[@-Z\\-_])/g;
 export const stripAnsi = (text) => text.replace(ANSI, "");
 /** Text from strangers, made safe to print as one terminal line. */
@@ -133,6 +136,44 @@ function orphaned(pgid) {
     const own = ps(process.pid, "pgid");
     return own !== undefined && own !== String(pgid);
 }
+/** A Linux process's name and parent; undefined where /proc has no such process or there is no /proc. */
+function procStat(pid) {
+    try {
+        const stat = /^\d+ \((.*)\) \S (\d+) /s.exec(readFileSync(`/proc/${pid}/stat`, "utf8"));
+        return stat ? { comm: stat[1], ppid: Number(stat[2]) } : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/** The npm process above a worker it started: its parent, or its grandparent where the shell npm runs the command in stays between them. */
+function npmAbove() {
+    const parent = procStat(process.ppid);
+    return parent && /^(sh|dash)$/.test(parent.comm) ? parent.ppid : undefined;
+}
+/**
+ * Calls `lost` once when what npx put above this process dies. npx passes a stop
+ * to the shell it runs the worker in, and dash dies without passing it on, so
+ * the worker never hears it and would run on with nothing above it.
+ */
+export function watchStarter(lost, intervalMs = 1_000) {
+    const parent = process.ppid;
+    const npm = npmAbove();
+    const timer = setInterval(() => {
+        if (process.ppid === parent && (npm === undefined || exists(npm)))
+            return;
+        clearInterval(timer);
+        lost();
+    }, intervalMs);
+    timer.unref();
+}
+/** Whether a supervisor started the npx above this process, as PID 1, as a child of PID 1 or as a systemd service. A stop from it can reach no further than npx. */
+export function npxUnderSupervisor(env = process.env) {
+    if (procStat(process.ppid) === undefined)
+        return false;
+    const npm = npmAbove() ?? process.ppid;
+    return npm === 1 || procStat(npm)?.ppid === 1 || env.INVOCATION_ID !== undefined;
+}
 /**
  * Stops the command an earlier run left running when it died, such as on
  * SIGKILL, so a restart never works on the same task twice at once. It gets
@@ -200,22 +241,36 @@ export function shellCommand(command, graceMs = 10_000) {
                     }, graceMs);
                 };
                 const timer = setTimeout(() => stop(`The command ran past ${Math.round(timeoutMs / 1000)} seconds and was stopped`), timeoutMs);
+                let drain;
+                child.on("exit", () => {
+                    clearTimeout(timer);
+                    // A background process can hold the pipes open long after the command has exited, and close would wait for it.
+                    drain = setTimeout(() => {
+                        child.stdout.destroy();
+                        child.stderr.destroy();
+                    }, DRAIN_MS);
+                });
                 child.stdout.on("data", (chunk) => {
                     size += chunk.byteLength;
                     if (size <= OUTPUT_CAP)
                         return void chunks.push(chunk);
-                    stop(`The command printed more than ${OUTPUT_CAP} bytes and was stopped; at most ${MAX_RESULT_BYTES} are accepted`);
+                    stop(`The command printed too much and was stopped; at most ${MAX_RESULT_BYTES} bytes are accepted`);
                 });
                 child.on("error", (error) => {
                     clearTimeout(timer);
                     clearTimeout(force);
+                    clearTimeout(drain);
                     reject(error);
                 });
                 child.on("close", (code, signal) => {
                     clearTimeout(timer);
                     clearTimeout(force);
-                    if (pid !== undefined)
+                    clearTimeout(drain);
+                    if (pid !== undefined) {
+                        // The task is over, and nothing a stranger's brief started should outlive it.
+                        signalGroup(pid, "SIGKILL");
                         running.delete(pid);
+                    }
                     if (running.size === 0)
                         process.off("exit", kill);
                     if (stopped !== undefined)
@@ -364,7 +419,7 @@ export function createWork(options) {
                         delete attempt.tx;
                         store.save();
                     }
-                    if (now + MARGIN_SEC >= task.deadline) {
+                    if (attempt.result === undefined && now + MARGIN_SEC >= task.deadline) {
                         finish(key);
                         fail();
                         log(`task ${task.id}: too close to the deadline to deliver; the poster is refunded`);
@@ -387,6 +442,7 @@ export function createWork(options) {
                             throw new PermanentError("The result contains the worker's private key, so it was not uploaded. Check what the command can read");
                         }
                         const bytes = toBytes(output).byteLength;
+                        // Not permanent: a retry runs the command again, and a model's next answer can fit.
                         if (bytes > MAX_RESULT_BYTES)
                             throw new Error(`The result is ${bytes} bytes; at most ${MAX_RESULT_BYTES} are accepted`);
                         attempt.result = output;
@@ -419,7 +475,8 @@ export function createWork(options) {
                     const wait = RETRY_SEC[Math.min(attempt.failures - 1, RETRY_SEC.length - 1)];
                     // The attempt may have run for many minutes, so the retry is timed from now.
                     const now = await chain.now().catch(wall);
-                    if (now + wait + MARGIN_SEC >= task.deadline) {
+                    // A retry with the result in hand only has to deliver it, so the margin for running the command does not apply.
+                    if (now + wait + (attempt.result === undefined ? MARGIN_SEC : 0n) >= task.deadline) {
                         finish(key);
                         log(`task ${task.id}: failed (${why}); no time left to retry, so the poster is refunded`);
                         continue;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "brainchain-worker",
-  "version": "0.12.2",
+  "version": "0.13.0",
   "description": "Earn on BrainChain and propose changes to it: stake, apply, do the tasks Jev hires your agent for, and bond BRAIN to put a change to Jev.",
   "license": "MIT",
   "author": "BrainChain <hello@usebrainchain.com>",
@@ -28,7 +28,7 @@
     "prepack": "npm run build"
   },
   "dependencies": {
-    "brainchain-verify": "^0.3.1",
+    "brainchain-verify": "^0.4.0",
     "viem": "^2.0.0"
   },
   "devDependencies": {