@twinsearth/w2m-dsh-plugin 1.2.6 → 1.3.0

package/src/agent/agent.mjs

@@ -23,8 +23,9 @@
 
 import { Readable } from 'node:stream';
 import { access, constants as fsConstants } from 'node:fs/promises';
-import { existsSync, mkdirSync, renameSync, unlinkSync, writeFileSync } from 'node:fs';
+import { existsSync, mkdirSync, mkdtempSync, renameSync, unlinkSync, writeFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
+import { tmpdir } from 'node:os';
 import { randomUUID , sign as signBytes } from 'node:crypto';
 
 import {
@@ -3027,6 +3028,58 @@ export function createAgent(options) {
       // never allowed to ask for, and the plugin's pre-flight refuses the dispatch as a whole for the
       // same reason. The refusal names the stage index, because "this chain is not allowed" is not
       // actionable without knowing which stage.
+      /**
+       * v1.2.10: a reduce task carries the artifacts it consumes, and the machine fetches them BEFORE the reduce command runs.
+       *
+       * The list is the caller's completeness decision (the relay validated its shape, never its truth), and the bytes are
+       * content-addressed, so the fetch verifies nothing less than the hash already did. A failed fetch refuses the whole task:
+       * a reduce that ran on half the inputs would produce a confident wrong answer, which is the failure this feature exists to
+       * make impossible.
+       */
+      let reduceEnv = null;
+      if (Array.isArray(offer.reduce_inputs) && offer.reduce_inputs.length > 0) {
+        try {
+          const inputDir = mkdtempSync(join(tmpdir(), 'w2m-reduce-'));
+          const manifest = [];
+          for (const artifact of offer.reduce_inputs) {
+            const response = await fetch(rabbitUrl + '/v1/fleet/blobs/' + encodeURIComponent(artifact.sha256), {
+              headers: token() ? { authorization: 'Bearer ' + token() } : {},
+            });
+            if (response.status !== 200) {
+              throw new Error('blob ' + artifact.sha256 + ' -> HTTP ' + response.status);
+            }
+            const body = Buffer.from(await response.arrayBuffer());
+            const file = join(inputDir, String(artifact.machine_id).replace(/[^A-Za-z0-9._-]/g, '_') + '.bin');
+            writeFileSync(file, body);
+            manifest.push({ machine_id: artifact.machine_id, sha256: artifact.sha256, bytes: body.length, file });
+          }
+          const manifestPath = join(inputDir, 'manifest.json');
+          writeFileSync(manifestPath, JSON.stringify({ reduce_of: offer.reduce_of ?? null, inputs: manifest }, null, 2));
+          reduceEnv = {
+            W2M_REDUCE_INPUTS_DIR: inputDir,
+            W2M_REDUCE_MANIFEST: manifestPath,
+            W2M_REDUCE_OF: String(offer.reduce_of ?? ''),
+          };
+        } catch (error) {
+          log('warn', offer.task_id + ': reduce input fetch failed', { error: error?.message ?? String(error) });
+          await finish({
+            offer,
+            cwdRel,
+            commandArgv,
+            anchorsBefore,
+            anchorsAfter: null,
+            execution: null,
+            status: 'refused',
+            refusalReason: 'REDUCE_INPUT_FETCH_FAILED',
+            warnings,
+            startedAt,
+            endedAt: new Date().toISOString(),
+            protocolViolations: ['reduce inputs could not be fetched: ' + (error?.message ?? String(error))],
+          });
+          return;
+        }
+      }
+
       for (const stage of plannedStages) {
         const stageAllowed = matchAllowedCommand(stage.command_argv, allowedCommands);
         if (stageAllowed.allowed) continue;
@@ -3120,6 +3173,7 @@ export function createAgent(options) {
           W2M_MODE: String(offer.mode ?? ''),
           W2M_MACHINE_ID: String(offer.machine_id ?? ''),
           W2M_MACHINE_NAME: String(offer.machine_name ?? ''),
+          ...(reduceEnv ?? {}),
         };
         const ran = await runArgv(stage.command_argv, {
           cwd: stageCwdOutcome.cwd,
@@ -3215,6 +3269,28 @@ export function createAgent(options) {
         }
       }
 
+      /**
+       * v1.2.7: a plain command's large stdout is a stage artifact too.
+       *
+       * The blob path used to live only in the pipeline loop, so a split/mapreduce map result was always inlined and a
+       * reduce plan could never see the data it reduces -- the missing-shard check was unreachable from a real task. A
+       * command that prints more than the inline cap now uploads its stdout exactly as a stage does. Best effort here: a
+       * failed upload must not turn a successful command into a failure, so the envelope stays inline and says why.
+       */
+      if (isPipeline !== true && execution !== null && lastBlobSha256 === null) {
+        const plainPlan = planStageOutput({ machineId: identity.machine_id, stageIndex: 0, stdoutBytes: execution.stdout_bytes });
+        if (plainPlan.ok === true && plainPlan.mode === 'blob') {
+          if (execution.truncated_stdout === true && !warnings.includes('STDOUT_TRUNCATED')) warnings.push('STDOUT_TRUNCATED');
+          const uploaded = await putBlobBytes(execution.stdout_sha256, execution.stdout);
+          if (uploaded.ok === true) {
+            lastBlobSha256 = uploaded.sha256;
+          } else {
+            log('warn', offer.task_id + ': plain stdout blob upload failed', { code: uploaded.code });
+            warnings.push('BLOB_UPLOAD_FAILED');
+          }
+        }
+      }
+
       heartbeat.setPhase('finalizing', 90);
       if (execution === null) {
         // Nothing ran at all (a refused first stage, or a budget already spent). `execution` stays

package/src/plugin/merge-hint.mjs

@@ -0,0 +1,35 @@
+/**
+ * The merge hint: what a reader of a shared memory document should do about a conflict.
+ *
+ * A 409 from the memory CAS already carries `behind_by`, `unread_records` and a sample. The reader still has to decide what
+ * "behind by 3" means for the thing they are about to write. This turns those numbers into a named next step -- re-read, compare,
+ * then append with the current revision -- and never merges automatically: an automatic merge would write records the caller has
+ * not seen. Pure: it reads the refusal the relay already sent.
+ */
+export const MERGE_UP_TO_DATE = 'up-to-date';
+export const MERGE_RE_READ = 're-read-then-append';
+
+/**
+ * @param {{conflict?: object|null}} input `conflict` is the 409 detail: { behind_by, unread_records, sample }.
+ * @returns {{state: string, behind_by: number, unread: number, sample_count: number, next: string}}
+ */
+export function renderMergeHint(input = {}) {
+  const conflict = input.conflict ?? null;
+  if (conflict === null || typeof conflict !== 'object') {
+    return { state: MERGE_UP_TO_DATE, behind_by: 0, unread: 0, sample_count: 0, next: 'append normally' };
+  }
+  const behindBy = Number.isInteger(conflict.behind_by) && conflict.behind_by > 0 ? conflict.behind_by : 0;
+  const unread = Array.isArray(conflict.unread_records) ? conflict.unread_records.length
+    : (Number.isInteger(conflict.unread) && conflict.unread > 0 ? conflict.unread : 0);
+  const sample = Array.isArray(conflict.sample) ? conflict.sample : [];
+  if (behindBy === 0 && unread === 0) {
+    return { state: MERGE_UP_TO_DATE, behind_by: 0, unread: 0, sample_count: 0, next: 'append normally' };
+  }
+  return {
+    state: MERGE_RE_READ,
+    behind_by: behindBy,
+    unread,
+    sample_count: sample.length,
+    next: 're-read the document, compare the records listed here, then append with the current revision -- do not merge unseen records',
+  };
+}

package/src/plugin/task-graph.mjs

@@ -1,147 +1,153 @@
-/**
- * The dependency graph of a fleet's tasks, as pure functions.
- *
- * ## Why this is separate from the relay's own validation
- *
- * The relay refuses a graph it cannot store (unknown prerequisite, cycle, a self-edge), but that is one call at one moment.
- * A reader looking at a task board asks a different question: of the tasks I can see, which are READY, which are WAITING on
- * something that is still running, and which are BLOCKED because a prerequisite failed or was cancelled? That question
- * depends on the tasks' current states, so it is answered here, over whatever list the reader has.
- *
- * ## The rule that makes "blocked" honest
- *
- * A prerequisite that is missing from the visible list is treated as WAITING, never as ready: the list may be truncated by a
- * limit, and inferring "no dependency" from "not in this page" is how a board would show work as available that is not.
- */
-export const TASK_GRAPH_REFUSAL = Object.freeze({
-  UNKNOWN_NODE: 'GRAPH_UNKNOWN_NODE',
-  CYCLE: 'GRAPH_CYCLE',
-  BAD_NODE: 'GRAPH_BAD_NODE',
-});
-
-const DONE = new Set(['completed', 'done']);
-const FAILED = new Set(['failed', 'cancelled', 'refused']);
-// States that mean the task is already in flight. It is not 'ready': ready means work that can be STARTED.
-const ACTIVE = new Set(['running', 'queued', 'offered', 'preparing', 'finalizing']);
-
-/**
- * @param {{nodes?: Array<{task_id?: string, depends_on?: string[]}>}} input
- * @returns {{ok: true, nodes: Map<string, string[]>} | {ok: false, code: string, detail: object}}
- */
-export function buildGraph({ nodes = [] } = {}) {
-  if (!Array.isArray(nodes)) return { ok: false, code: TASK_GRAPH_REFUSAL.BAD_NODE, detail: {} };
-  const map = new Map();
-  for (const node of nodes) {
-    const id = typeof node?.task_id === 'string' && node.task_id !== '' ? node.task_id : null;
-    if (id === null) return { ok: false, code: TASK_GRAPH_REFUSAL.BAD_NODE, detail: { node } };
-    const deps = Array.isArray(node.depends_on) ? node.depends_on.filter((dep) => typeof dep === 'string' && dep !== '') : [];
-    map.set(id, deps);
-  }
-  return { ok: true, nodes: map };
-}
-
-/**
- * A stable topological order, or the cycle that prevents one.
- *
- * Ties are broken by id so the same graph renders in the same order on every machine -- a board that reshuffles between two
- * reads looks like it is changing.
- *
- * @param {Map<string, string[]>} graph
- * @returns {{ok: true, order: string[]} | {ok: false, code: string, detail: {cycle: string[]}}}
- */
-export function topoOrder(graph) {
-  const indegree = new Map();
-  const dependents = new Map();
-  for (const id of graph.keys()) {
-    indegree.set(id, 0);
-    dependents.set(id, []);
-  }
-  for (const [id, deps] of graph.entries()) {
-    for (const dep of deps) {
-      // A prerequisite OUTSIDE this page is not an edge in it. The relay refuses an unknown prerequisite when the task is
-      // created, so a missing one here means the reader's list is truncated -- and treating that as a broken graph would
-      // make an ordinary limited read report a cycle.
-      if (!graph.has(dep)) continue;
-      indegree.set(id, (indegree.get(id) ?? 0) + 1);
-      if (!dependents.has(dep)) dependents.set(dep, []);
-      dependents.get(dep).push(id);
-    }
-  }
-  const ready = [...indegree.entries()].filter(([, count]) => count === 0).map(([id]) => id).sort();
-  const order = [];
-  while (ready.length > 0) {
-    const id = ready.shift();
-    order.push(id);
-    for (const next of (dependents.get(id) ?? []).slice().sort()) {
-      indegree.set(next, indegree.get(next) - 1);
-      if (indegree.get(next) === 0) { ready.push(next); ready.sort(); }
-    }
-  }
-  if (order.length !== graph.size) {
-    const remaining = [...graph.keys()].filter((id) => !order.includes(id));
-    return { ok: false, code: TASK_GRAPH_REFUSAL.CYCLE, detail: { cycle: remaining } };
-  }
-  return { ok: true, order };
-}
-
-/**
- * Classify one task against the states of the tasks a reader can see.
- *
- * @param {{task_id: string, depends_on?: string[], status?: string, statuses?: Map<string, string>}} input
- * @returns {'ready'|'waiting'|'blocked'|'done'|'active'|'unknown'}
- */
-export function classifyTask({ task_id, depends_on = [], status = null, statuses = new Map() } = {}) {
-  if (typeof task_id !== 'string' || task_id === '') return 'unknown';
-  if (typeof status === 'string' && DONE.has(status)) return 'done';
-  if (typeof status === 'string' && FAILED.has(status)) return 'blocked';
-  let waiting = false;
-  for (const dep of depends_on) {
-    const depStatus = statuses.get(dep);
-    if (depStatus === undefined) { waiting = true; continue; }
-    if (FAILED.has(depStatus)) return 'blocked';
-    if (!DONE.has(depStatus)) waiting = true;
-  }
-  if (waiting) return 'waiting';
-  // In flight ONLY once its prerequisites are satisfied: a queued task behind a failed prerequisite is blocked, not active.
-  if (typeof status === 'string' && ACTIVE.has(status)) return 'active';
-  return 'ready';
-}
-
-/**
- * The whole board, in one pass: every visible task classified, and the readies in dependency order.
- *
- * @param {{tasks?: Array<{task_id?: string, depends_on?: string[], status?: string}>}} input
- * @returns {{ok: true, by_task: Record<string, string>, ready: string[], waiting: string[], blocked: string[], done: string[], active: string[], order: string[]} |
- *           {ok: false, code: string, detail: object}}
- */
-export function buildBoard({ tasks = [] } = {}) {
-  const built = buildGraph({ nodes: tasks });
-  if (built.ok !== true) return built;
-  const statuses = new Map();
-  for (const task of tasks) {
-    const id = typeof task?.task_id === 'string' ? task.task_id : null;
-    if (id !== null) statuses.set(id, typeof task?.status === 'string' ? task.status : 'unknown');
-  }
-  const byTask = {};
-  const buckets = { ready: [], waiting: [], blocked: [], done: [], active: [] };
-  for (const [id, deps] of built.nodes.entries()) {
-    const kind = classifyTask({ task_id: id, depends_on: deps, status: statuses.get(id), statuses });
-    byTask[id] = kind;
-    if (kind === 'done') buckets.done.push(id);
-    else if (kind === 'active') buckets.active.push(id);
-    else if (kind === 'blocked') buckets.blocked.push(id);
-    else if (kind === 'waiting') buckets.waiting.push(id);
-    else buckets.ready.push(id);
-  }
-  const order = topoOrder(built.nodes);
-  return {
-    ok: true,
-    by_task: byTask,
-    ready: buckets.ready.sort(),
-    waiting: buckets.waiting.sort(),
-    blocked: buckets.blocked.sort(),
-    done: buckets.done.sort(),
-    active: buckets.active.sort(),
-    order: order.ok === true ? order.order : [],
-  };
-}
+/**
+ * The dependency graph of a fleet's tasks, as pure functions.
+ *
+ * ## Why this is separate from the relay's own validation
+ *
+ * The relay refuses a graph it cannot store (unknown prerequisite, cycle, a self-edge), but that is one call at one moment.
+ * A reader looking at a task board asks a different question: of the tasks I can see, which are READY, which are WAITING on
+ * something that is still running, and which are BLOCKED because a prerequisite failed or was cancelled? That question
+ * depends on the tasks' current states, so it is answered here, over whatever list the reader has.
+ *
+ * ## The rule that makes "blocked" honest
+ *
+ * A prerequisite that is missing from the visible list is treated as WAITING, never as ready: the list may be truncated by a
+ * limit, and inferring "no dependency" from "not in this page" is how a board would show work as available that is not.
+ */
+export const TASK_GRAPH_REFUSAL = Object.freeze({
+  UNKNOWN_NODE: 'GRAPH_UNKNOWN_NODE',
+  CYCLE: 'GRAPH_CYCLE',
+  BAD_NODE: 'GRAPH_BAD_NODE',
+});
+
+const DONE = new Set(['completed', 'done']);
+const FAILED = new Set(['failed', 'cancelled', 'refused']);
+// States that mean the task is already in flight. It is not 'ready': ready means work that can be STARTED.
+const ACTIVE = new Set(['running', 'queued', 'offered', 'preparing', 'finalizing', 'active']);
+/**
+ * v1.2.7: the relay's scheduler reports 'waiting' for a task whose prerequisites are unsettled. Reading that as "ready" was the
+ * exact board/scheduler disagreement this set closes.
+ */
+const WAITING = new Set(['waiting']);
+
+/**
+ * @param {{nodes?: Array<{task_id?: string, depends_on?: string[]}>}} input
+ * @returns {{ok: true, nodes: Map<string, string[]>} | {ok: false, code: string, detail: object}}
+ */
+export function buildGraph({ nodes = [] } = {}) {
+  if (!Array.isArray(nodes)) return { ok: false, code: TASK_GRAPH_REFUSAL.BAD_NODE, detail: {} };
+  const map = new Map();
+  for (const node of nodes) {
+    const id = typeof node?.task_id === 'string' && node.task_id !== '' ? node.task_id : null;
+    if (id === null) return { ok: false, code: TASK_GRAPH_REFUSAL.BAD_NODE, detail: { node } };
+    const deps = Array.isArray(node.depends_on) ? node.depends_on.filter((dep) => typeof dep === 'string' && dep !== '') : [];
+    map.set(id, deps);
+  }
+  return { ok: true, nodes: map };
+}
+
+/**
+ * A stable topological order, or the cycle that prevents one.
+ *
+ * Ties are broken by id so the same graph renders in the same order on every machine -- a board that reshuffles between two
+ * reads looks like it is changing.
+ *
+ * @param {Map<string, string[]>} graph
+ * @returns {{ok: true, order: string[]} | {ok: false, code: string, detail: {cycle: string[]}}}
+ */
+export function topoOrder(graph) {
+  const indegree = new Map();
+  const dependents = new Map();
+  for (const id of graph.keys()) {
+    indegree.set(id, 0);
+    dependents.set(id, []);
+  }
+  for (const [id, deps] of graph.entries()) {
+    for (const dep of deps) {
+      // A prerequisite OUTSIDE this page is not an edge in it. The relay refuses an unknown prerequisite when the task is
+      // created, so a missing one here means the reader's list is truncated -- and treating that as a broken graph would
+      // make an ordinary limited read report a cycle.
+      if (!graph.has(dep)) continue;
+      indegree.set(id, (indegree.get(id) ?? 0) + 1);
+      if (!dependents.has(dep)) dependents.set(dep, []);
+      dependents.get(dep).push(id);
+    }
+  }
+  const ready = [...indegree.entries()].filter(([, count]) => count === 0).map(([id]) => id).sort();
+  const order = [];
+  while (ready.length > 0) {
+    const id = ready.shift();
+    order.push(id);
+    for (const next of (dependents.get(id) ?? []).slice().sort()) {
+      indegree.set(next, indegree.get(next) - 1);
+      if (indegree.get(next) === 0) { ready.push(next); ready.sort(); }
+    }
+  }
+  if (order.length !== graph.size) {
+    const remaining = [...graph.keys()].filter((id) => !order.includes(id));
+    return { ok: false, code: TASK_GRAPH_REFUSAL.CYCLE, detail: { cycle: remaining } };
+  }
+  return { ok: true, order };
+}
+
+/**
+ * Classify one task against the states of the tasks a reader can see.
+ *
+ * @param {{task_id: string, depends_on?: string[], status?: string, statuses?: Map<string, string>}} input
+ * @returns {'ready'|'waiting'|'blocked'|'done'|'active'|'unknown'}
+ */
+export function classifyTask({ task_id, depends_on = [], status = null, statuses = new Map() } = {}) {
+  if (typeof task_id !== 'string' || task_id === '') return 'unknown';
+  if (typeof status === 'string' && DONE.has(status)) return 'done';
+  if (typeof status === 'string' && FAILED.has(status)) return 'blocked';
+  let waiting = false;
+  for (const dep of depends_on) {
+    const depStatus = statuses.get(dep);
+    if (depStatus === undefined) { waiting = true; continue; }
+    if (FAILED.has(depStatus)) return 'blocked';
+    if (!DONE.has(depStatus)) waiting = true;
+  }
+  if (waiting) return 'waiting';
+  // In flight ONLY once its prerequisites are satisfied: a queued task behind a failed prerequisite is blocked, not active.
+  if (typeof status === 'string' && WAITING.has(status)) return 'waiting';
+  if (typeof status === 'string' && ACTIVE.has(status)) return 'active';
+  return 'ready';
+}
+
+/**
+ * The whole board, in one pass: every visible task classified, and the readies in dependency order.
+ *
+ * @param {{tasks?: Array<{task_id?: string, depends_on?: string[], status?: string}>}} input
+ * @returns {{ok: true, by_task: Record<string, string>, ready: string[], waiting: string[], blocked: string[], done: string[], active: string[], order: string[]} |
+ *           {ok: false, code: string, detail: object}}
+ */
+export function buildBoard({ tasks = [] } = {}) {
+  const built = buildGraph({ nodes: tasks });
+  if (built.ok !== true) return built;
+  const statuses = new Map();
+  for (const task of tasks) {
+    const id = typeof task?.task_id === 'string' ? task.task_id : null;
+    if (id !== null) statuses.set(id, typeof task?.status === 'string' ? task.status : 'unknown');
+  }
+  const byTask = {};
+  const buckets = { ready: [], waiting: [], blocked: [], done: [], active: [] };
+  for (const [id, deps] of built.nodes.entries()) {
+    const kind = classifyTask({ task_id: id, depends_on: deps, status: statuses.get(id), statuses });
+    byTask[id] = kind;
+    if (kind === 'done') buckets.done.push(id);
+    else if (kind === 'active') buckets.active.push(id);
+    else if (kind === 'blocked') buckets.blocked.push(id);
+    else if (kind === 'waiting') buckets.waiting.push(id);
+    else buckets.ready.push(id);
+  }
+  const order = topoOrder(built.nodes);
+  return {
+    ok: true,
+    by_task: byTask,
+    ready: buckets.ready.sort(),
+    waiting: buckets.waiting.sort(),
+    blocked: buckets.blocked.sort(),
+    done: buckets.done.sort(),
+    active: buckets.active.sort(),
+    order: order.ok === true ? order.order : [],
+  };
+}