@stravica-ai/digital-operator-pa 0.1.65 → 0.1.67

package/dist/engineering-pa-cli/src/pa/status-board.mjs

@@ -0,0 +1,259 @@
+// status-board.mjs - schema, agent resolution and atomic write for the
+// per-agent daily status board.
+//
+// The board lives at `agents/<agent>/workspace/status-board.json` in the
+// operator repo. The workspace Status page reads it (through
+// `/__api/status-board`) and `pa status set | patch | show` writes it. This
+// module is the single validator both sides use, so the page and the CLI
+// can never disagree about what a valid board is.
+//
+// Shape (camelCase keys):
+//
+//   {
+//     "updatedAt": "2026-10-22T09:25:00Z",   required, ISO timestamp
+//     "headline": "One sentence on today.",   required, string
+//     "onMe": [ { question (required), options?, since?, workstream?, ids?, link? } ],
+//     "groups": [ { label, items: [ { title (required), status, where, stage, note, ids, updatedAt, link? } ] } ],
+//     "done": [ same shape as an item, plus optional workstream ]
+//   }
+//
+// Legacy alias: boards written before the platform feature used the key
+// `onBaz` and the status value `onBaz`. `normaliseBoard` maps both to `onMe`
+// on input, so an existing board file loads unchanged; every write goes out
+// as `onMe`.
+//
+// Unknown keys are kept (the reference board carries `since` on an item).
+// An unknown status value is refused on write (`strictStatus`) and allowed
+// on read, where the page shows it as a grey chip so a typo stays visible.
+
+import { existsSync, readdirSync } from 'node:fs';
+import { mkdir, readFile, rename, unlink, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+
+import { readPreferencesSync } from '../../../engineering-pa-core/src/preferences.mjs';
+
+export const STATUS_VALUES = ['queued', 'running', 'review', 'blocked', 'onMe', 'done'];
+export const LEGACY_ON_ME_KEY = 'onBaz';
+export const BOARD_FILE = 'status-board.json';
+
+// Agent folder names: lowercase letters, digits and hyphens. Anything else
+// (dots, slashes) is refused so a query parameter can never walk out of
+// `agents/`.
+const AGENT_RE = /^[a-z0-9][a-z0-9-]*$/;
+
+export function isValidAgentName(name) {
+  return typeof name === 'string' && AGENT_RE.test(name);
+}
+
+/**
+ * Resolve the running PA's agent folder name. Order:
+ *   1. operator/config/preferences.json `paName`
+ *   2. the single `agents/*` folder holding a next-session.md
+ *      (`dave` when several do and one of them is `dave`)
+ *   3. `dave`, the documented default for deployments that predate
+ *      preferences
+ * The workspace's `/__api/next-session` uses the same function.
+ */
+export function resolveAgentName(repoRoot) {
+  try {
+    const prefs = readPreferencesSync(repoRoot);
+    if (prefs && typeof prefs.paName === 'string' && prefs.paName.length) return prefs.paName;
+  } catch { /* non-fatal */ }
+
+  const agentsDir = join(repoRoot, 'agents');
+  if (existsSync(agentsDir)) {
+    let entries;
+    try { entries = readdirSync(agentsDir, { withFileTypes: true }); } catch { entries = []; }
+    const hits = [];
+    for (const e of entries) {
+      if (e.isDirectory() && existsSync(join(agentsDir, e.name, 'next-session.md'))) hits.push(e.name);
+    }
+    if (hits.length === 1) return hits[0];
+    if (hits.length > 1 && hits.includes('dave')) return 'dave';
+  }
+  return 'dave';
+}
+
+/** Repo-relative path of an agent's board file. */
+export function boardRelPath(agent) {
+  return 'agents/' + agent + '/workspace/' + BOARD_FILE;
+}
+
+/** Absolute path of an agent's board file. */
+export function boardPath(repoRoot, agent) {
+  return join(repoRoot, 'agents', agent, 'workspace', BOARD_FILE);
+}
+
+/**
+ * Map the legacy `onBaz` key and status value to `onMe`. Returns a new
+ * object with key order preserved (the alias takes the old key's slot).
+ * Throws when a board carries both keys, since one would silently win.
+ */
+export function normaliseBoard(input) {
+  if (!isPlainObject(input)) return input;
+  if (LEGACY_ON_ME_KEY in input && 'onMe' in input) {
+    throw new BoardError(['board has both "onMe" and the legacy "onBaz"; keep one']);
+  }
+  const out = {};
+  for (const k of Object.keys(input)) {
+    if (k === LEGACY_ON_ME_KEY) out.onMe = input[k];
+    else out[k] = input[k];
+  }
+  if (Array.isArray(out.groups)) {
+    out.groups = out.groups.map((g) => (isPlainObject(g) && Array.isArray(g.items)
+      ? { ...g, items: g.items.map(normaliseStatus) }
+      : g));
+  }
+  if (Array.isArray(out.done)) out.done = out.done.map(normaliseStatus);
+  return out;
+}
+
+function normaliseStatus(it) {
+  if (isPlainObject(it) && it.status === LEGACY_ON_ME_KEY) return { ...it, status: 'onMe' };
+  return it;
+}
+
+export class BoardError extends Error {
+  constructor(problems) {
+    super('invalid status board: ' + problems.join('; '));
+    this.name = 'BoardError';
+    this.problems = problems;
+  }
+}
+
+/**
+ * Validate a board (after `normaliseBoard`). Returns a list of problems;
+ * an empty list means valid.
+ *
+ * @param {object} board
+ * @param {{ strictStatus?: boolean }} [opts] strictStatus refuses status
+ *   values outside STATUS_VALUES (write side). Off for the read side.
+ */
+export function validateBoard(board, opts = {}) {
+  const strictStatus = opts.strictStatus === true;
+  const problems = [];
+  if (!isPlainObject(board)) return ['board must be a JSON object'];
+
+  checkIso(board.updatedAt, 'updatedAt', true, problems);
+  if (typeof board.headline !== 'string') problems.push('headline is required and must be a string');
+  if (LEGACY_ON_ME_KEY in board) problems.push('legacy key "onBaz" must be normalised to "onMe"');
+
+  if (board.onMe !== undefined) {
+    if (!Array.isArray(board.onMe)) problems.push('onMe must be an array');
+    else board.onMe.forEach((a, i) => checkAsk(a, 'onMe[' + i + ']', problems));
+  }
+  if (board.groups !== undefined) {
+    if (!Array.isArray(board.groups)) problems.push('groups must be an array');
+    else board.groups.forEach((g, gi) => {
+      const at = 'groups[' + gi + ']';
+      if (!isPlainObject(g)) { problems.push(at + ' must be an object'); return; }
+      checkString(g.label, at + '.label', false, problems);
+      if (!Array.isArray(g.items)) { problems.push(at + '.items must be an array'); return; }
+      g.items.forEach((it, ii) => checkItem(it, at + '.items[' + ii + ']', strictStatus, problems));
+    });
+  }
+  if (board.done !== undefined) {
+    if (!Array.isArray(board.done)) problems.push('done must be an array');
+    else board.done.forEach((it, i) => checkItem(it, 'done[' + i + ']', strictStatus, problems));
+  }
+  return problems;
+}
+
+function checkItem(it, at, strictStatus, problems) {
+  if (!isPlainObject(it)) { problems.push(at + ' must be an object'); return; }
+  checkString(it.title, at + '.title', true, problems);
+  if (it.status !== undefined) {
+    if (typeof it.status !== 'string') problems.push(at + '.status must be a string');
+    else if (strictStatus && !STATUS_VALUES.includes(it.status)) {
+      problems.push(at + '.status "' + it.status + '" is not one of ' + STATUS_VALUES.join(', '));
+    }
+  }
+  for (const k of ['where', 'stage', 'note', 'link', 'workstream']) checkString(it[k], at + '.' + k, false, problems);
+  checkStringArray(it.ids, at + '.ids', problems);
+  checkIso(it.updatedAt, at + '.updatedAt', false, problems);
+  checkIso(it.since, at + '.since', false, problems);
+}
+
+function checkAsk(a, at, problems) {
+  if (!isPlainObject(a)) { problems.push(at + ' must be an object'); return; }
+  checkString(a.question, at + '.question', true, problems);
+  checkStringArray(a.options, at + '.options', problems);
+  checkStringArray(a.ids, at + '.ids', problems);
+  for (const k of ['workstream', 'link']) checkString(a[k], at + '.' + k, false, problems);
+  checkIso(a.since, at + '.since', false, problems);
+}
+
+function checkString(v, at, required, problems) {
+  if (v === undefined) {
+    if (required) problems.push(at + ' is required');
+    return;
+  }
+  if (typeof v !== 'string') problems.push(at + ' must be a string');
+  else if (required && !v.trim()) problems.push(at + ' must not be empty');
+}
+
+function checkStringArray(v, at, problems) {
+  if (v === undefined) return;
+  if (!Array.isArray(v) || v.some((x) => typeof x !== 'string')) problems.push(at + ' must be an array of strings');
+}
+
+function checkIso(v, at, required, problems) {
+  if (v === undefined) {
+    if (required) problems.push(at + ' is required');
+    return;
+  }
+  if (typeof v !== 'string' || Number.isNaN(Date.parse(v))) problems.push(at + ' must be an ISO timestamp');
+}
+
+function isPlainObject(v) {
+  return v !== null && typeof v === 'object' && !Array.isArray(v);
+}
+
+/**
+ * Parse board text, normalise the legacy alias and validate. Returns
+ * `{ board, problems }`; throws a SyntaxError on unparseable text.
+ */
+export function parseBoard(text, opts = {}) {
+  const raw = JSON.parse(text);
+  let board;
+  try {
+    board = normaliseBoard(raw);
+  } catch (err) {
+    if (err instanceof BoardError) return { board: raw, problems: err.problems };
+    throw err;
+  }
+  return { board, problems: validateBoard(board, opts) };
+}
+
+/** Read and parse an agent's board. Returns null when the file is absent. */
+export async function readBoard(file, opts = {}) {
+  let text;
+  try {
+    text = await readFile(file, 'utf8');
+  } catch (err) {
+    if (err.code === 'ENOENT') return null;
+    throw err;
+  }
+  return parseBoard(text, opts);
+}
+
+/**
+ * Validate (strict) and write a board atomically: a temp file beside the
+ * target, then rename over it, so a reader never sees a half-written file
+ * and a refused board never lands. 2-space indent, trailing newline.
+ */
+export async function writeBoardAtomic(file, board) {
+  const normalised = normaliseBoard(board);
+  const problems = validateBoard(normalised, { strictStatus: true });
+  if (problems.length) throw new BoardError(problems);
+  await mkdir(dirname(file), { recursive: true });
+  const tmp = file + '.tmp-' + process.pid + '-' + Date.now().toString(36) + Math.random().toString(36).slice(2, 8);
+  try {
+    await writeFile(tmp, JSON.stringify(normalised, null, 2) + '\n', 'utf8');
+    await rename(tmp, file);
+  } catch (err) {
+    try { await unlink(tmp); } catch { /* already gone */ }
+    throw err;
+  }
+  return normalised;
+}

package/dist/engineering-pa-cli/src/research/README.md

@@ -167,6 +167,26 @@ of `--list`/`--retry`/`--append` is given, an invalid `<sessionId>`, bad JSON, o
 invalid record; `2` if an explicitly named session does not exist, or `--append` was
 handed zero records; `3` disk read/write failure.
 
+### `sources`: merge into the source registry
+
+```
+research sources --merge <sessionId> [--json]
+```
+
+Reads a source registry on stdin (`{ "items": [...] }` or a bare array of items) and
+merges it into the session's `sources.json` (w-2026-10-19-dave-009). The registry
+never loses items: every existing item is kept in place, an item with a new `id` is
+appended, and an item whose `id` is already present only fills fields the existing
+one lacks. Merging the same set twice adds nothing, so every workflow writer (the
+registry checkpoint, the finaliser, deep's finaliser) sends the full set through this
+verb instead of overwriting the file. The write goes through the store chokepoint
+(`store.writeSources` -> `validateSources`).
+
+Exit: `0` merged (prints `merged N new source(s) into <id> (M total)`, or
+`{ added, total }` with `--json`); `1` no `--merge`, a missing or invalid
+`<sessionId>`, bad JSON, or an item that fails the sources schema (nothing written);
+`2` the session does not exist; `3` disk read/write failure.
+
 ### `prune` — delete old / matching sessions
 
 ```
@@ -198,10 +218,25 @@ Sets `manifest.status` to a terminal value and bumps `updatedAt`:
 - **`done`** — the session finalised successfully. A completed research run (light
   recipe or the `research-normal.js` workflow) calls it to move the session out of
   `draft`/`in-progress` so `list --status done` becomes meaningful.
-- **`aborted`** — the session was abandoned before a usable answer. The
-  `research-normal.js` workflow's `>10%`-unsourced guardrail (build-spec §8) stamps a
-  run `aborted` so it reads correctly in `list`/`show` instead of lingering as
-  `in-progress`.
+- **`aborted`**: the run stopped short of a finished answer. The
+  `research-normal.js` workflow stamps `aborted` itself on every abort path, so the
+  session reads correctly in `list`/`show` instead of lingering as `draft`
+  (w-2026-10-19-dave-009):
+  - **Early unsourced guard** (after Search+Extract, before Rank, Verify, Deepen,
+    Synthesize and Finalise): more than 25 percent of the extracted assertions on
+    sourced-class sub-questions could not be grounded in a quote. The registry
+    checkpoint has already merged every searcher source into `sources.json` and
+    appended every `gatedDropped` record to `gates.json`; the same agent runs
+    `mark aborted`. No synthesis ran, so there is no `answer.md`.
+  - **Late abort** (after synthesis): the synth receipts show more than 10 percent
+    unsourced claims on sourced-class sub-questions (build-spec §8), or a planned
+    sub-question has no finding (coverage gap). The finaliser still merges the
+    registry, writes a partial `answer.md` (headed `# Partial answer (run aborted)`,
+    unsourced assertions tagged `[UNSOURCED]`, a `## Unsourced claims` section),
+    round-trips `show --json`, and runs `mark aborted`. The work is never discarded.
+
+  Sub-questions the planner classed `claimClass: "analysis"` (allocation shapes,
+  trade-offs, recommendations) are left out of both guards.
 
 This is the one verb that mutates an existing session's status. Both transitions are
 **one-way**: a session already `done` or `aborted` is refused (exit `2`), the same
@@ -295,9 +330,9 @@ All camelCase.
 | File | Location | Written by | Purpose |
 |---|---|---|---|
 | `manifest.json` | session root | this tool (`run`) | session identity, mode, gate policy, status, budget, assumptions |
-| `plan.json` | session root | `run` (skeleton) → workflow (subQuestions) | dependency-classified decomposition, rounds, verification tier |
+| `plan.json` | session root | `run` (skeleton) → workflow (subQuestions) | dependency-classified decomposition (each sub-question may carry `claimClass`: `sourced`, the default, or `analysis`), rounds, verification tier |
 | `findings/sq-<id>.json` | `findings/` | **workers** | one finding per sub-question: summary, quotes, sources, confidence, votes |
-| `sources.json` | session root | workflow | aggregated source registry referenced by findings |
+| `sources.json` | session root | `run` (empty) → `sources --merge` (workflow registry checkpoint after Search, after a deepen round, and the finaliser) | aggregated source registry referenced by findings; merged, never overwritten |
 | `gates.json` | session root | `gates --append` / `gates --retry` | human-gate retry queue |
 
 Enum values follow the build-spec's own spelling: `gatePolicy` is
@@ -322,7 +357,7 @@ packages/cli/src/research/
     validate.mjs             # shared hand-rolled validation primitives (zero-dep)
     manifest.mjs plan.mjs findings.mjs sources.mjs gates.mjs
   commands/
-    run.mjs list.mjs show.mjs gates.mjs prune.mjs mark.mjs paths.mjs
+    run.mjs list.mjs show.mjs gates.mjs sources.mjs prune.mjs mark.mjs paths.mjs
   schema/
     *.schema.json            # documentation-grade JSON Schemas (camelCase)
   briefs/                    # dispatch briefs the /research skill workflow feeds to subagents

package/dist/engineering-pa-cli/src/research/cli.mjs

@@ -41,6 +41,11 @@ Usage:
       --append reads gate records (JSON array on stdin, or --record '<json>' for a
       single one) and appends them to the session's gates.json with fresh g-N ids.
 
+  research sources --merge <sessionId> [--json]
+      Merge a source registry ({ "items": [...] } or an array, on stdin) into the
+      session's sources.json. Existing items are never dropped or overwritten; new
+      ids are appended. Idempotent, so every workflow writer can send the full set.
+
   research prune [--older-than <days>] [--status <status>] [--dry-run]
       Delete sessions older than N days and/or matching a status. At least one
       filter is required. --dry-run prints what would be deleted without deleting.
@@ -92,6 +97,10 @@ export async function main(argv) {
         const { gatesCommand } = await import('./commands/gates.mjs');
         return (await gatesCommand(rest)) ?? 0;
       }
+      case 'sources': {
+        const { sourcesCommand } = await import('./commands/sources.mjs');
+        return (await sourcesCommand(rest)) ?? 0;
+      }
       case 'prune': {
         const { pruneCommand } = await import('./commands/prune.mjs');
         return (await pruneCommand(rest)) ?? 0;

package/dist/engineering-pa-cli/src/research/commands/sources.mjs

@@ -0,0 +1,64 @@
+// research sources --merge <sessionId> [--json]
+//
+// Reads a source registry from stdin ({ "items": [...] } or a bare array of items)
+// and merges it into the session's sources.json (w-2026-10-19-dave-009). Existing
+// items are never dropped or overwritten: new ids are appended, a known id only gains
+// fields it lacked. Merging the same set twice is a no-op, so the workflow's
+// persister and finaliser can both send the full registry. The write goes through
+// the store chokepoint (store.writeSources -> validateSources).
+import { parseArgs } from 'node:util';
+import { getDefaultStore } from '../store/index.mjs';
+import { mergeSources } from '../store/sources.mjs';
+
+function usage(msg) {
+  return Object.assign(new Error(`usage: research sources --merge <sessionId> [--json]${msg ? `\n${msg}` : ''}`), { exitCode: 1 });
+}
+function notFound(msg) {
+  return Object.assign(new Error(msg), { exitCode: 2 });
+}
+
+async function readAllStdin(stream = process.stdin) {
+  const chunks = [];
+  for await (const chunk of stream) chunks.push(typeof chunk === 'string' ? Buffer.from(chunk) : chunk);
+  return Buffer.concat(chunks).toString('utf8');
+}
+
+export async function sourcesCommand(argv, { store, stdout = process.stdout, readStdin = readAllStdin } = {}) {
+  let parsed;
+  try {
+    parsed = parseArgs({ args: argv, options: { merge: { type: 'boolean' }, json: { type: 'boolean' } }, allowPositionals: true, strict: true });
+  } catch (e) {
+    throw usage(e.message);
+  }
+  if (!parsed.values.merge) throw usage('--merge is required');
+  const id = parsed.positionals[0];
+  if (!id) throw usage('--merge requires a <sessionId>');
+  if (parsed.positionals.length > 1) throw usage('--merge takes exactly one <sessionId>');
+  if (!store) store = await getDefaultStore();
+  if (!store.isValidSessionId(id)) throw usage(`"${id}" is not a valid sessionId`);
+  if (!store.exists(id)) throw notFound(`session ${id} not found`);
+
+  const text = await readStdin();
+  let raw;
+  try {
+    raw = JSON.parse(text);
+  } catch (e) {
+    throw usage(`invalid JSON on stdin: ${e.message}`);
+  }
+  const incoming = Array.isArray(raw) ? { items: raw } : raw;
+  const existing = await store.readSources(id);
+  let merged;
+  try {
+    merged = mergeSources(existing, incoming, `session ${id} sources`);
+  } catch (e) {
+    // A malformed incoming registry is caller input: usage error, nothing written.
+    throw usage(e.message);
+  }
+  await store.writeSources(id, merged.sources);
+  if (parsed.values.json) {
+    stdout.write(JSON.stringify({ added: merged.added, total: merged.sources.items.length }, null, 2) + '\n');
+    return 0;
+  }
+  stdout.write(`merged ${merged.added} new source(s) into ${id} (${merged.sources.items.length} total)\n`);
+  return 0;
+}

package/dist/engineering-pa-cli/src/research/engine/deep.mjs

@@ -17,7 +17,7 @@ import { flattenClaims, rankAndCapPerSq } from './rank.mjs';
 import { adjudicate } from './verify.mjs';
 import { verifyAgent } from './agents.mjs';
 import { runDeepenRound, deepenStopReason } from './deepen.mjs';
-import { buildSourcesJson, buildGateRecords, bucketsFor, synthOrDegrade } from './finalise.mjs';
+import { buildSourcesJson, buildGateRecords, bucketsFor, synthOrDegrade, sourcesMergeStep } from './finalise.mjs';
 
 // Guaranteed floor of top claims kept per sub-question when the claim budget is capped
 // (deep-mode fix for SQ starvation). 1 = every planned sub-question keeps its single best
@@ -401,8 +401,7 @@ export async function runDeepFinalise(ctx, { sqResults, roundsJson, costFields,
     `You are the **finaliser** for deep-research session \`${ctx.sessionId}\` (output mode: \`${ctx.baseOutputMode}\`).`,
     `Do these in order; every JSON below is already validated by the workflow — write it VERBATIM.`,
     `1. ${gatesStep}`,
-    `2. Write this EXACT JSON to \`${ctx.sessionDir}/sources.json\`:`,
-    '```json', sourcesJson, '```',
+    sourcesMergeStep(ctx, sourcesJson, 2),
     `3. Write this EXACT JSON to \`${ctx.sessionDir}/rounds.json\` (the deepen audit trail):`,
     '```json', roundsJson, '```',
     `4. Compose cost.json. Take these workflow-measured fields VERBATIM:`,

package/dist/engineering-pa-cli/src/research/engine/deepen.mjs

@@ -37,6 +37,25 @@ export function deepenStopReason({ sufficient, round, roundCap, budgetRemaining,
   return null;
 }
 
+// Give every deepen proposal an id no live sub-question holds (w-2026-10-19-dave-009).
+// A reused id (deep round 2 proposing sq-d1 again, or a clash with a planner id) would
+// overwrite that sub-question's entry in sqResults, and its searcher sources with it,
+// so the registry built from sqResults would lose them. Clashing ids take the next free
+// sq-dN; fresh ids are kept. Pure.
+export function uniqueDeepenIds(proposals, liveIds) {
+  const taken = new Set(liveIds);
+  for (const p of proposals) if (!taken.has(p.id)) taken.add(p.id);
+  const seen = new Set(liveIds);
+  let n = 1;
+  return proposals.map((p) => {
+    if (!seen.has(p.id)) { seen.add(p.id); return p; }
+    while (taken.has(`sq-d${n}`)) n += 1;
+    const id = `sq-d${n}`;
+    taken.add(id); seen.add(id);
+    return { ...p, id };
+  });
+}
+
 // One gap-driven deepen round. Asks the deepener whether a real gap remains; if so,
 // searches+extracts the proposed facets, appends their claims to `state.retained`,
 // and re-verifies newly-central claims. Mutates state.{sqResults,urlBySourceId,
@@ -64,7 +83,8 @@ export async function runDeepenRound(ctx, state, { lenses = VERIFY_LENSES, quoru
   let deepenAdded = 0;
   let newSqs = [];
   if (!deepen.sufficient && (deepen.newSubQuestions || []).length) {
-    newSqs = deepen.newSubQuestions.slice(0, 4).map((s) => ({ ...s, kind: 'independent', dependsOn: [] }));
+    const liveIds = [...subQuestions.map((s) => s.id), ...sqResults.keys()];
+    newSqs = uniqueDeepenIds(deepen.newSubQuestions.slice(0, 4), liveIds).map((s) => ({ ...s, kind: 'independent', dependsOn: [] }));
     deepenAdded = newSqs.length;
     log(`deepen: gap round — ${deepenAdded} new sub-question(s)`);
     const deepPairs = await pipeline(