@stravica-ai/digital-operator-pa 0.1.0

package/dist/engineering-pa-core/src/distill/run.mjs

@@ -0,0 +1,230 @@
+// distill — orchestrator.
+//
+// Flow:
+//   1. resolvePrereqs          (repo root, paName, librarian def, claude on PATH)
+//   2. countStagedObservations
+//   3. invoke claude
+//   4. append runs.jsonl       (audit record)
+//
+// **Git is handled by core/tools/sync, not by distill.** Distill writes
+// promotion outputs into the working tree and stops; the per-host sync
+// tool (running every 30 minutes) picks them up, composes a commit body
+// from each cron tool's runs.jsonl, and does the pull / commit / push.
+//
+// Prior to 2026-05-11 this tool also did `git fetch + pull --rebase` as
+// a pre-flight, then `git add / commit / push` as a post-flight. That
+// design raced against the other cron tools (`wsd-ingest` every 15m,
+// `upstream-triage` every 30m) which were doing the same thing on their
+// own clocks — a 2-second window between `isTreeClean` and `pull --rebase`
+// was enough to trip `fail-rebase` and `fail-dirty-tree` audits 12 times
+// in late April. Stripping git from the cron writers and centralising it
+// in `core/tools/sync` is the structural fix.
+//
+// Exit codes (§6 tooling standard):
+//   0 ok | 1 usage | 2 prerequisite missing | 3 claude transport failure
+//
+// Audit-record contract (runs.jsonl):
+//   Every non-dry-run invocation appends exactly one JSON line. The record
+//   carries both `exitCode` (for scripts / cron) and `status` (semantic,
+//   for monitors). Status vocabulary is closed — consumers may match on
+//   prefixes (`noop-*` = healthy, `fail-*` = operational failure, `ok` =
+//   successful distillation). `reason` is a human-readable one-liner,
+//   omitted on `ok`. `log` is null on exits that happened before the
+//   per-run log file was created.
+
+import { appendFile, readFile } from 'node:fs/promises';
+import { basename, join } from 'node:path';
+
+import { resolvePrereqs } from './prereqs.mjs';
+import { countStagedObservations } from './staging.mjs';
+import { DISTILL_PROMPT, claudeArgs } from './prompt.mjs';
+
+function log(line, { stderr = process.stderr } = {}) {
+  stderr.write(`[distill] ${line}\n`);
+}
+
+function opError(msg, code = 2) {
+  const err = new Error(msg);
+  err.distillExitCode = code;
+  return err;
+}
+
+// Append a one-line audit record to runs.jsonl. Errors writing the
+// audit record are swallowed with a stderr note — losing the audit
+// line is less bad than masking the real exit code of the sweep.
+async function writeAudit(runsLog, record, stderr) {
+  try {
+    await appendFile(runsLog, JSON.stringify(record) + '\n');
+  } catch (e) {
+    log(`warning: failed to append audit record to ${runsLog}: ${e.message}`, { stderr });
+  }
+}
+
+// Composition joiner for the bundled-definition + operator-supplement
+// resolution outcome. Pure string building — the precedence statement is
+// part of the composed system prompt, so the librarian session knows the
+// supplement wins wherever it conflicts with the bundled text.
+export function composeLibrarianDef(bundledContent, supplementContent) {
+  return (
+    bundledContent.replace(/\s+$/, '') +
+    '\n\n---\n\n' +
+    '# Operator supplement\n\n' +
+    'Everything below this line is the operator\'s supplement to the bundled\n' +
+    'Librarian definition above. **Wherever the supplement conflicts with the\n' +
+    'bundled definition, the supplement takes precedence — follow the\n' +
+    'supplement.** Where the supplement is silent, the bundled definition\n' +
+    'applies unchanged.\n\n' +
+    supplementContent.replace(/\s+$/, '') +
+    '\n'
+  );
+}
+
+export async function runDistill({
+  dryRun = false,
+  runner,
+  prereqs,
+  stderr = process.stderr,
+} = {}) {
+  if (!runner) throw new Error('runDistill: runner is required');
+
+  // Step 1 — prerequisites.
+  const env = prereqs ?? (await resolvePrereqs());
+  const { repoRoot, paName, librarianDef, librarianSupplement, logDir, runsLog } = env;
+
+  // Start timing as soon as we have a commitment to run. Early exits
+  // (empty staging) still record durationSec — it will typically be 0
+  // but reflects time spent in preconditions.
+  const startedAtIso = runner.tsIso();
+  const startedAtMs = Date.now();
+
+  // Step 2 — count staged observations.
+  const stagingCount = await countStagedObservations(repoRoot);
+  if (stagingCount === 0) {
+    log('no staged observations to process — nothing to do', { stderr });
+    if (!dryRun) {
+      await writeAudit(runsLog, {
+        ts: startedAtIso,
+        status: 'noop-empty-staging',
+        exitCode: 0,
+        durationSec: Math.round((Date.now() - startedAtMs) / 1000),
+        stagingCount: 0,
+        log: null,
+        reason: 'no staged observations to process',
+      }, stderr);
+    }
+    return 0;
+  }
+  log(`found ${stagingCount} staged observation(s)`, { stderr });
+
+  // Step 3 — assemble & invoke.
+  //
+  // Two timestamp shapes for two jobs:
+  //   * `ts` — filename-safe (colons replaced with dashes) for the log
+  //     filename and session-name slug. Must not contain `:` because
+  //     filesystems/tools dislike it.
+  //   * `tsIso` — strict RFC-3339 / ISO-8601 for the `runs.jsonl` audit
+  //     field, so downstream consumers can `new Date(entry.ts)` without
+  //     getting NaN. The filename-safe form looks like ISO but is not
+  //     parseable — don't conflate the two.
+  const ts = runner.ts();
+  const tsIso = runner.tsIso();
+  const runLogPath = join(logDir, `${ts}.log`);
+  const sessionName = `distill-${isoCompactForSession(runner.now())}`;
+
+  // Assemble the librarian system prompt. Three shapes:
+  //   * no supplement resolved            -> the resolved file, byte-for-byte
+  //     (explicit opt, full operator override, or bundled alone);
+  //   * supplement resolved, has content  -> bundled + supplement composed
+  //     with the operator-wins precedence statement;
+  //   * supplement resolved, whitespace-only -> treated as absent. Appending
+  //     an empty supplement under an operator-wins banner would claim a
+  //     customisation that does not exist; the bundled definition ships alone
+  //     and the dry-run/plan output says so.
+  const librarianBase = await readFile(librarianDef, 'utf8');
+  let supplementContent = null;
+  if (librarianSupplement) {
+    const raw = await readFile(librarianSupplement, 'utf8');
+    if (raw.trim() !== '') supplementContent = raw;
+  }
+  const librarianIdentity =
+    supplementContent === null ? librarianBase : composeLibrarianDef(librarianBase, supplementContent);
+  const supplementPlanLine =
+    supplementContent !== null
+      ? `${librarianSupplement} (${supplementContent.length} bytes, appended; operator-wins on conflict)`
+      : librarianSupplement
+        ? `${librarianSupplement} (empty — ignored)`
+        : 'none';
+
+  if (dryRun) {
+    log('DRY RUN — would invoke:', { stderr });
+    log(`  cwd:                  ${repoRoot}`, { stderr });
+    log(`  pa name:              ${paName}`, { stderr });
+    log(`  staging count:        ${stagingCount}`, { stderr });
+    log(`  log file:             ${runLogPath}`, { stderr });
+    log(`  session name:         ${sessionName}`, { stderr });
+    log(`  appended system:      ${librarianDef} (${librarianBase.length} bytes)`, { stderr });
+    log(`  operator supplement:  ${supplementPlanLine}`, { stderr });
+    log(`  composed total:       ${librarianIdentity.length} bytes`, { stderr });
+    log(`  prompt:               <inline> (${DISTILL_PROMPT.length} bytes)`, { stderr });
+    log('  command:              claude --print --permission-mode bypassPermissions \\', { stderr });
+    log(`                          --no-session-persistence --name ${sessionName} \\`, { stderr });
+    log('                          --append-system-prompt <librarian definition> \\', { stderr });
+    log('                          <distill prompt as positional arg>', { stderr });
+    log('  git:                  none — sync (core/tools/sync) picks up changes', { stderr });
+    return 0;
+  }
+
+  log(`starting distillation sweep — log: ${runLogPath}`, { stderr });
+  const start = Date.now();
+
+  const claudeArgv = claudeArgs({ sessionName, librarianDef: librarianIdentity });
+  const claudeResult = await runner.spawn('claude', claudeArgv, { cwd: repoRoot });
+
+  // Persist claude stdout/stderr to the per-run log file.
+  await appendFile(runLogPath, claudeResult.stdout || '');
+  await appendFile(runLogPath, claudeResult.stderr || '');
+
+  const end = Date.now();
+  const durationSec = Math.round((end - start) / 1000);
+
+  if (claudeResult.code !== 0) {
+    log(`distillation FAILED (exit ${claudeResult.code}) — see ${runLogPath}`, { stderr });
+    await writeAudit(runsLog, {
+      ts: tsIso,
+      status: 'fail-claude',
+      exitCode: 3,
+      durationSec,
+      stagingCount,
+      log: basename(runLogPath),
+      reason: `claude exited ${claudeResult.code}`,
+    }, stderr);
+    return 3;
+  }
+
+  log(`distillation OK (${durationSec}s) — log: ${runLogPath}`, { stderr });
+  log('changes (if any) will be committed by the next sync run', { stderr });
+
+  // Step 4 — append success audit. Sync will pick up any file changes
+  // the librarian produced on its next 30-minute tick.
+  await writeAudit(runsLog, {
+    ts: tsIso,
+    status: 'ok',
+    exitCode: 0,
+    durationSec,
+    stagingCount,
+    log: basename(runLogPath),
+  }, stderr);
+  return 0;
+}
+
+function isoCompactForSession(d) {
+  // Matches the historical `$(date -u +%Y%m%d-%H%M)` shape used pre-Node.
+  const pad = (n) => String(n).padStart(2, '0');
+  return (
+    `${d.getUTCFullYear()}${pad(d.getUTCMonth() + 1)}${pad(d.getUTCDate())}` +
+    `-${pad(d.getUTCHours())}${pad(d.getUTCMinutes())}`
+  );
+}
+
+// Export for unit test.
+export { opError };

package/dist/engineering-pa-core/src/distill/runner.mjs

@@ -0,0 +1,86 @@
+// distill — chokepoint for external effects.
+//
+// Every process spawn, file write (for run logs), and ts/now call
+// funnels through here. This module is the ONLY place that touches
+// the outside world at runtime. Tests inject a fake `make()` factory
+// to assert on what would have happened without actually spawning.
+//
+// Contract:
+//   - spawn(cmd, args, { stdin?, capture? }) -> { code, stdout, stderr }
+//     * No shell interpolation. Arguments are passed as an array to
+//       child_process.spawn with shell:false. A malicious staging-file
+//       name cannot become a shell command because the shell is never
+//       invoked.
+//     * stdin is optional; when provided it is written as-is and the
+//       stream is closed.
+//     * capture:true collects stdout/stderr into strings (default).
+//       capture:false streams directly to the parent process.
+//   - now() -> Date — single source of wall-clock time, for testing.
+//   - ts() -> ISO-ish UTC timestamp `YYYY-MM-DDTHH-MM-SSZ` suitable
+//     for filenames (no colons). Not a valid RFC-3339 string — do not
+//     feed this to `new Date()`, use `tsIso()` for parseable output.
+//   - tsIso() -> strict RFC-3339 / ISO-8601 UTC timestamp
+//     `YYYY-MM-DDTHH:MM:SSZ` (second precision). Safe to round-trip via
+//     `new Date()`; use this for any field a downstream reader may parse.
+
+import { spawn as realSpawn } from 'node:child_process';
+
+export function createRunner({ spawner = realSpawn, clock = () => new Date() } = {}) {
+  return {
+    async spawn(cmd, args, { stdin, capture = true, cwd, env } = {}) {
+      return new Promise((resolve, reject) => {
+        const child = spawner(cmd, args, {
+          shell: false,
+          cwd,
+          env: env ?? process.env,
+          stdio: capture ? ['pipe', 'pipe', 'pipe'] : ['pipe', 'inherit', 'inherit'],
+        });
+
+        let stdout = '';
+        let stderr = '';
+
+        if (capture) {
+          child.stdout.on('data', (d) => {
+            stdout += d.toString('utf8');
+          });
+          child.stderr.on('data', (d) => {
+            stderr += d.toString('utf8');
+          });
+        }
+
+        child.on('error', (err) => reject(err));
+        child.on('close', (code) => resolve({ code: code ?? 0, stdout, stderr }));
+
+        if (stdin !== undefined) {
+          child.stdin.end(stdin);
+        } else {
+          child.stdin.end();
+        }
+      });
+    },
+
+    now() {
+      return clock();
+    },
+
+    ts() {
+      const d = clock();
+      const pad = (n) => String(n).padStart(2, '0');
+      return (
+        `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())}` +
+        `T${pad(d.getUTCHours())}-${pad(d.getUTCMinutes())}-${pad(d.getUTCSeconds())}Z`
+      );
+    },
+
+    tsIso() {
+      const d = clock();
+      const pad = (n) => String(n).padStart(2, '0');
+      return (
+        `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())}` +
+        `T${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}Z`
+      );
+    },
+  };
+}
+
+export const defaultRunner = createRunner();

package/dist/engineering-pa-core/src/distill/staging.mjs

@@ -0,0 +1,59 @@
+// distill — staging observation counter.
+//
+// Walks `agents/*/staging/` (depth ≤ 2) counting *.md files, excluding
+// anything under `_archived/`. Pure filesystem logic, no spawning. The
+// original shell script used `find -maxdepth 2`; we preserve that.
+
+import { readdir } from 'node:fs/promises';
+import { join } from 'node:path';
+
+export async function countStagedObservations(repoRoot) {
+  const agentsDir = join(repoRoot, 'agents');
+  let agents;
+  try {
+    agents = await readdir(agentsDir, { withFileTypes: true });
+  } catch (err) {
+    if (err.code === 'ENOENT') return 0;
+    throw err;
+  }
+
+  let count = 0;
+  for (const a of agents) {
+    if (!a.isDirectory()) continue;
+    const stagingDir = join(agentsDir, a.name, 'staging');
+    count += await countMdsDepth2(stagingDir);
+  }
+  return count;
+}
+
+async function countMdsDepth2(dir) {
+  let entries;
+  try {
+    entries = await readdir(dir, { withFileTypes: true });
+  } catch (err) {
+    if (err.code === 'ENOENT' || err.code === 'ENOTDIR') return 0;
+    throw err;
+  }
+  let n = 0;
+  for (const e of entries) {
+    if (e.name === '_archived') continue;
+    if (e.isFile() && e.name.endsWith('.md')) {
+      n++;
+      continue;
+    }
+    if (e.isDirectory()) {
+      // depth 2: list one more level, skip _archived subtrees.
+      const sub = join(dir, e.name);
+      let subEntries;
+      try {
+        subEntries = await readdir(sub, { withFileTypes: true });
+      } catch {
+        continue;
+      }
+      for (const s of subEntries) {
+        if (s.isFile() && s.name.endsWith('.md')) n++;
+      }
+    }
+  }
+  return n;
+}

package/dist/engineering-pa-core/src/knowledge/retrieval/backlink-indexer.mjs

@@ -0,0 +1,154 @@
+#!/usr/bin/env node
+// backlink-indexer.mjs — Scans markdown files for internal links and builds a backlink index.
+//
+// Finds wiki-style [[target]] links and markdown [text](relative/path.md) links
+// in all .md files under operator/knowledge/ and core/knowledge/.
+//
+// Output: operator/knowledge/.backlinks.json
+//
+// CLI: node backlink-indexer.mjs [--root /path/to/repo]
+
+import { readdir, readFile, writeFile, stat } from 'node:fs/promises';
+import { join, resolve, dirname, relative, normalize } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const DEFAULT_ROOT = resolve(__dirname, '../../../..');
+
+const USAGE = `
+backlink-indexer — Build a backlink index from markdown internal links
+
+Usage:
+  node backlink-indexer.mjs [options]
+
+Options:
+  --root <path>    Repository root (default: 4 levels up from this script)
+  --help, -h       Show this help
+
+Scans all .md files under operator/knowledge/ and core/knowledge/.
+Finds:
+  - Wiki-style links: [[target]]
+  - Markdown links:   [text](relative/path.md)
+
+Output: operator/knowledge/.backlinks.json
+`.trim();
+
+// ── Helpers ─────────────────────────────────────────────────────────
+
+async function walkDir(dir) {
+  const results = [];
+  let entries;
+  try {
+    entries = await readdir(dir, { withFileTypes: true });
+  } catch (err) {
+    if (err.code === 'ENOENT') return results;
+    throw err;
+  }
+  for (const entry of entries) {
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (entry.name.startsWith('.')) continue;
+      results.push(...await walkDir(full));
+    } else if (entry.isFile() && entry.name.endsWith('.md')) {
+      results.push(full);
+    }
+  }
+  return results;
+}
+
+function extractLinks(content, sourcePath, repoRoot) {
+  const links = [];
+  const sourceDir = dirname(sourcePath);
+
+  // Wiki-style: [[target]]
+  const wikiRe = /\[\[([^\]]+)\]\]/g;
+  let m;
+  while ((m = wikiRe.exec(content)) !== null) {
+    let target = m[1].trim();
+    // Resolve as relative to repo root
+    if (!target.endsWith('.md')) target += '.md';
+    const resolved = normalize(target);
+    links.push(resolved);
+  }
+
+  // Markdown-style: [text](relative/path.md)
+  // Exclude external URLs and anchors
+  const mdRe = /\[([^\]]*)\]\(([^)]+)\)/g;
+  while ((m = mdRe.exec(content)) !== null) {
+    const href = m[2].trim();
+    if (href.startsWith('http://') || href.startsWith('https://') || href.startsWith('#') || href.startsWith('mailto:')) continue;
+    // Strip anchor fragment
+    const path = href.split('#')[0];
+    if (!path) continue;
+    if (!path.endsWith('.md')) continue;
+    // Resolve relative to source file's directory
+    const abs = resolve(sourceDir, path);
+    const rel = relative(repoRoot, abs);
+    links.push(normalize(rel));
+  }
+
+  return links;
+}
+
+// ── Main ────────────────────────────────────────────────────────────
+
+async function main(args) {
+  let root = DEFAULT_ROOT;
+
+  for (let i = 0; i < args.length; i++) {
+    const a = args[i];
+    if (a === '--help' || a === '-h') {
+      console.log(USAGE);
+      return 0;
+    }
+    if (a === '--root') {
+      root = resolve(args[++i]);
+    }
+  }
+
+  const scanDirs = [
+    join(root, 'operator', 'knowledge'),
+    join(root, 'core', 'knowledge'),
+  ];
+
+  // Collect all markdown files
+  const allFiles = [];
+  for (const dir of scanDirs) {
+    allFiles.push(...await walkDir(dir));
+  }
+
+  // Build backlinks map: target -> [sources]
+  const backlinks = {};
+
+  for (const filePath of allFiles) {
+    const content = await readFile(filePath, 'utf8');
+    const relSource = relative(root, filePath);
+    const links = extractLinks(content, filePath, root);
+
+    for (const target of links) {
+      if (!backlinks[target]) backlinks[target] = [];
+      if (!backlinks[target].includes(relSource)) {
+        backlinks[target].push(relSource);
+      }
+    }
+  }
+
+  // Sort source arrays for deterministic output
+  for (const target of Object.keys(backlinks)) {
+    backlinks[target].sort();
+  }
+
+  const output = {
+    version: '1.0',
+    generatedAt: new Date().toISOString(),
+    backlinks,
+  };
+
+  const outPath = join(root, 'operator', 'knowledge', '.backlinks.json');
+  await writeFile(outPath, JSON.stringify(output, null, 2) + '\n', 'utf8');
+  console.log(`Wrote ${outPath} (${Object.keys(backlinks).length} targets indexed)`);
+  return 0;
+}
+
+const exitCode = await main(process.argv.slice(2));
+process.exit(exitCode ?? 0);