@bongos/core 1.17.0

package/scripts/gds/run-routine.js

@@ -0,0 +1,197 @@
+#!/usr/bin/env node
+// scripts/gds/run-routine.js — System 4 (task 1355): the deterministic-first
+// cron dispatcher. Several scheduled routines are an Opus session wrapping a
+// single committed deterministic script (peer-vote-tally literally says "do NOT
+// compute by hand — run tally.js"). Booting a model to shell one node command
+// pays a session boot + per-turn cache_read for nothing. This dispatcher reads a
+// routine's SKILL.md frontmatter and runs it at the cheapest sufficient tier:
+//
+//   mode: deterministic — execFileSync each step as plain node; NO model launched.
+//                         This is the WHOLE execution.
+//   mode: hybrid        — run the deterministic steps; only spawn `claude -p` when
+//                         a step emits a non-empty {"needs_judgment": …} payload,
+//                         feeding the model ONLY that payload (not the transcript).
+//   mode: judgment      — spawn `claude -p` over the SKILL body (status quo).
+//
+// Each step routes its stdout by `sink: discord|gds|none`. A failed step exits
+// non-zero after a best-effort #ops alert. Runs the steps with
+// OTB_COST_CRON_ROUTINE set so System 1 tags any (hybrid) model spend as
+// cron:<routine>. Usage: node scripts/gds/run-routine.js <routine-name> [--dry-run]
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { execFileSync } = require('child_process');
+
+const REPO_ROOT = path.resolve(__dirname, '..', '..');
+const TASKS_DIR = path.join(REPO_ROOT, '.claude', 'scheduled-tasks');
+const VALID_MODES = new Set(['deterministic', 'hybrid', 'judgment']);
+const VALID_SINKS = new Set(['discord', 'gds', 'none']);
+const ALERT_CHANNEL = process.env.OTB_ROUTINE_ALERT_CHANNEL || 'ship_feed';
+
+function unquote(s) {
+  const t = String(s == null ? '' : s).trim();
+  if ((t.startsWith('"') && t.endsWith('"')) || (t.startsWith("'") && t.endsWith("'"))) return t.slice(1, -1);
+  return t;
+}
+
+// Pure: parse mode + steps[] from SKILL.md frontmatter. A focused YAML subset
+// (no js-yaml dep): scalar `mode:`/`description:` and a `steps:` list of
+// `- command: …` / `sink: …` items. Unknown keys are ignored. Exported + tested.
+function parseRoutineFrontmatter(skillMd) {
+  const norm = String(skillMd == null ? '' : skillMd).replace(/\r\n/g, '\n');
+  const empty = { mode: null, steps: [], description: null };
+  if (!norm.startsWith('---\n')) return empty;
+  const lines = norm.split('\n');
+  let close = -1;
+  for (let i = 1; i < lines.length; i++) {
+    if (lines[i] === '---' || lines[i] === '...') { close = i; break; }
+  }
+  if (close === -1) return empty;
+
+  let mode = null;
+  let description = null;
+  const steps = [];
+  let inSteps = false;
+  let cur = null;
+  const pushCur = () => { if (cur && cur.command) steps.push({ command: cur.command, sink: cur.sink || 'none' }); cur = null; };
+
+  for (let i = 1; i < close; i++) {
+    const line = lines[i].replace(/\s+$/, '');
+    if (line.trim() === '' || line.trim().startsWith('#')) continue;
+    const topKv = /^([A-Za-z0-9_.-]+):\s*(.*)$/.exec(line);
+    const isTopLevel = /^[A-Za-z0-9_.-]+:/.test(line); // no leading indent
+
+    if (inSteps) {
+      const item = /^\s*-\s*(.*)$/.exec(line);
+      if (item) {
+        pushCur();
+        cur = { command: null, sink: 'none' };
+        const kv = /^([A-Za-z0-9_.-]+):\s*(.*)$/.exec(item[1]);
+        if (kv && kv[1] === 'command') cur.command = unquote(kv[2]);
+        else if (kv && kv[1] === 'sink') cur.sink = unquote(kv[2]) || 'none';
+        else if (item[1].trim()) cur.command = unquote(item[1]); // bare "- node x.js"
+        continue;
+      }
+      const sub = /^\s+([A-Za-z0-9_.-]+):\s*(.*)$/.exec(line);
+      if (sub && cur) {
+        if (sub[1] === 'command') cur.command = unquote(sub[2]);
+        else if (sub[1] === 'sink') cur.sink = unquote(sub[2]) || 'none';
+        continue;
+      }
+      if (isTopLevel) { pushCur(); inSteps = false; } // a new top-level key ends the list
+    }
+
+    if (!inSteps && isTopLevel && topKv) {
+      if (topKv[1] === 'mode') mode = unquote(topKv[2]) || null;
+      else if (topKv[1] === 'description') description = topKv[2].trim();
+      else if (topKv[1] === 'steps' && topKv[2].trim() === '') inSteps = true;
+    }
+  }
+  pushCur();
+  return { mode, steps, description };
+}
+
+// Pure: split a repo-authored command string into argv, honoring simple quotes.
+// Commands come from committed SKILL.md (not user input), so this is a tidy
+// tokenizer, not a security boundary.
+function tokenize(cmd) {
+  const out = [];
+  const re = /"([^"]*)"|'([^']*)'|(\S+)/g;
+  let m;
+  while ((m = re.exec(String(cmd || ''))) !== null) out.push(m[1] ?? m[2] ?? m[3]);
+  return out;
+}
+
+function opsAlert(message) {
+  try {
+    execFileSync(process.execPath, [path.join('scripts', 'discord', 'post.js'), '--channel', ALERT_CHANNEL, String(message).slice(0, 1800)],
+      { cwd: REPO_ROOT, stdio: 'ignore' });
+  } catch (_) { /* best-effort: the alert must never mask the real failure */ }
+}
+
+function routeOutput(sink, stdout, routineName) {
+  const text = String(stdout == null ? '' : stdout).trim();
+  if (text) console.log(text);
+  if (sink === 'discord' && text) {
+    try {
+      execFileSync(process.execPath, [path.join('scripts', 'discord', 'post.js'), '--channel', ALERT_CHANNEL, `[${routineName}] ${text}`.slice(0, 1800)],
+        { cwd: REPO_ROOT, stdio: 'ignore' });
+    } catch (_) { /* best-effort */ }
+  }
+  // sink 'gds' / 'none': the step's command performed its own GDS side effects.
+}
+
+function runStep(step, routineName, { dryRun }) {
+  const argv = tokenize(step.command);
+  if (!argv.length) return '';
+  if (dryRun) { console.log(`[dry-run] would run (${step.sink}): ${step.command}`); return ''; }
+  const out = execFileSync(argv[0], argv.slice(1), {
+    cwd: REPO_ROOT,
+    encoding: 'utf8',
+    env: { ...process.env, OTB_COST_CRON_ROUTINE: routineName },
+  });
+  routeOutput(step.sink, out, routineName);
+  return out;
+}
+
+function runJudgment(routineName, skillBody, { dryRun }) {
+  if (dryRun) { console.log(`[dry-run] judgment mode → would spawn claude -p over ${routineName}/SKILL.md`); return; }
+  // Status quo: a wrapped session reading the SKILL body. Reuse the shared spawn.
+  const grader = require('../../modules/grading/grader');
+  return grader.runSubagent({ prompt: skillBody, opts: { cwd: REPO_ROOT } });
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+  const dryRun = args.includes('--dry-run');
+  const name = args.find((a) => !a.startsWith('--'));
+  if (!name) { console.error('usage: run-routine.js <routine-name> [--dry-run]'); process.exit(2); }
+
+  const skillPath = path.join(TASKS_DIR, name, 'SKILL.md');
+  let skill;
+  try { skill = fs.readFileSync(skillPath, 'utf8'); }
+  catch (_) { console.error(`run-routine: no SKILL.md for routine '${name}' at ${skillPath}`); process.exit(2); }
+
+  const { mode, steps } = parseRoutineFrontmatter(skill);
+  if (!VALID_MODES.has(mode)) {
+    console.error(`run-routine: routine '${name}' has no/invalid 'mode:' (got ${JSON.stringify(mode)}). Add mode: deterministic|hybrid|judgment to its SKILL.md frontmatter.`);
+    process.exit(2);
+  }
+  for (const s of steps) {
+    if (!VALID_SINKS.has(s.sink)) { console.error(`run-routine: step has invalid sink '${s.sink}' (want ${[...VALID_SINKS].join('|')})`); process.exit(2); }
+  }
+
+  try {
+    if (mode === 'judgment') {
+      await runJudgment(name, skill, { dryRun });
+    } else {
+      // deterministic + hybrid both run the steps as code first.
+      for (const step of steps) {
+        const out = runStep(step, name, { dryRun });
+        if (mode === 'hybrid' && !dryRun) {
+          // Only escalate to a model when a step asks for judgment.
+          let payload = null;
+          try { const j = JSON.parse(String(out).trim().split('\n').pop() || '{}'); if (j && j.needs_judgment) payload = j.needs_judgment; } catch (_) {}
+          if (payload) {
+            const grader = require('../../modules/grading/grader');
+            await grader.runSubagent({ prompt: String(payload), opts: { cwd: REPO_ROOT } });
+          }
+        }
+      }
+    }
+  } catch (err) {
+    const msg = `routine '${name}' FAILED: ${err && err.message ? err.message : err}`;
+    console.error(`run-routine: ${msg}`);
+    opsAlert(`⚠ ${msg}`);
+    process.exit(1);
+  }
+  process.exit(0);
+}
+
+if (require.main === module) {
+  main();
+}
+
+module.exports = { parseRoutineFrontmatter, tokenize, VALID_MODES, VALID_SINKS };

package/scripts/gds/run-unit-tests.js

@@ -0,0 +1,178 @@
+#!/usr/bin/env node
+// scripts/gds/run-unit-tests.js — run the DB-free unit suite as one command,
+// for the #856 PR-check gate (ADR 0042). Auto-merge can't land a functional
+// regression once this is a required check.
+//
+// Runs every tests/*.mjs EXCEPT the documented exclusions below, each in its own
+// `node` process with a timeout, and exits non-zero if ANY fail. New pure tests
+// are picked up automatically (no allowlist to maintain) — only the two
+// exclusion categories need upkeep.
+//
+// Exclusions:
+//   INTEGRATION — need a live Postgres/server/network. They belong in the
+//     integration-smoke gate (a separate task with a Postgres service container),
+//     not this DB-free unit gate. (Classified by a DB/network-marker scan +
+//     verified by running them: they fail standalone.)
+//   KNOWN_FAILING — currently red on main, tracked separately so this gate stays
+//     trustworthy (a gate that's already red gates nothing). See idea #225.
+//
+// Module-owned tests (ADR 0107 / task 1768): a contributed module may ALSO ship
+// its own DB-free tests at modules/<key>/tests/*.mjs. These are discovered by
+// directory the same way the loader discovers modules/*/module.json (no
+// allowlist to maintain here either) and gated by isModuleEnabled(key) — a
+// module this instance doesn't enable contributes no tests to this gate, same
+// as the curated MODULE_TESTS/MODULE_DISABLED mechanism above.
+
+const fs = require('node:fs');
+const path = require('node:path');
+const { execFileSync } = require('node:child_process');
+const { isModuleEnabled } = require('../../src/modules');
+
+const TESTS_DIR = path.resolve(__dirname, '..', '..', 'tests');
+const MODULES_DIR = path.resolve(__dirname, '..', '..', 'modules');
+const PER_TEST_TIMEOUT_MS = 90_000;
+
+// Feature-module test sets (R59 / task 1198). A test that exercises a feature
+// MODULE runs only when this instance enables that module — a vanilla Cloud
+// Bongos instance carries no game/art code, so those tests are not part of its
+// suite. OTB enables both modules (config/modules.json), so nothing is skipped
+// here in this repo's CI; the gating only takes effect on an instance that turns
+// the module off. Curated like INTEGRATION/KNOWN_FAILING (not auto-derived) so a
+// new test is opted in deliberately.
+const MODULE_TESTS = {
+  game: [
+    'preview_server', 'sandbox_stage', 'walkability', 'snapshot_picker',
+    'terrain_picker', 'local_preview',
+  ],
+  'art-pipeline': [
+    'anti_grid', 'autotile_variants', 'autotile_xverify', 'art_key', 'art_stage',
+  ],
+};
+
+// Tests skipped because their feature module is OFF on THIS instance.
+const MODULE_DISABLED = new Set();
+for (const [mod, names] of Object.entries(MODULE_TESTS)) {
+  let on = true;
+  try { on = isModuleEnabled(mod); } catch { on = true; } // resolve failure → don't skip
+  if (!on) for (const n of names) MODULE_DISABLED.add(n);
+}
+
+// Shared test scaffolding (task 1096, C12) — exported helpers, not a test file;
+// running it standalone is a no-op. Skip so it doesn't inflate the count.
+const SUPPORT_MODULES = new Set(['helpers']);
+
+// Need a live DB / server / network — run in the integration-smoke gate instead.
+const INTEGRATION = new Set([
+  'asset_version', 'auth_flow_slice1', 'backfill_sessions', 'bfg', 'bfg_write',
+  'box_connect_e2e', 'box_onboard', 'boxes', 'cf_tunnel', 'cli_token_reissue',
+  'discord_autojoin', 'discord_oauth', 'linkify_refs', 'memory_isolation',
+  'memory_pull', 'parallel-load', 'session_records', 'setup_smoke',
+  'status-probe', 'subdomain_auth',
+]);
+
+// Excluded so this gate is trustworthy in a CLEAN checkout (the CI condition):
+//   - autotile_variants / autotile_xverify: depend on a python-generated artifact
+//     (modules/art-pipeline/pipeline/_autotile_tables.json) / build output absent in a fresh clone,
+//     so they exit non-zero standalone. Belong in an art-pipeline gate, not here.
+//   - anti_grid: runs modules/art-pipeline/pipeline/anti_grid_eval.py, which needs numpy (absent in
+//     the Node-only CI runner) -> ModuleNotFoundError. Same python-artifact class
+//     as autotile; belongs in an art-pipeline gate (task 859 / ADR 0042 cutover).
+//
+// grader_panel was excluded here (2 "failing" assertions, idea #225) until #954
+// (V4.R16) diagnosed them as STALE — they predated #481's Narc-advisory change,
+// NOT a grader regression. Now green and back under this gate so the quality-
+// gate brain can't silently regress.
+const KNOWN_FAILING = new Set([
+  'autotile_variants',
+  'autotile_xverify',
+  'anti_grid',
+]);
+
+// Discover modules/<key>/tests/*.mjs. Pure-ish (reads disk; `isEnabled` is
+// injected so this is unit-testable with zero real modules/ or DB config —
+// mirrors the injected-dependency pattern the module loader itself uses).
+// Returns { included: [{key,name,file}], skippedDisabled: [{key,name}] }.
+// A key that isEnabled() can't resolve (bad/missing manifest — not a real
+// mounted module) is treated as disabled: it isn't mounted, so its tests
+// don't run either.
+function discoverModuleTests({ modulesDir = MODULES_DIR, isEnabled = isModuleEnabled } = {}) {
+  const included = [];
+  const skippedDisabled = [];
+  let entries;
+  try { entries = fs.readdirSync(modulesDir, { withFileTypes: true }); }
+  catch (e) { if (e.code === 'ENOENT') return { included, skippedDisabled }; throw e; }
+
+  for (const ent of entries) {
+    if (!ent.isDirectory()) continue;
+    const key = ent.name;
+    const testsDir = path.join(modulesDir, key, 'tests');
+    let files;
+    try { files = fs.readdirSync(testsDir).filter((f) => f.endsWith('.mjs')); }
+    catch (e) { if (e.code === 'ENOENT') continue; throw e; }
+    if (files.length === 0) continue;
+
+    let enabled;
+    try { enabled = isEnabled(key); } catch { enabled = false; }
+
+    for (const f of files) {
+      const entry = { key, name: f.replace(/\.mjs$/, ''), file: path.join(testsDir, f) };
+      (enabled ? included : skippedDisabled).push(entry);
+    }
+  }
+  included.sort((a, b) => (a.key + a.name).localeCompare(b.key + b.name));
+  return { included, skippedDisabled };
+}
+
+function main() {
+  const files = fs
+    .readdirSync(TESTS_DIR)
+    .filter((f) => f.endsWith('.mjs'))
+    .map((f) => f.replace(/\.mjs$/, ''))
+    .filter((name) => !INTEGRATION.has(name) && !KNOWN_FAILING.has(name)
+      && !SUPPORT_MODULES.has(name) && !MODULE_DISABLED.has(name))
+    .sort();
+
+  const { included: moduleTests, skippedDisabled: moduleTestsSkipped } = discoverModuleTests();
+
+  const moduleNote = MODULE_DISABLED.size
+    ? ` + ${MODULE_DISABLED.size} disabled-module (${[...MODULE_DISABLED].sort().join(', ')})`
+    : '';
+  const modOwnedSkipNote = moduleTestsSkipped.length
+    ? ` + ${moduleTestsSkipped.length} module-owned test(s) skipped (module disabled: ${[...new Set(moduleTestsSkipped.map((t) => t.key))].sort().join(', ')})`
+    : '';
+  const total = files.length + moduleTests.length;
+  console.log(`run-unit-tests: ${total} DB-free unit test file(s) (${files.length} core + ${moduleTests.length} module-owned; excluded ${INTEGRATION.size} integration + ${KNOWN_FAILING.size} known-failing${moduleNote}${modOwnedSkipNote})`);
+
+  const failures = [];
+  const runOne = (displayName, filePath) => {
+    try {
+      execFileSync(process.execPath, [filePath], {
+        stdio: ['ignore', 'ignore', 'pipe'],
+        timeout: PER_TEST_TIMEOUT_MS,
+        env: { ...process.env, NODE_ENV: 'test' },
+      });
+      console.log(`  ok    ${displayName}`);
+    } catch (err) {
+      const why = err.killed ? 'TIMEOUT' : `exit ${err.status}`;
+      const stderr = (err.stderr ? err.stderr.toString() : '').trim().split('\n').slice(-3).join('\n      ');
+      console.log(`  FAIL  ${displayName} (${why})`);
+      if (stderr) console.log(`      ${stderr}`);
+      failures.push(displayName);
+    }
+  };
+
+  for (const name of files) runOne(name, path.join(TESTS_DIR, `${name}.mjs`));
+  for (const { key, name, file } of moduleTests) runOne(`${key}/${name}`, file);
+
+  console.log('');
+  if (failures.length === 0) {
+    console.log(`run-unit-tests: all ${total} passed.`);
+    process.exit(0);
+  }
+  console.log(`run-unit-tests: ${failures.length} FAILED — ${failures.join(', ')}`);
+  process.exit(1);
+}
+
+if (require.main === module) main();
+
+module.exports = { discoverModuleTests };