forge-workflow 0.1.0-beta.3 → 0.1.0-beta.5

package/scripts/doc-asserting-tests.js

@@ -0,0 +1,158 @@
+#!/usr/bin/env node
+'use strict';
+
+/**
+ * Runs (or lists) the test suites that assert on changed markdown.
+ *
+ * A markdown-only PR does not match the `paths:` filter in
+ * `.github/workflows/test.yml`, so the full matrix never runs and
+ * `required-checks-bypass.yml` reports the required "Test Suite" check green.
+ * Several suites nevertheless assert on markdown CONTENT, so such a PR merged
+ * green and then broke master for the next unrelated code PR — twice (kernel
+ * issue 63556816: the README size badge in #307/#310, and the AGENTS.md
+ * convention test after #325).
+ *
+ * This script closes that gap by running exactly the doc-asserting suites for the
+ * changed markdown. It shares `lib/doc-assertions.js` with the local push lane
+ * (`lib/commands/test.js`), so CI and local selection cannot drift apart.
+ *
+ * Usage:
+ *   node scripts/doc-asserting-tests.js [--base <ref>] [--list]
+ *
+ *   --base <ref>  Compare against <ref> (default: origin/<default-branch> or HEAD~1).
+ *   --list        Print the selected suites instead of running them.
+ */
+
+const { execFileSync, spawnSync } = require('node:child_process');
+const fs = require('node:fs');
+const path = require('node:path');
+
+const { selectDocAssertingTests } = require('../lib/doc-assertions');
+
+const REPO_ROOT = path.resolve(__dirname, '..');
+
+/** Wall-clock ceiling for the doc-asserting lane; it is a small, fast subset. */
+const LANE_TIMEOUT_MS = 5 * 60 * 1000;
+
+/**
+ * Parses the supported command-line flags.
+ *
+ * @param {string[]} argv Raw arguments (without node/script).
+ * @returns {{base: string|null, list: boolean}} Parsed options.
+ */
+function parseArgs(argv) {
+	const baseIndex = argv.indexOf('--base');
+	return {
+		base: baseIndex !== -1 ? argv[baseIndex + 1] || null : null,
+		list: argv.includes('--list'),
+	};
+}
+
+/**
+ * Resolves the ref the PR should be compared against.
+ *
+ * @param {string|null} explicitBase Base supplied with `--base`.
+ * @returns {string} A git ref usable in `git diff <ref>...HEAD`.
+ */
+function resolveBase(explicitBase) {
+	if (explicitBase) return explicitBase;
+	for (const ref of ['origin/master', 'origin/main']) {
+		try {
+			execFileSync('git', ['rev-parse', '--verify', ref], { cwd: REPO_ROOT, stdio: 'pipe', timeout: 5000 });
+			return ref;
+		} catch (_e) { /* intentional: ref not present in this checkout, try next */ } // NOSONAR S2486
+	}
+	return 'HEAD~1';
+}
+
+/**
+ * Parses `git diff --name-only` output into repository-relative paths.
+ *
+ * Split out so the parsing contract (trimmed, no blanks) is testable without a live
+ * repository: CI checks out shallow, so a test that reaches for `HEAD~1` passes locally
+ * and fails on the runner.
+ *
+ * @param {string} output Raw `git diff --name-only` stdout.
+ * @returns {string[]} Changed file paths.
+ */
+function parseChangedFiles(output) {
+	return output.split('\n').map((line) => line.trim()).filter(Boolean);
+}
+
+/**
+ * Lists repository-relative paths changed against the base ref.
+ *
+ * @param {string} base Base git ref.
+ * @returns {string[]} Changed file paths.
+ */
+function changedFilesSince(base) {
+	const output = execFileSync('git', ['diff', '--name-only', `${base}...HEAD`], {
+		cwd: REPO_ROOT,
+		encoding: 'utf8',
+		timeout: 15000,
+	});
+	return parseChangedFiles(output);
+}
+
+/**
+ * Turns a `spawnSync` result into a process exit status.
+ *
+ * A spawn that never produced a status (`error`, or killed by a signal so `status`
+ * is null) MUST report FAILURE. Reporting 0 there would re-create the bug this lane
+ * exists to remove: a required check reporting green without running the tests.
+ *
+ * @param {{status: number|null, error?: Error}} result Result of `spawnSync`.
+ * @returns {number} Exit status; non-zero whenever the run did not demonstrably pass.
+ */
+function resolveSpawnStatus(result) {
+	if (!result || result.error) return 1;
+	return result.status ?? 1;
+}
+
+function main() {
+	const { base: explicitBase, list } = parseArgs(process.argv.slice(2));
+	const base = resolveBase(explicitBase);
+
+	let changedFiles;
+	try {
+		changedFiles = changedFilesSince(base);
+	} catch (error) {
+		// Fail closed: if the change set cannot be determined we cannot prove the
+		// doc-asserting suites are unaffected.
+		console.error(`Could not compute changed files against ${base}: ${error.message}`);
+		return 1;
+	}
+
+	const suites = selectDocAssertingTests(changedFiles, REPO_ROOT, fs);
+
+	if (suites.length === 0) {
+		console.log('No changed markdown asserts on by any test suite — nothing to run.');
+		return 0;
+	}
+
+	if (list) {
+		console.log(suites.join('\n'));
+		return 0;
+	}
+
+	console.log(`Running ${suites.length} doc-asserting suite${suites.length === 1 ? '' : 's'} for changed markdown:`);
+	for (const suite of suites) console.log(`  ${suite}`);
+
+	const result = spawnSync('bun', ['test', ...suites], {
+		cwd: REPO_ROOT,
+		stdio: 'inherit',
+		timeout: LANE_TIMEOUT_MS,
+		shell: process.platform === 'win32',
+	});
+
+	if (result.error) {
+		console.error(`Failed to run doc-asserting suites: ${result.error.message}`);
+	}
+	return resolveSpawnStatus(result);
+}
+
+if (require.main === module) {
+	process.exit(main());
+}
+
+module.exports = { changedFilesSince, parseArgs, parseChangedFiles, resolveBase, resolveSpawnStatus };

package/scripts/forge-team/index.sh

@@ -4,7 +4,6 @@
 # Subcommands:
 #   workload    Show team workload by developer
 #   epic        Epic progress rollup
-#   dashboard   Team health dashboard
 #   add         Add developer to team map
 #   verify      Check 1:1 Beads<>GitHub enforcement
 #   sync        Manual GitHub<>Beads sync
@@ -22,7 +21,6 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 [[ -f "$SCRIPT_DIR/lib/identity.sh" ]] && source "$SCRIPT_DIR/lib/identity.sh"
 [[ -f "$SCRIPT_DIR/lib/workload.sh" ]] && source "$SCRIPT_DIR/lib/workload.sh"
 [[ -f "$SCRIPT_DIR/lib/epic.sh" ]] && source "$SCRIPT_DIR/lib/epic.sh"
-[[ -f "$SCRIPT_DIR/lib/dashboard.sh" ]] && source "$SCRIPT_DIR/lib/dashboard.sh"
 [[ -f "$SCRIPT_DIR/lib/hooks.sh" ]] && source "$SCRIPT_DIR/lib/hooks.sh"
 [[ -f "$SCRIPT_DIR/lib/verify.sh" ]] && source "$SCRIPT_DIR/lib/verify.sh"
 [[ -f "$SCRIPT_DIR/lib/claim.sh" ]] && source "$SCRIPT_DIR/lib/claim.sh"
@@ -34,7 +32,6 @@ FORGE_SCRIPTS="$(cd "$SCRIPT_DIR/.." && pwd)"
 # ── Stub implementations ──
 # cmd_workload provided by lib/workload.sh
 # cmd_epic provided by lib/epic.sh
-# cmd_dashboard provided by lib/dashboard.sh
 cmd_add() { auto_detect_identity "$@"; }
 # cmd_verify provided by lib/verify.sh
 cmd_sync() { forge_team_sync "$@"; }
@@ -47,7 +44,6 @@ Usage: forge team <subcommand> [args...]
 Subcommands:
   workload [--developer=<user>] [--me]  Show team workload
   epic <issue-id>                        Epic progress rollup
-  dashboard                              Team health dashboard
   add [--github=<user>]                  Add developer to team map
   verify                                 Check 1:1 Beads<>GitHub sync
   sync                                   Manual GitHub<>Beads sync
@@ -69,7 +65,6 @@ main() {
   case "$subcommand" in
     workload)   cmd_workload "$@" ;;
     epic)       cmd_epic "$@" ;;
-    dashboard)  cmd_dashboard "$@" ;;
     add)        cmd_add "$@" ;;
     verify)     cmd_verify "$@" ;;
     sync)       cmd_sync "$@" ;;

package/scripts/forge-team/tests/dispatcher.test.sh

@@ -60,7 +60,7 @@ assert_output_contains "unknown subcommand shows error" "unknown subcommand" bas
 # Test 4: Each stub subcommand is reachable (exits 0)
 echo ""
 echo "-- stub subcommands reachable --"
-for cmd in workload epic dashboard add verify sync claim; do
+for cmd in workload epic add verify sync claim; do
   assert_exit "$cmd exits 0" 0 bash "$DISPATCHER" "$cmd"
   assert_output_contains "$cmd outputs stub message" "not implemented" bash "$DISPATCHER" "$cmd"
 done

package/scripts/forge-team/tests/workflow-integration.test.sh

@@ -17,7 +17,6 @@ assert_contains_file() {
 echo "── /status integration ──"
 STATUS="$SCRIPT_DIR/.claude/commands/status.md"
 assert_contains_file "status calls workload --me" "forge-team.*workload.*--me" "$STATUS"
-assert_contains_file "status calls dashboard" "forge-team.*dashboard" "$STATUS"
 
 echo ""
 echo "── /plan integration ──"

package/scripts/lib/behavioral-eval-runner.js

@@ -0,0 +1,310 @@
+'use strict';
+
+const { loadTier, evaluateCase } = require('./immutable-eval-corpus');
+const { createEvalEvidence, appendEvalEvidence } = require('./eval-evidence');
+
+const RESULT_FIELDS = Object.freeze(['evidence', 'attribution']);
+const ATTRIBUTION_FIELDS = Object.freeze([
+  'model', 'effort', 'role', 'hashes', 'startedAt', 'endedAt', 'activeMs',
+  'passiveMs', 'tokens', 'retries', 'compactions',
+]);
+const ATTRIBUTION_HASH_FIELDS = Object.freeze(['prompt', 'skill', 'tool']);
+const BINDING_FIELDS = Object.freeze(['repoSha', 'configHash', 'budgetHash']);
+const ARM_FIELDS = Object.freeze(['id', 'model', 'config', 'budget']);
+const STRUCTURAL_FAILURE_PREFIXES = Object.freeze([
+  'evidence.', 'binding.', 'case_id.', 'packet.', 'split.', 'trial.', 'metrics.',
+  'manifest.', 'observation.',
+]);
+const SAFE_RUNTIME_FAILURES = new Set([
+  'runtime.execution_failed', 'runtime.token_budget_exceeded', 'runtime.usage_unparseable',
+]);
+
+function hasExactFields(value, fields) {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
+  const keys = Object.keys(value);
+  return keys.length === fields.length && fields.every((field) => Object.hasOwn(value, field));
+}
+
+function validExecutorResult(result) {
+  return hasExactFields(result, RESULT_FIELDS) &&
+    hasExactFields(result.attribution, ATTRIBUTION_FIELDS) &&
+    hasExactFields(result.attribution.hashes, ATTRIBUTION_HASH_FIELDS);
+}
+
+function isStructuralFailure(failure) {
+  return STRUCTURAL_FAILURE_PREFIXES.some((prefix) => failure.startsWith(prefix));
+}
+
+function snapshotBinding(binding) {
+  if (!hasExactFields(binding, BINDING_FIELDS)) return null;
+  if (!/^[0-9a-f]{40}$/.test(binding.repoSha)) return null;
+  if (!/^[0-9a-f]{64}$/.test(binding.configHash)) return null;
+  if (!/^[0-9a-f]{64}$/.test(binding.budgetHash)) return null;
+  return Object.freeze({
+    repoSha: binding.repoSha,
+    configHash: binding.configHash,
+    budgetHash: binding.budgetHash,
+  });
+}
+
+function snapshotArms(arms) {
+  if (!Array.isArray(arms) || arms.length !== 4) return null;
+  const snapshots = [];
+  for (const arm of arms) {
+    if (!hasExactFields(arm, ARM_FIELDS)) return null;
+    if ([arm.id, arm.model, arm.config, arm.budget]
+      .some((value) => typeof value !== 'string' || value.length === 0)) return null;
+    if (!['current', 'bounded'].includes(arm.config)) return null;
+    snapshots.push(Object.freeze({ ...arm }));
+  }
+  if (new Set(snapshots.map((arm) => arm.id)).size !== 4) return null;
+  if (new Set(snapshots.map((arm) => arm.model)).size !== 2) return null;
+  const matrix = new Set(snapshots.map((arm) => `${arm.model}\0${arm.config}`));
+  if (matrix.size !== 4) return null;
+  return Object.freeze(snapshots);
+}
+
+function explicitHardFailure(evaluation, result) {
+  return result.evidence?.observation?.hardFailure === true
+    || evaluation.hardFailure === true;
+}
+
+function buildEnvelope(input, identity, binding, evaluation, result, hardFailure) {
+  const attribution = result.attribution;
+  return createEvalEvidence({
+    issue_id: input.issueId,
+    pr: input.pr,
+    head_sha: binding.repoSha,
+    model: attribution.model,
+    effort: attribution.effort,
+    role: attribution.role,
+    hashes: {
+      eval_set: result.evidence.packetHash,
+      prompt: attribution.hashes.prompt,
+      skill: attribution.hashes.skill,
+      tool: attribution.hashes.tool,
+    },
+    started_at: attribution.startedAt,
+    ended_at: attribution.endedAt,
+    active_ms: attribution.activeMs,
+    passive_ms: attribution.passiveMs,
+    tokens: {
+      input: attribution.tokens.input,
+      output: attribution.tokens.output,
+      cached: attribution.tokens.cached,
+    },
+    retries: attribution.retries,
+    compactions: attribution.compactions,
+    gates: [{ name: 'behavioral-case', passed: evaluation.passed }],
+    run_identity: {
+      arm_id: identity.armId,
+      case_id: identity.caseId,
+      risk: identity.risk,
+      split: identity.split,
+      model: identity.model,
+      config: identity.config,
+      budget: identity.budget,
+      tier: input.tier,
+      trial_index: identity.trialIndex,
+      config_hash: binding.configHash,
+      budget_hash: binding.budgetHash,
+    },
+    case_result: {
+      status: evaluation.passed ? 'PASS' : 'FAIL',
+      hard_failure: hardFailure,
+      latency_ms: result.evidence.metrics.durationMs,
+      tokens: result.evidence.metrics.tokensUsed,
+    },
+  });
+}
+
+function finding(identity, status, failures, evidence, hardFailure = false) {
+  return {
+    caseId: identity.caseId,
+    risk: identity.risk,
+    split: identity.split,
+    model: identity.model,
+    config: identity.config,
+    budget: identity.budget,
+    trialIndex: identity.trialIndex,
+    status,
+    hardFailure,
+    latencyMs: Number.isFinite(evidence?.metrics?.durationMs) ? evidence.metrics.durationMs : 0,
+    tokens: Number.isFinite(evidence?.metrics?.tokensUsed) ? evidence.metrics.tokensUsed : 0,
+    failures,
+  };
+}
+
+function incompleteResult(tier, arms, expectedRuns, reason) {
+  return {
+    status: 'INCOMPLETE',
+    tier,
+    arms,
+    expectedRuns,
+    completedRuns: 0,
+    passedRuns: 0,
+    failedRuns: 0,
+    incompleteRuns: expectedRuns,
+    findings: [{ status: 'INCOMPLETE', failures: [reason] }],
+  };
+}
+
+function identityFor(packet, trialIndex, arm) {
+  return {
+    armId: arm.id,
+    caseId: packet.caseId,
+    risk: packet.risk,
+    split: packet.split,
+    model: arm.model,
+    config: arm.config,
+    budget: arm.budget,
+    trialIndex,
+  };
+}
+
+async function invokeExecutor(input, packet, trialIndex, arm, binding) {
+  try {
+    return await input.executor(Object.freeze({
+      armId: arm.id,
+      model: arm.model,
+      config: arm.config,
+      budget: arm.budget,
+      packet,
+      trialIndex,
+      binding,
+      skillName: input.skillName,
+    }));
+  } catch (error) {
+    const failure = error?.message;
+    return SAFE_RUNTIME_FAILURES.has(failure) ? { runtimeFailure: failure } : null;
+  }
+}
+
+async function persistEvidence(input, append, identity, binding, evaluation, result, hardFailure) {
+  try {
+    const envelope = buildEnvelope(input, identity, binding, evaluation, result, hardFailure);
+    const appended = await append(input.projectRoot, envelope, input.appendOptions || {});
+    if (appended?.conflict) return 'evidence.conflict';
+    if (!appended || appended.ok !== true) return 'evidence.append_failed';
+    return null;
+  } catch {
+    return 'evidence.append_failed';
+  }
+}
+
+async function executeArm({ input, corpus, packet, trialIndex, arm, binding, append }) {
+  const identity = identityFor(packet, trialIndex, arm);
+  const result = await invokeExecutor(input, packet, trialIndex, arm, binding);
+  if (result?.runtimeFailure) {
+    return {
+      status: 'INCOMPLETE',
+      finding: finding(identity, 'INCOMPLETE', [result.runtimeFailure]),
+    };
+  }
+  if (!validExecutorResult(result)) {
+    return { status: 'INCOMPLETE', finding: finding(identity, 'INCOMPLETE', ['evidence.malformed']) };
+  }
+  if (result.attribution.model !== arm.model) {
+    return {
+      status: 'INCOMPLETE',
+      finding: finding(identity, 'INCOMPLETE', ['attribution.model_mismatch'], result.evidence),
+    };
+  }
+
+  const evaluation = evaluateCase({
+    packet,
+    allPackets: corpus.allPackets,
+    manifest: corpus.manifest,
+    evidence: result.evidence,
+    expectedBinding: binding,
+  });
+  const hardFailure = explicitHardFailure(evaluation, result);
+  if (evaluation.failures.some(isStructuralFailure)) {
+    return {
+      status: 'INCOMPLETE',
+      finding: finding(identity, 'INCOMPLETE', evaluation.failures, result.evidence, hardFailure),
+    };
+  }
+
+  const persistenceFailure = await persistEvidence(
+    input, append, identity, binding, evaluation, result, hardFailure,
+  );
+  if (persistenceFailure) {
+    return {
+      status: 'INCOMPLETE',
+      finding: finding(identity, 'INCOMPLETE', [persistenceFailure], result.evidence, hardFailure),
+    };
+  }
+  const status = evaluation.passed ? 'PASS' : 'FAIL';
+  return {
+    status,
+    finding: finding(
+      identity, status, evaluation.passed ? [] : evaluation.failures, result.evidence, hardFailure,
+    ),
+  };
+}
+
+function recordOutcome(counts, outcome, findings) {
+  findings.push(outcome.finding);
+  if (outcome.status === 'INCOMPLETE') {
+    counts.incompleteRuns += 1;
+    return;
+  }
+  counts.completedRuns += 1;
+  if (outcome.status === 'PASS') counts.passedRuns += 1;
+  else counts.failedRuns += 1;
+}
+
+function finalStatus(counts, expectedRuns) {
+  if (counts.incompleteRuns > 0 || counts.completedRuns !== expectedRuns) return 'INCOMPLETE';
+  return counts.failedRuns > 0 ? 'FAIL' : 'PASS';
+}
+
+/**
+ * Execute the frozen corpus through four opaque, matched executor arms.
+ * The executor may observe arm ids and immutable packets, but it receives no
+ * merge capability. Only a strict, privacy-safe attribution envelope persists.
+ */
+async function runBehavioralEvaluation(input) {
+  let corpus;
+  try {
+    corpus = loadTier(input.tier);
+  } catch (error) {
+    return incompleteResult(input.tier, input.arms || [], 0, error.message);
+  }
+
+  const suppliedArms = Array.isArray(input.arms) ? input.arms : [];
+  const trialIndices = corpus.manifest.trialIndices;
+  const expectedRuns = corpus.cases.length * trialIndices.length * 4;
+  const arms = snapshotArms(suppliedArms);
+  if (!arms) return incompleteResult(input.tier, suppliedArms, expectedRuns, 'arms.invalid');
+  if (typeof input.executor !== 'function') {
+    return incompleteResult(input.tier, arms, expectedRuns, 'executor.missing');
+  }
+  const binding = snapshotBinding(input.binding);
+  if (!binding) return incompleteResult(input.tier, arms, expectedRuns, 'binding.invalid');
+
+  const append = input.appendEvidence || appendEvalEvidence;
+  const findings = [];
+  const counts = { completedRuns: 0, passedRuns: 0, failedRuns: 0, incompleteRuns: 0 };
+
+  for (const packet of corpus.cases) {
+    for (const trialIndex of trialIndices) {
+      for (const arm of arms) {
+        const outcome = await executeArm({ input, corpus, packet, trialIndex, arm, binding, append });
+        recordOutcome(counts, outcome, findings);
+      }
+    }
+  }
+
+  return {
+    status: finalStatus(counts, expectedRuns),
+    tier: input.tier,
+    arms,
+    expectedRuns,
+    ...counts,
+    findings,
+  };
+}
+
+module.exports = { runBehavioralEvaluation };