@ionivetech/mugiwara 0.8.0 → 0.8.2

package/src/integrity.ts

@@ -10,29 +10,73 @@
 //   3. Evidence — cited wave/evidence paths exist.
 import { existsSync, readdirSync, readFileSync } from 'node:fs';
 import { isAbsolute, join, relative } from 'node:path';
+import { loadPolicy } from './policy.ts';
 
-export type IntegrityIssue = { kind: 'dangling-path' | 'secret' | 'evidence' | 'evidence-thin'; detail: string };
+export type IntegrityIssue = {
+  kind: 'dangling-path' | 'secret' | 'secret-warn' | 'evidence' | 'evidence-thin';
+  detail: string;
+  severity?: 'block' | 'warn';
+};
 
-const SECRET_PATTERNS: Array<[RegExp, string]> = [
-  [/AKIA[0-9A-Z]{16}/, 'AWS access key id'],
-  [/-----BEGIN [A-Z ]*PRIVATE KEY-----/, 'private key block'],
-  [/gh[pousr]_[A-Za-z0-9]{20,}/, 'GitHub token'],
-  [/xox[baprs]-[A-Za-z0-9-]{10,}/, 'Slack token'],
-  [/sk-[A-Za-z0-9]{32,}/, 'API key (sk-…)'],
-  [/eyJhbGciOi[A-Za-z0-9_.-]{20,}/, 'JWT pasted verbatim'],
-  [/(api[_-]?key|secret|passwd|password)\s*[=:]\s*["'][^"'\s]{8,}["']/i, 'credential assignment'],
+export type SecretSeverity = 'block' | 'warn';
+export type SecretPattern = [RegExp, string, SecretSeverity];
+
+export const SECRET_PATTERNS: Array<SecretPattern> = [
+  [/AKIA[0-9A-Z]{16}/, 'AWS access key id', 'block'],
+  [/-----BEGIN [A-Z ]*PRIVATE KEY-----/, 'private key block', 'block'],
+  [/gh[pousr]_[A-Za-z0-9]{20,}/, 'GitHub token', 'block'],
+  [/xox[baprs]-[A-Za-z0-9-]{10,}/, 'Slack token', 'block'],
+  [/sk-[A-Za-z0-9]{32,}/, 'API key (sk-…)', 'block'],
+  [/eyJhbGciOi[A-Za-z0-9_.-]{20,}/, 'JWT pasted verbatim', 'block'],
+  [/(api[_-]?key|secret|passwd|password)\s*[=:]\s*["'][^"'\s]{8,}["']/i, 'credential assignment', 'block'],
+  [/AIza[0-9A-Za-z_-]{35}/, 'Google API key', 'block'],
+  [/ya29\.[0-9A-Za-z_-]{20,}/, 'Google OAuth token', 'block'],
+  [/\b[a-z][a-z0-9+.-]*:\/\/[^\s:@/]+:[^\s@/]{4,}@[^\s/]+/i, 'connection string with inline credential', 'block'],
+  [/\bAC[a-f0-9]{32}\b/, 'Twilio account SID', 'block'],
+  [/\bSK[a-f0-9]{32}\b/, 'Twilio API key', 'block'],
+  [/\bglpat-[A-Za-z0-9_-]{20,}/, 'GitLab token', 'block'],
+  [/\bnpm_[A-Za-z0-9]{36}\b/, 'npm token', 'block'],
+  [/\bdop_v1_[a-f0-9]{64}\b/, 'DigitalOcean token', 'block'],
+  [/\b(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|3[47][0-9]{13})\b/, 'card-number shape (verify before committing)', 'warn'],
 ];
 
 const ALLOW_SECRET = 'mugiwara:allow-secret';
 
+function loadExtraPatterns(projectRoot: string): Array<SecretPattern> {
+  try {
+    const policy = loadPolicy(projectRoot);
+    const extras = (policy as unknown as { integrity?: { extra_secret_patterns?: Array<{ pattern: string; label: string; severity?: SecretSeverity }> } })?.integrity?.extra_secret_patterns;
+    if (!extras || !Array.isArray(extras)) return [];
+    const out: Array<SecretPattern> = [];
+    for (const e of extras) {
+      if (!e || typeof (e as Record<string, unknown>).pattern !== 'string' || typeof (e as Record<string, unknown>).label !== 'string') continue;
+      const rec = e as { pattern: string; label: string; severity?: SecretSeverity };
+      const sev: SecretSeverity = rec.severity === 'warn' ? 'warn' : 'block';
+      try {
+        const re = new RegExp(rec.pattern);
+        out.push([re, rec.label, sev]);
+      } catch {
+        // invalid regex — skip
+      }
+    }
+    return out;
+  } catch {
+    return [];
+  }
+}
+
 /** Secret shapes per line; a line carrying the allow marker is skipped — deliberate examples stay possible. */
-function findSecrets(body: string): Array<{ label: string; hit: string }> {
-  const out: Array<{ label: string; hit: string }> = [];
+export function findSecrets(
+  body: string,
+  extra?: Array<SecretPattern>,
+): Array<{ label: string; hit: string; severity: SecretSeverity }> {
+  const out: Array<{ label: string; hit: string; severity: SecretSeverity }> = [];
+  const patterns: Array<SecretPattern> = extra ? [...SECRET_PATTERNS, ...extra] : SECRET_PATTERNS;
   for (const line of body.split(/\r?\n/)) {
     if (line.includes(ALLOW_SECRET)) continue;
-    for (const [re, label] of SECRET_PATTERNS) {
+    for (const [re, label, severity] of patterns) {
       const hit = line.match(re);
-      if (hit) out.push({ label, hit: hit[0] });
+      if (hit) out.push({ label, hit: hit[0], severity: (severity ?? 'block') as SecretSeverity });
     }
   }
   return out;
@@ -89,6 +133,7 @@ function collectPassCitedPaths(missionDir: string): string[] {
 export function checkTrail(missionDir: string, projectRoot: string): IntegrityIssue[] {
   const issues: IntegrityIssue[] = [];
   const files = trailFiles(missionDir);
+  const extraPatterns = loadExtraPatterns(projectRoot);
 
   // 1 + 2: per-file link resolution and secret scan
   for (const f of files) {
@@ -105,33 +150,66 @@ export function checkTrail(missionDir: string, projectRoot: string): IntegrityIs
         });
       }
     }
-    for (const { label, hit } of findSecrets(body)) {
+    for (const { label, hit, severity } of findSecrets(body, extraPatterns.length ? extraPatterns : undefined)) {
+      const isWarn = severity === 'warn';
       issues.push({
-        kind: 'secret',
+        kind: isWarn ? 'secret-warn' : 'secret',
         detail: `${relative(projectRoot, f)} matches ${label}: ${hit.slice(0, 12)}…`,
+        severity: isWarn ? 'warn' : 'block',
       });
     }
   }
 
   // 3: evidence entries recorded as repo paths must exist
   const evidencePaths: string[] = [];
-  const evidenceFile = join(missionDir, 'state.json');
-  if (existsSync(evidenceFile)) {
+  // Solo layout writes state.json; team layout writes <member>.json per member.
+  // Reading only state.json left the evidence gate dead on the team path (B2).
+  const stateFiles = existsSync(missionDir)
+    ? readdirSync(missionDir)
+        .filter((n) => n.endsWith('.json') && n !== 'continue.json' && !n.startsWith('continue-'))
+        .sort()
+    : [];
+  for (const name of stateFiles) {
+    const evidenceFile = join(missionDir, name);
     try {
       const s = JSON.parse(readFileSync(evidenceFile, 'utf8')) as { evidence?: unknown };
-      if (Array.isArray(s.evidence)) {
-        for (const e of s.evidence) {
-          if (typeof e !== 'string' || !e.trim()) continue;
-          evidencePaths.push(e);
-          const cand = join(projectRoot, e);
-          if (!isAbsolute(e) && !existsSync(cand) && !existsSync(join(missionDir, e))) {
-            issues.push({ kind: 'evidence', detail: `state.json evidence "${e}" does not exist` });
-          }
+      if (!Array.isArray(s.evidence)) continue;
+      for (const e of s.evidence) {
+        if (typeof e !== 'string' || !e.trim()) continue;
+        evidencePaths.push(e);
+        const cand = join(projectRoot, e);
+        if (!isAbsolute(e) && !existsSync(cand) && !existsSync(join(missionDir, e))) {
+          issues.push({ kind: 'evidence', detail: `${name} evidence "${e}" does not exist` });
         }
       }
     } catch { /* corrupt state — the state reader owns that error */ }
   }
 
+  // Iron Law: no evidence = not complete. Absent evidence is a different failure
+  // from thin evidence, and previously went unreported entirely. (B7)
+  if (evidencePaths.length === 0) {
+    let severity: 'warn' | 'block' = 'warn';
+    try {
+      const policy = loadPolicy(projectRoot);
+      const lanes = (policy as unknown as { evidence?: { require_nonempty_for_lanes?: string[] } })?.evidence?.require_nonempty_for_lanes;
+      if (Array.isArray(lanes) && lanes.length) {
+        const stateLanes = new Set<string>();
+        for (const name of stateFiles) {
+          try {
+            const s = JSON.parse(readFileSync(join(missionDir, name), 'utf8')) as { lane?: unknown };
+            if (typeof s.lane === 'string') stateLanes.add(s.lane);
+          } catch { /* ignore */ }
+        }
+        if ([...stateLanes].some((l) => lanes.includes(l))) severity = 'block';
+      }
+    } catch { /* policy read failure -> keep warn */ }
+    issues.push({
+      kind: 'evidence',
+      severity,
+      detail: 'mission declares no evidence — closing with zero recorded checks',
+    });
+  }
+
   // 4: evidence-content spot check (T7): a PASS verdict that cites an evidence
   // path must point at a file that exists AND contains command-output shape
   // (backticked command or exit-status token). Fake-but-consistent trails
@@ -139,6 +217,8 @@ export function checkTrail(missionDir: string, projectRoot: string): IntegrityIs
   const passCited = collectPassCitedPaths(missionDir);
   for (const e of passCited) {
     if (!e.trim() || isAbsolute(e)) continue;
+    // state/continue are machine-generated JSON, not evidence with command output — skip thin check
+    if (/(?:^|\/)(state|continue)(-[^\/]*)?\.json$/.test(e)) continue;
     const candMission = join(missionDir, e);
     const candRoot = join(projectRoot, e);
     const resolved = existsSync(candMission) ? candMission : existsSync(candRoot) ? candRoot : null;

package/src/mission.ts

@@ -7,9 +7,11 @@ import { checkTrail, formatIssues } from './integrity.ts';
 import { checkMissionArtifacts } from './check-artifacts.ts';
 import { generateRollback } from './rollback.ts';
 import { writeProvenance } from './provenance.ts';
+import { loadPolicy } from './policy.ts';
+import { verifyReport } from './sign.ts';
 import { rankFiles, renderRouting } from './routing.ts';
-import { formatFootprint, measureContextChars, readBudgetConfig } from './budget.ts';
-import { budgetForLane, costEnvelope, appendCostEvent } from './cost.ts';
+import { formatFootprint, measureContextChars, readBudgetConfig, shouldCompress, compressThreshold } from './budget.ts';
+import { budgetForLane, costEnvelope, appendCostEvent, COMPRESSED_KIND } from './cost.ts';
 import { loadRegistry } from './evidence.ts';
 import { computeContextMetrics, contextStatus } from './context.ts';
 import { buildCostLedger, renderAdaptationSection } from './reporting.ts';
@@ -31,6 +33,52 @@ function primaryState(dir: string, files: string[]): Record<string, unknown> | n
   }
 }
 
+/** Extract tasks from state.json — handles nested tasks:{done,total} (current savepoint) and legacy flat fields. */
+function tasksFromState(state: Record<string, unknown> | null): { done: number; total: number } {
+  if (!state) return { done: 0, total: 0 };
+  const t = (state as Record<string, unknown>).tasks as { done?: unknown; total?: unknown } | undefined;
+  if (t && typeof t.done === 'number' && typeof t.total === 'number') return { done: t.done, total: t.total };
+  const done = Number((state as Record<string, unknown>).tasks_done);
+  const total = Number((state as Record<string, unknown>).tasks_total);
+  return { done: Number.isFinite(done) ? done : 0, total: Number.isFinite(total) ? total : 0 };
+}
+
+/** Fresh task counts from plan.md, with sub-plan fallback for large campaigns (>3 phases). */
+function countPlanTasks(missionDir: string): { done: number; total: number } {
+  let total = 0;
+  let done = 0;
+  const planFile = join(missionDir, 'plan.md');
+  if (existsSync(planFile)) {
+    try {
+      const text = readFileSync(planFile, 'utf8');
+      for (const line of text.split(/\r?\n/)) {
+        if (/^\s*-\s*\[[ xX]\]/.test(line)) {
+          total++;
+          if (/^\s*-\s*\[[xX]\]/.test(line)) done++;
+        }
+      }
+    } catch { /* best-effort */ }
+  }
+  if (total === 0) {
+    const subPlanDir = join(missionDir, 'sub-plan');
+    if (existsSync(subPlanDir)) {
+      try {
+        for (const f of readdirSync(subPlanDir)) {
+          if (!f.endsWith('.md')) continue;
+          const text = readFileSync(join(subPlanDir, f), 'utf8');
+          for (const line of text.split(/\r?\n/)) {
+            if (/^\s*-\s*\[[ xX]\]/.test(line)) {
+              total++;
+              if (/^\s*-\s*\[[xX]\]/.test(line)) done++;
+            }
+          }
+        }
+      } catch { /* best-effort */ }
+    }
+  }
+  return { done, total };
+}
+
 /** Files the mission changed, base..branch. Empty on any git failure — routing is best-effort. */
 function changedFiles(projectDir: string, state: Record<string, unknown> | null): string[] {
   const base = typeof state?.base_sha === 'string' ? state.base_sha : '';
@@ -101,6 +149,11 @@ export function resetMission(projectDir: string, keepLogs: boolean, force?: bool
     if (existsSync(join(root, 'lessons.md'))) kept.push('lessons.md');
     else if (existsSync(join(root, join('logs', 'lessons.md')))) kept.push(join('logs', 'lessons.md'));
   }
+  // index.md is the archive history — after a full reset (missions gone) its entries are dangling (point to deleted report.md). A reset is a fresh start, so clear it; next archive recreates "# Mission index" header.
+  if (removed.includes('missions') && existsSync(join(root, 'index.md'))) {
+    rmSync(join(root, 'index.md'));
+    removed.push('index.md');
+  }
   for (const f of ['config', 'manifest.json', 'backup']) {
     if (existsSync(join(root, f))) kept.push(f);
   }
@@ -122,10 +175,20 @@ export function archiveMission(projectDir: string, mission: string, opts: { dryR
 
   // Closure integrity gate: the trail validates itself before it
   // folds. Dangling links, secrets, or missing evidence fail the archive.
+  // Card-number shapes are warn-only — they do not block archive.
   if (!dryRun) {
     const issues = checkTrail(dir, projectDir);
-    if (issues.length) {
-      throw new Error(`closure integrity gate failed — fix these before archiving:\n${formatIssues(issues)}`);
+    const blocking = issues.filter((i) => i.kind !== 'secret-warn' && i.severity !== 'warn');
+    if (blocking.length) {
+      throw new Error(`closure integrity gate failed — fix these before archiving:\n${formatIssues(blocking)}`);
+    }
+    if (issues.length && blocking.length === 0) {
+      // warn-only — log but do not fail
+      // console.warn is best-effort; archive proceeds
+      try {
+        const warnText = formatIssues(issues);
+        if (warnText) console.warn(`closure integrity warnings (non-blocking):\n${warnText}`);
+      } catch { /* ignore */ }
     }
     // Artifact gate (roadmap v0.8 item 4): Lane 2+ missions must carry
     // plan.md + flows/* evidence — a mission without its trail does not fold.
@@ -133,6 +196,20 @@ export function archiveMission(projectDir: string, mission: string, opts: { dryR
     if (!artifacts.ok) {
       throw new Error(`archive artifact gate failed — missing: ${artifacts.missing.join(', ')} (lane ${artifacts.lane}). Write the evidence trail before archiving.`);
     }
+    // Attestation gate (D4): when attestation.required true, report must be signed and trusted.
+    try {
+      const policy = loadPolicy(projectDir);
+      if (policy?.attestation?.required) {
+        const v = verifyReport(projectDir, dir);
+        if (!v.ok) {
+          throw new Error(`closure integrity gate failed — attestation required but report not signed/trusted: ${v.message}`);
+        }
+      }
+    } catch (e) {
+      if ((e as Error).message.startsWith('closure integrity gate failed — attestation required')) throw e;
+      if ((e as Error).message.startsWith('unknown policy key')) throw e;
+      // other policy load errors are best-effort — do not block archive
+    }
   }
 
   const files = readdirSync(dir);
@@ -249,6 +326,38 @@ export function archiveMission(projectDir: string, mission: string, opts: { dryR
       costSection += renderAdaptationSection(dir);
     } catch { /* best-effort */ }
 
+    // T4: auto-compress when context >80% budget — compress flows → stub, not throw
+    // (record compressed event; closure still recorded; hard gate only at 100%)
+    if (shouldCompress(budget, chars)) {
+      try {
+        const flowsDir = join(dir, 'flows');
+        const wavesDir = join(dir, 'waves');
+        const targetDir = existsSync(flowsDir) ? flowsDir : existsSync(wavesDir) ? wavesDir : null;
+        if (targetDir && existsSync(targetDir)) {
+          const flowFiles = readdirSync(targetDir).filter(f => f.endsWith('.md'));
+          if (flowFiles.length) {
+            const pct = Math.round((chars / budget) * 100);
+            const stub = `# Compressed trail\n\nTrail ${chars} chars exceeds ${pct}% of budget ${budget} (threshold ${compressThreshold(budget)}) — flows archived as stub to preserve budget. Original flows: ${flowFiles.join(', ')}\n`;
+            for (const f of flowFiles) {
+              try { rmSync(join(targetDir, f), { force: true }); } catch {}
+            }
+            writeFileSync(join(targetDir, '00-compressed.md'), stub);
+          }
+        }
+        appendCostEvent(dir, {
+          kind: COMPRESSED_KIND,
+          mission,
+          tokens_est: est,
+          budget: laneBudget,
+          status: 'compressed',
+          context_chars: chars,
+          context_status: ctxStatus,
+          context_metrics: metrics,
+        });
+        costSection += `\n| **Compressed** | yes — ${chars} chars >80% of ${budget} — flows stubbed as 00-compressed.md |`;
+      } catch { /* compress best-effort — never blocks archive */ }
+    }
+
     // Cost Governor: record the closure cost event — the mission's final
     // cost snapshot, folded into report.md with the rest of the trail.
     // (Phase 1 — native cost governor; pure append, never rewrites state.)
@@ -265,6 +374,7 @@ export function archiveMission(projectDir: string, mission: string, opts: { dryR
     // M2: the closure event (with context_status possibly 'over') is recorded
     // BEFORE the hard gate throws — an over-budget closure still leaves a
     // ledger row so the over-budget condition is observable, never erased.
+    // T4: over 80% already compressed above; hard fail only at 100% preserves gate-selftest.
     if (budget && chars > budget) {
       throw new Error(`closure context budget failed — ${footprintLine}. Trim the trail or raise context_budget_chars.`);
     }
@@ -368,17 +478,23 @@ export function archiveMission(projectDir: string, mission: string, opts: { dryR
       });
       if (rb) kept.push(join('missions', mission, rb.file));
       try {
+        const st = tasksFromState(state);
+        const pt = countPlanTasks(dir);
+        const tasks_done = pt.total > 0 ? pt.done : st.done;
+        const tasks_total = pt.total > 0 ? pt.total : st.total;
+        const baseShaForNote = typeof state.base_sha === 'string' ? state.base_sha : undefined;
         writeProvenance(projectDir, dir, {
           mission,
           actor: typeof state.actor === 'string' ? state.actor : '',
           lane: typeof state.lane === 'string' ? state.lane : '',
           mode: typeof state.mode === 'string' ? state.mode : '',
           branch: state.branch,
-          tasks_done: Number(state.tasks_done) || 0,
-          tasks_total: Number(state.tasks_total) || 0,
+          tasks_done,
+          tasks_total,
           evidence: Array.isArray(state.evidence) ? (state.evidence as string[]) : [],
           models: stageModels,
-        });
+          base_sha: baseShaForNote,
+        } as never, baseShaForNote);
         kept.push(join('missions', mission, 'provenance.md'));
       } catch { /* provenance is additive; archive proceeds */ }
     }