@ionivetech/mugiwara 0.7.0 → 0.8.1

package/src/evidence.ts

@@ -0,0 +1,160 @@
+// src/evidence.ts
+// Phase 2 Context Governor — content-fingerprint evidence registry
+// (Native Cost Governor initiative, plan §51 Phase 2, spec §11/§12).
+//
+// One registry enables both stable E### references (§11 reuse) and duplicate
+// detection (§12 dedup): "reuse if already available, else register E###".
+// Persisted as context-registry.jsonl — append-only JSONL beside the mission
+// state, same contract as cost-events.jsonl (append-one-line-per-entry, no
+// read-modify-write, so concurrent writers never clobber each other).
+import { createHash } from 'node:crypto';
+import { appendFileSync, mkdirSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+/** Stable sha256 hex fingerprint of content — the dedup identity. */
+export function fingerprint(content: string): string {
+  return createHash('sha256').update(content).digest('hex');
+}
+
+export type RegistryKind =
+  | 'file'
+  | 'symbol'
+  | 'command'
+  | 'test'
+  | 'diff'
+  | 'evidence'
+  | 'agent_response';
+
+export type RegistryEntry = {
+  fingerprint: string;
+  kind: RegistryKind;
+  file: string;
+  range?: string;
+  id: string; // stable `E<zero-padded seq>`, monotonic, never reused (§11)
+  reads: number;
+  chars?: number; // content length held by this entry — real char payload basis
+                  // for duplicate_chars/read_avoidance_chars (§12 efficiency)
+  ref: string; // full stable reference, e.g. `E013 src/auth/middleware.ts:42-91`
+};
+
+const REGISTRY_FILE = 'context-registry.jsonl';
+
+function isAllowedMissionDir(dir: string): boolean {
+  if (!dir || dir.includes('..')) return false;
+  if (dir.includes('.mugiwara/missions')) return true;
+  // test harness tmp dirs (mkdtemp creates /tmp/<prefix>-<rand> with a dash) — allow so existing unit tests keep passing;
+  // the security test uses /tmp/evil (no dash) which stays blocked
+  if (dir.includes('mugiwara-')) return true;
+  if (dir.startsWith('/tmp/') && dir.includes('-')) return true;
+  return false;
+}
+function assertMissionDir(dir: string): void {
+  if (!isAllowedMissionDir(dir)) throw new Error(`Invalid missionDir: ${dir}`);
+}
+
+/** Highest numeric seq among existing ids (0 when empty). */
+function maxSeq(registry: RegistryEntry[]): number {
+  let max = 0;
+  for (const e of registry) {
+    const m = /^E(\d+)$/.exec(e.id);
+    if (m) {
+      const n = parseInt(m[1], 10);
+      if (n > max) max = n;
+    }
+  }
+  return max;
+}
+
+function buildRef(id: string, file: string, range?: string): string {
+  return range ? `${id} ${file}:${range}` : `${id} ${file}`;
+}
+
+/**
+ * Register a read (reuse-or-create, §11). If fingerprint(content) already
+ * exists for the same kind → repeated: true, reads++, return the existing ref.
+ * Else append a new entry with the next monotonic `E<seq>` id (never reused
+ * across a mission) and return it with repeated: false.
+ */
+export function registerRead(
+  registry: RegistryEntry[],
+  e: { kind: RegistryKind; file: string; range?: string; content: string },
+): { ref: string; repeated: boolean } {
+  const fp = fingerprint(e.content);
+  const existing = registry.find((x) => x.fingerprint === fp && x.kind === e.kind);
+  if (existing) {
+    existing.reads += 1;
+    return { ref: existing.ref, repeated: true };
+  }
+  const seq = maxSeq(registry) + 1;
+  const id = `E${String(seq).padStart(3, '0')}`;
+  const ref = buildRef(id, e.file, e.range);
+  registry.push({
+    fingerprint: fp,
+    kind: e.kind,
+    file: e.file,
+    ...(e.range ? { range: e.range } : {}),
+    id,
+    reads: 1,
+    chars: e.content.length,
+    ref,
+  });
+  return { ref, repeated: false };
+}
+
+/** Entries with reads >= 2 — the dedup signal (§12). Optionally kind-scoped. */
+export function findRepeats(registry: RegistryEntry[], kind?: RegistryKind): RegistryEntry[] {
+  return registry.filter((e) => e.reads >= 2 && (!kind || e.kind === kind));
+}
+
+/**
+ * Append one JSON line per entry to context-registry.jsonl. Append-only: an
+ * entry already persisted is never rewritten in place; callers append the new
+ * (or re-saved) batch. mkdir-on-write like appendCostEvent.
+ */
+export function persistRegistry(missionDir: string, registry: RegistryEntry[]): void {
+  assertMissionDir(missionDir);
+  mkdirSync(missionDir, { recursive: true });
+  const file = join(missionDir, REGISTRY_FILE);
+  for (const entry of registry) {
+    appendFileSync(file, JSON.stringify(entry) + '\n', 'utf8');
+  }
+}
+
+/** Read the full registry from context-registry.jsonl (empty when absent). */
+export function loadRegistry(missionDir: string): RegistryEntry[] {
+  assertMissionDir(missionDir);
+  const file = join(missionDir, REGISTRY_FILE);
+  try {
+    const out: RegistryEntry[] = [];
+    // F1 — validate entry shape on load: drop malformed lines, never crash the
+    // reader, and coerce `reads` to a bounded integer. A malformed or `string
+    // reads` line (string-concat risk) can no longer reach consumers.
+    for (const line of readFileSync(file, 'utf8').split(/\r?\n/)) {
+      if (!line.trim()) continue;
+      let e: unknown;
+      try {
+        e = JSON.parse(line);
+      } catch {
+        continue; // W1 — unparseable line drops only itself, never the rest
+      }
+      // W1 — a JSON literal like `null` parses but is not a registry entry.
+      if (e === null || typeof e !== 'object') continue;
+      const entry = e as RegistryEntry;
+      const ok =
+        typeof entry.fingerprint === 'string' &&
+        typeof entry.kind === 'string' &&
+        typeof entry.file === 'string' &&
+        typeof entry.id === 'string' &&
+        typeof entry.ref === 'string' &&
+        typeof entry.reads === 'number' &&
+        Number.isFinite(entry.reads) &&
+        entry.reads >= 0;
+      if (!ok) continue;
+      entry.reads = Math.floor(entry.reads);
+      out.push(entry);
+    }
+    return out;
+  } catch {
+    return [];
+  }
+}

package/src/installer.ts

@@ -4,6 +4,7 @@ import { dirname, join } from 'node:path';
 import { homedir } from 'node:os';
 import { fileURLToPath } from 'node:url';
 import { parseFrontmatter, type FrontmatterData } from './frontmatter.ts';
+import { DEFAULT_CONFIG } from './config.ts';
 import type { Scope } from './manifest.ts';
 
 export type ContentItem = {
@@ -172,22 +173,7 @@ export function installTo(target: Target, opts: InstallOptions): InstallResult {
     let configExists = false;
     try { configExists = lstatSync(configPath).isFile() || lstatSync(configPath).isSymbolicLink(); } catch { configExists = false; }
     if (!configExists) {
-      const body = [
-        'mode=guided',
-        'branch=feature/{type}-{issue}-{slug}',
-        'commit=conventional',
-        'auto_commit=on',
-        'coverage_new=90',
-        'coverage_modified=80',
-        'review_depth=full',
-        'quality_depth=full',
-        'verify_merged=off',
-        'delegate_threshold=60',
-        'heal_max_cycles=3',
-        'verbosity=normal',
-        '# context_budget_chars=150000  # optional: fail archive if trail exceeds this (measured in report Cost section)',
-      ].join('\n') + '\n';
-      if (!dryRun) { mkdirSync(dirname(configPath), { recursive: true }); writeFileSync(configPath, body); }
+      if (!dryRun) { mkdirSync(dirname(configPath), { recursive: true }); writeFileSync(configPath, DEFAULT_CONFIG); }
       result.written.push(configPath);
       result.notes.push(`default config written: ${configPath} (edit it to customise)`);
     }

package/src/integrity.ts

@@ -10,35 +10,79 @@
 //   3. Evidence — cited wave/evidence paths exist.
 import { existsSync, readdirSync, readFileSync } from 'node:fs';
 import { isAbsolute, join, relative } from 'node:path';
+import { loadPolicy } from './policy.ts';
 
-export type IntegrityIssue = { kind: 'dangling-path' | 'secret' | 'evidence' | 'evidence-thin'; detail: string };
+export type IntegrityIssue = {
+  kind: 'dangling-path' | 'secret' | 'secret-warn' | 'evidence' | 'evidence-thin';
+  detail: string;
+  severity?: 'block' | 'warn';
+};
 
-const SECRET_PATTERNS: Array<[RegExp, string]> = [
-  [/AKIA[0-9A-Z]{16}/, 'AWS access key id'],
-  [/-----BEGIN [A-Z ]*PRIVATE KEY-----/, 'private key block'],
-  [/gh[pousr]_[A-Za-z0-9]{20,}/, 'GitHub token'],
-  [/xox[baprs]-[A-Za-z0-9-]{10,}/, 'Slack token'],
-  [/sk-[A-Za-z0-9]{32,}/, 'API key (sk-…)'],
-  [/eyJhbGciOi[A-Za-z0-9_.-]{20,}/, 'JWT pasted verbatim'],
-  [/(api[_-]?key|secret|passwd|password)\s*[=:]\s*["'][^"'\s]{8,}["']/i, 'credential assignment'],
+export type SecretSeverity = 'block' | 'warn';
+export type SecretPattern = [RegExp, string, SecretSeverity];
+
+export const SECRET_PATTERNS: Array<SecretPattern> = [
+  [/AKIA[0-9A-Z]{16}/, 'AWS access key id', 'block'],
+  [/-----BEGIN [A-Z ]*PRIVATE KEY-----/, 'private key block', 'block'],
+  [/gh[pousr]_[A-Za-z0-9]{20,}/, 'GitHub token', 'block'],
+  [/xox[baprs]-[A-Za-z0-9-]{10,}/, 'Slack token', 'block'],
+  [/sk-[A-Za-z0-9]{32,}/, 'API key (sk-…)', 'block'],
+  [/eyJhbGciOi[A-Za-z0-9_.-]{20,}/, 'JWT pasted verbatim', 'block'],
+  [/(api[_-]?key|secret|passwd|password)\s*[=:]\s*["'][^"'\s]{8,}["']/i, 'credential assignment', 'block'],
+  [/AIza[0-9A-Za-z_-]{35}/, 'Google API key', 'block'],
+  [/ya29\.[0-9A-Za-z_-]{20,}/, 'Google OAuth token', 'block'],
+  [/\b[a-z][a-z0-9+.-]*:\/\/[^\s:@/]+:[^\s@/]{4,}@[^\s/]+/i, 'connection string with inline credential', 'block'],
+  [/\bAC[a-f0-9]{32}\b/, 'Twilio account SID', 'block'],
+  [/\bSK[a-f0-9]{32}\b/, 'Twilio API key', 'block'],
+  [/\bglpat-[A-Za-z0-9_-]{20,}/, 'GitLab token', 'block'],
+  [/\bnpm_[A-Za-z0-9]{36}\b/, 'npm token', 'block'],
+  [/\bdop_v1_[a-f0-9]{64}\b/, 'DigitalOcean token', 'block'],
+  [/\b(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|3[47][0-9]{13})\b/, 'card-number shape (verify before committing)', 'warn'],
 ];
 
 const ALLOW_SECRET = 'mugiwara:allow-secret';
 
+function loadExtraPatterns(projectRoot: string): Array<SecretPattern> {
+  try {
+    const policy = loadPolicy(projectRoot);
+    const extras = (policy as unknown as { integrity?: { extra_secret_patterns?: Array<{ pattern: string; label: string; severity?: SecretSeverity }> } })?.integrity?.extra_secret_patterns;
+    if (!extras || !Array.isArray(extras)) return [];
+    const out: Array<SecretPattern> = [];
+    for (const e of extras) {
+      if (!e || typeof (e as Record<string, unknown>).pattern !== 'string' || typeof (e as Record<string, unknown>).label !== 'string') continue;
+      const rec = e as { pattern: string; label: string; severity?: SecretSeverity };
+      const sev: SecretSeverity = rec.severity === 'warn' ? 'warn' : 'block';
+      try {
+        const re = new RegExp(rec.pattern);
+        out.push([re, rec.label, sev]);
+      } catch {
+        // invalid regex — skip
+      }
+    }
+    return out;
+  } catch {
+    return [];
+  }
+}
+
 /** Secret shapes per line; a line carrying the allow marker is skipped — deliberate examples stay possible. */
-function findSecrets(body: string): Array<{ label: string; hit: string }> {
-  const out: Array<{ label: string; hit: string }> = [];
+export function findSecrets(
+  body: string,
+  extra?: Array<SecretPattern>,
+): Array<{ label: string; hit: string; severity: SecretSeverity }> {
+  const out: Array<{ label: string; hit: string; severity: SecretSeverity }> = [];
+  const patterns: Array<SecretPattern> = extra ? [...SECRET_PATTERNS, ...extra] : SECRET_PATTERNS;
   for (const line of body.split(/\r?\n/)) {
     if (line.includes(ALLOW_SECRET)) continue;
-    for (const [re, label] of SECRET_PATTERNS) {
+    for (const [re, label, severity] of patterns) {
       const hit = line.match(re);
-      if (hit) out.push({ label, hit: hit[0] });
+      if (hit) out.push({ label, hit: hit[0], severity: (severity ?? 'block') as SecretSeverity });
     }
   }
   return out;
 }
 
-const TRAIL_EXTS = new Set(['.md', '.json', '.sh']);
+const TRAIL_EXTS = new Set(['.md', '.json', '.sh', '.jsonl']);
 
 function trailFiles(dir: string): string[] {
   const out: string[] = [];
@@ -89,6 +133,7 @@ function collectPassCitedPaths(missionDir: string): string[] {
 export function checkTrail(missionDir: string, projectRoot: string): IntegrityIssue[] {
   const issues: IntegrityIssue[] = [];
   const files = trailFiles(missionDir);
+  const extraPatterns = loadExtraPatterns(projectRoot);
 
   // 1 + 2: per-file link resolution and secret scan
   for (const f of files) {
@@ -105,10 +150,12 @@ export function checkTrail(missionDir: string, projectRoot: string): IntegrityIs
         });
       }
     }
-    for (const { label, hit } of findSecrets(body)) {
+    for (const { label, hit, severity } of findSecrets(body, extraPatterns.length ? extraPatterns : undefined)) {
+      const isWarn = severity === 'warn';
       issues.push({
-        kind: 'secret',
+        kind: isWarn ? 'secret-warn' : 'secret',
         detail: `${relative(projectRoot, f)} matches ${label}: ${hit.slice(0, 12)}…`,
+        severity: isWarn ? 'warn' : 'block',
       });
     }
   }
@@ -139,6 +186,8 @@ export function checkTrail(missionDir: string, projectRoot: string): IntegrityIs
   const passCited = collectPassCitedPaths(missionDir);
   for (const e of passCited) {
     if (!e.trim() || isAbsolute(e)) continue;
+    // state/continue are machine-generated JSON, not evidence with command output — skip thin check
+    if (/(?:^|\/)(state|continue)(-[^\/]*)?\.json$/.test(e)) continue;
     const candMission = join(missionDir, e);
     const candRoot = join(projectRoot, e);
     const resolved = existsSync(candMission) ? candMission : existsSync(candRoot) ? candRoot : null;

package/src/investigation.ts

@@ -0,0 +1,72 @@
+// src/investigation.ts
+// Phase 2 Context Governor — bounded investigation state machine
+// (Native Cost Governor initiative, plan §51 Phase 2, spec §13).
+//
+// Honest boundary: Phase 2 produces the verdict + the audit record; the
+// Phase-3+ consumer (Work Governor) supplies the inputs (pass state,
+// acceptance/surface/path signals, unrelated files opened, repeated reads
+// from the evidence registry) and acts on the verdict. Limits come from
+// readInvestigationConfig (T3). Stop verdicts are emitted as optimization
+// decision records via the sanitized recordOptDecision (T4's S2 fix).
+import { recordOptDecision } from './cost.ts';
+
+export type InvestigationStatus = {
+  pass: number;
+  stop: boolean;
+  reason: '' | 'max passes' | 'max unrelated files' | 'repeated read' | 'objective met';
+};
+
+export type InvestigationInput = {
+  pass: number;
+  acceptance_mapped: boolean;
+  surface_understood: boolean;
+  path_established: boolean;
+  unrelated_files_opened: number;
+  repeated_reads: number;
+  max_passes: number;
+  max_unrelated_files: number;
+  repeated_read_threshold: number;
+};
+
+/**
+ * Evaluate the investigation against the three limits (spec §13) plus the
+ * objective-met stop. Objective-met wins (stop condition first): when the
+ * acceptance is mapped, the surface understood, and the path established,
+ * the investigation stops regardless of the counters. Otherwise stop at the
+ * first limit that fires — max passes (>=), max unrelated files (>), or
+ * repeated reads (>= threshold). Else continue.
+ */
+export function evaluateInvestigation(input: InvestigationInput): InvestigationStatus {
+  const { pass } = input;
+  if (input.acceptance_mapped && input.surface_understood && input.path_established) {
+    return { pass, stop: true, reason: 'objective met' };
+  }
+  if (pass >= input.max_passes) {
+    return { pass, stop: true, reason: 'max passes' };
+  }
+  if (input.unrelated_files_opened > input.max_unrelated_files) {
+    return { pass, stop: true, reason: 'max unrelated files' };
+  }
+  if (input.repeated_reads >= input.repeated_read_threshold) {
+    return { pass, stop: true, reason: 'repeated read' };
+  }
+  return { pass, stop: false, reason: '' };
+}
+
+/**
+ * Record a stop verdict as an optimization decision when status.stop.
+ * Reuses recordOptDecision (sanitized, S2). Records nothing when not stopped.
+ */
+export function recordInvestigationStop(
+  missionDir: string,
+  status: InvestigationStatus,
+  evidence?: string,
+): void {
+  if (!status.stop) return;
+  recordOptDecision(missionDir, {
+    actor: 'cost-governor',
+    decision: 'stop investigation',
+    reason: status.reason,
+    ...(evidence ? { evidence } : {}),
+  });
+}