elanous 0.2.40 → 0.2.41

package/scripts/release-loop/gate-shard-load-flaky.ts

@@ -0,0 +1,256 @@
+#!/usr/bin/env bun
+import { readFileSync, writeFileSync } from 'node:fs';
+
+function decodeXml(value: string): string {
+  return value.replace(/&quot;/g, '"').replace(/&apos;/g, "'").replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>').replace(/&amp;/g, '&');
+}
+
+/** Remove legal comments and reduce CDATA before tag parsing, without accepting markup inside tags. */
+function stripCommentsAndCdata(xml: string): string | null {
+  let source = '';
+  let inMarkup = false;
+  let quote: '"' | "'" | undefined;
+  const offset = xml.startsWith('\uFEFF') ? 1 : 0;
+  for (let index = offset; index < xml.length;) {
+    if (xml.startsWith('<!--', index)) {
+      if (inMarkup) return null;
+      const end = xml.indexOf('-->', index + 4);
+      if (end < 0 || xml.slice(index + 4, end).includes('--')) return null;
+      index = end + 3;
+      continue;
+    }
+    if (xml.startsWith('<?', index)) {
+      const end = xml.indexOf('?>', index + 2);
+      if (inMarkup || index !== offset || end < 0) return null;
+      const declaration = xml.slice(index, end + 2);
+      if (!/^<\?xml\s+version=["']1\.0["'](?:\s+encoding=["'][A-Za-z][A-Za-z0-9._-]*["'])?(?:\s+standalone=["'](?:yes|no)["'])?\s*\?>$/.test(declaration)) return null;
+      index = end + 2;
+      continue;
+    }
+    if (xml.startsWith('<![CDATA[', index)) {
+      if (inMarkup) return null;
+      const end = xml.indexOf(']]>', index + 9);
+      if (end < 0) return null;
+      source += xml.slice(index + 9, end).trim() ? '__CDATA_TEXT__' : ' ';
+      index = end + 3;
+      continue;
+    }
+    const character = xml[index]!;
+    if (inMarkup) {
+      if (quote) {
+        if (character === quote) quote = undefined;
+        else if (character === '<') return null;
+      } else if (character === '"' || character === "'") quote = character;
+      else if (character === '>') inMarkup = false;
+      else if (character === '<') return null;
+    } else if (character === '<') inMarkup = true;
+    source += character;
+    index++;
+  }
+  return source;
+}
+
+type JunitCase = { file?: string; id?: string; failures: number; errors: number; skipped: number };
+type JunitDocument = { cases: JunitCase[]; suiteFailed: boolean; unattributedSuiteFailure: boolean };
+function parseJunit(xml: string): JunitDocument | null {
+  if (/<!DOCTYPE/i.test(xml)) return null;
+  for (const character of xml) {
+    const codepoint = character.codePointAt(0)!;
+    const xml10Character = codepoint === 0x9 || codepoint === 0xa || codepoint === 0xd
+      || (codepoint >= 0x20 && codepoint <= 0xd7ff) || (codepoint >= 0xe000 && codepoint <= 0xfffd)
+      || (codepoint >= 0x10000 && codepoint <= 0x10ffff);
+    if (!xml10Character) return null;
+  }
+  const source = stripCommentsAndCdata(xml);
+  if (source === null) return null;
+  if (/<\?|\?>/.test(source)) return null;
+  if (/<!--|-->|<!\[CDATA\[|\]\]>/.test(source)) return null;
+  const stack: Array<{
+    name: string;
+    file?: string;
+    suiteTitle?: string;
+    declaredTests?: number;
+    declaredFailures?: number;
+    declaredErrors?: number;
+    testCount: number;
+    caseStart: number;
+  }> = [];
+  const cases: JunitCase[] = [];
+  let roots = 0;
+  let sawSuite = false;
+  let suiteFailed = false;
+  let unattributedSuiteFailure = false;
+  let rootName: string | undefined;
+  let cursor = 0;
+  if (/&(?!(?:amp|lt|gt|quot|apos|#\d+|#x[\da-f]+);)/i.test(source)) return null;
+  for (const entity of source.matchAll(/&#(x[\da-f]+|\d+);/gi)) {
+    const codepoint = entity[1]!.startsWith('x') ? Number.parseInt(entity[1]!.slice(1), 16) : Number.parseInt(entity[1]!, 10);
+    const xml10Character = codepoint === 0x9 || codepoint === 0xa || codepoint === 0xd
+      || (codepoint >= 0x20 && codepoint <= 0xd7ff) || (codepoint >= 0xe000 && codepoint <= 0xfffd)
+      || (codepoint >= 0x10000 && codepoint <= 0x10ffff);
+    if (!Number.isInteger(codepoint) || !xml10Character) return null;
+  }
+  for (const match of source.matchAll(/<([^<>]+)>/g)) {
+    const between = source.slice(cursor, match.index);
+    if (between.includes('<') || between.includes(']]>') || (!stack.length && between.trim())) return null;
+    cursor = match.index! + match[0].length;
+    const body = match[1]!.trim();
+    // Comments and CDATA have already been stripped above, and the one XML declaration was
+    // removed before tokenization. Every other declaration is unsupported and invalid here.
+    if (!body || body.startsWith('!')) return null;
+    const close = body.startsWith('/');
+    const selfClose = body.endsWith('/');
+    const tagMatch = /^\/?([A-Za-z_:][\w:.-]*)/.exec(body);
+    if (!tagMatch) return null;
+    const name = tagMatch[1]!;
+    const attrsText = body.slice(tagMatch[0].length, body.length - (selfClose ? 1 : 0));
+    const attrs: Record<string, string> = {};
+    const attrPattern = /([A-Za-z_:][\w:.-]*)\s*=\s*(?:"([^"]*)"|'([^']*)')/g;
+    for (const attr of attrsText.matchAll(attrPattern)) {
+      if (attrs[attr[1]!] !== undefined) return null;
+      attrs[attr[1]!] = decodeXml(attr[2] ?? attr[3] ?? '');
+    }
+    if (!close && attrsText.replace(attrPattern, '').trim()) return null;
+    if (close && attrsText.trim()) return null;
+    if (close) {
+      if (selfClose) return null;
+      const closed = stack.pop();
+      if (!closed || closed.name !== name || (closed.declaredTests !== undefined && closed.declaredTests !== closed.testCount)) return null;
+      if (closed.declaredFailures !== undefined
+        && closed.declaredFailures !== cases.slice(closed.caseStart).reduce((count, testcase) => count + testcase.failures, 0)) return null;
+      if (closed.declaredErrors !== undefined
+        && closed.declaredErrors !== cases.slice(closed.caseStart).reduce((count, testcase) => count + testcase.errors, 0)) return null;
+      continue;
+    }
+    if (name === 'testcase' && stack.at(-1)?.name !== 'testsuite') return null;
+    if ((name === 'failure' || name === 'error') && stack.at(-1)?.name !== 'testcase') return null;
+    if (!stack.length) { roots++; rootName = name; }
+    let file = attrs.file;
+    let declaredTests: number | undefined;
+    let declaredFailures: number | undefined;
+    let declaredErrors: number | undefined;
+    if ((name === 'testsuite' || name === 'testsuites') && attrs.tests !== undefined) {
+      declaredTests = Number(attrs.tests);
+      if (!Number.isSafeInteger(declaredTests) || declaredTests < 0) return null;
+    }
+    if (name === 'testsuite' || name === 'testsuites') {
+      for (const [attribute, set] of [['failures', (value: number) => { declaredFailures = value; }], ['errors', (value: number) => { declaredErrors = value; }]] as const) {
+        const count = attrs[attribute];
+        if (count === undefined) continue;
+        if (!/^\d+$/.test(count) || !Number.isSafeInteger(Number(count))) return null;
+        set(Number(count));
+      }
+    }
+    if (name === 'testsuite') {
+      file ??= attrs.name;
+      sawSuite = true;
+      if ((declaredFailures ?? 0) > 0 || (declaredErrors ?? 0) > 0) suiteFailed = true;
+      const inherited = [...stack].reverse().find((entry) => entry.name === 'testsuite')?.file;
+      if (!file && inherited) file = inherited;
+    } else {
+      const inherited = [...stack].reverse().find((entry) => entry.name === 'testsuite')?.file;
+      if (!file && inherited) file = inherited;
+      if (name === 'testsuites' && ((declaredFailures ?? 0) > 0 || (declaredErrors ?? 0) > 0)) suiteFailed = true;
+    }
+    if (name === 'testcase') {
+      for (const ancestor of stack) if (ancestor.name === 'testsuite' || ancestor.name === 'testsuites') ancestor.testCount++;
+      const suiteEntries = stack.filter((entry) => entry.name === 'testsuite');
+      // Some JUnit writers repeat the file path as the outer suite name. Keep the
+      // remaining hierarchy so sibling suites with the same testcase name stay distinct.
+      const identitySuites = suiteEntries[0]?.suiteTitle === file ? suiteEntries.slice(1) : suiteEntries;
+      const suiteNames = identitySuites.map((entry) => entry.suiteTitle ?? '').filter(Boolean);
+      const testcaseName = attrs.name;
+      const idName = [...suiteNames, ...(testcaseName ? [testcaseName] : [])].filter(Boolean).join(' > ');
+      cases.push({ file, ...(file && idName ? { id: `${file} > ${idName}` } : {}), failures: 0, errors: 0, skipped: 0 });
+    }
+    if (name === 'failure' || name === 'error' || name === 'skipped') {
+      if (stack.some((entry) => entry.name === 'testcase')) {
+        const current = cases.at(-1);
+        if (current) current[name === 'failure' ? 'failures' : name === 'error' ? 'errors' : 'skipped']++;
+      } else if (stack.some((entry) => entry.name === 'testsuite')) { suiteFailed = true; unattributedSuiteFailure = true; }
+    }
+    if (!selfClose) stack.push({ name, file, ...(name === 'testsuite' && attrs.name ? { suiteTitle: attrs.name } : {}), ...(declaredTests !== undefined ? { declaredTests } : {}),
+      ...(declaredFailures !== undefined ? { declaredFailures } : {}), ...(declaredErrors !== undefined ? { declaredErrors } : {}),
+      testCount: 0, caseStart: cases.length });
+    else if ((declaredTests !== undefined && declaredTests !== 0) || (declaredFailures !== undefined && declaredFailures !== 0)
+      || (declaredErrors !== undefined && declaredErrors !== 0)) return null;
+  }
+  const trailingText = source.slice(cursor);
+  if (trailingText.includes('<') || trailingText.includes(']]>') || trailingText.trim() || stack.length || roots !== 1
+    || !['testsuite', 'testsuites'].includes(rootName ?? '') || !sawSuite || cases.length === 0) return null;
+  return { cases, suiteFailed, unattributedSuiteFailure };
+}
+
+/** A null count means the XML was malformed or incomplete. */
+export function junitTestcaseCount(xml: string): number | null {
+  return parseJunit(xml)?.cases.length ?? null;
+}
+
+/** Unique file attribution for every testcase; null means malformed, incomplete, or unattributed evidence. */
+export function junitTestcaseFiles(xml: string): string[] | null {
+  const parsed = parseJunit(xml);
+  if (!parsed || parsed.unattributedSuiteFailure || parsed.cases.some((testcase) => !testcase.file)) return null;
+  return [...new Set(parsed.cases.map((testcase) => testcase.file!))].sort();
+}
+
+/** Named failing/error cases, preserving duplicate identities so a retry must rerun each occurrence. */
+export function junitFailedCaseIds(xml: string): string[] | null {
+  const parsed = parseJunit(xml);
+  if (!parsed || parsed.unattributedSuiteFailure) return null;
+  const failed = parsed.cases.filter((testcase) => testcase.failures > 0 || testcase.errors > 0);
+  if (failed.some((testcase) => !testcase.id)) return null;
+  const identityCounts = new Map<string, number>();
+  for (const testcase of parsed.cases) if (testcase.id) identityCounts.set(testcase.id, (identityCounts.get(testcase.id) ?? 0) + 1);
+  // A duplicate testcase identity cannot show which occurrence failed. Treat it as
+  // ambiguous evidence instead of allowing a passing sibling to clear the failure.
+  if (failed.some((testcase) => identityCounts.get(testcase.id!)! > 1)) return null;
+  return failed.map((testcase) => testcase.id!).sort();
+}
+
+/** Named cases that completed without failure, error, or skip. */
+export function junitPassedCaseIds(xml: string): string[] | null {
+  const parsed = parseJunit(xml);
+  if (!parsed) return null;
+  return parsed.cases.flatMap((testcase) => testcase.id && !testcase.failures && !testcase.errors && !testcase.skipped ? [testcase.id] : []).sort();
+}
+
+/** Attribute files for testcases that have a failure/error node; malformed XML returns null (fail closed). */
+export function failedJunitFiles(xml: string, allowed: readonly string[]): string[] | null {
+  const allow = new Set(allowed);
+  const parsed = parseJunit(xml);
+  if (!parsed || parsed.unattributedSuiteFailure || parsed.cases.some((testcase) => !testcase.file || !allow.has(testcase.file))) return null;
+  return [...new Set(parsed.cases.filter((testcase) => testcase.failures > 0 || testcase.errors > 0).map((testcase) => testcase.file!))].sort();
+}
+
+interface RetryVerdict { clean: boolean; tests: number; failures: number; errors: number; }
+export function cleanRetryVerdict(rc: number | undefined, log: string | undefined, junit: string | undefined, file: string, expectedFailureIds?: readonly string[]): RetryVerdict {
+  const summary = log?.match(/Ran (\d+) tests? across ([1-9]\d*) files?\.?/);
+  const pass = Number(log?.match(/^\s*(\d+)\s+pass\s*$/m)?.[1] ?? NaN);
+  const failures = Number(log?.match(/^\s*(\d+)\s+fail\s*$/m)?.[1] ?? NaN);
+  const errors = Number(log?.match(/^\s*(\d+)\s+errors?\s*$/m)?.[1] ?? 0);
+  const tests = Number(summary?.[1] ?? 0);
+  const parsed = junit ? parseJunit(junit) : null;
+  const cases = parsed?.cases;
+  const exactFile = cases?.every((testcase) => testcase.file === file) ?? false;
+  const hasFailure = parsed?.suiteFailed || (cases?.some((testcase) => testcase.failures > 0 || testcase.errors > 0 || testcase.skipped > 0) ?? true);
+  const rerunIds = junitPassedCaseIds(junit ?? '') ?? [];
+  const rerunCounts = new Map<string, number>();
+  for (const id of rerunIds) rerunCounts.set(id, (rerunCounts.get(id) ?? 0) + 1);
+  const expectedCounts = new Map<string, number>();
+  for (const id of expectedFailureIds ?? []) expectedCounts.set(id, (expectedCounts.get(id) ?? 0) + 1);
+  const reranOriginalFailures = expectedFailureIds === undefined
+    || (expectedFailureIds.length > 0 && [...expectedCounts].every(([id, count]) => (rerunCounts.get(id) ?? 0) >= count));
+  const clean = rc === 0 && tests > 0 && Number(summary?.[2]) === 1 && pass === tests && failures === 0 && errors === 0
+    && cases?.length === tests && exactFile && !hasFailure && reranOriginalFailures && !/^# Unhandled error between tests/m.test(log ?? '');
+  return { clean, tests, failures: Number.isFinite(failures) ? failures : -1, errors: Number.isFinite(errors) ? errors : -1 };
+}
+
+if (import.meta.main) {
+  const [xmlPath, outputPath, manifestPath] = process.argv.slice(2);
+  if (!xmlPath || !outputPath || !manifestPath) process.exit(2);
+  let result: string[] | null = null;
+  try { result = failedJunitFiles(readFileSync(xmlPath, 'utf8'), readFileSync(manifestPath, 'utf8').split(/\r?\n/).filter(Boolean)); } catch { /* fail closed */ }
+  if (!result) process.exit(1);
+  writeFileSync(outputPath, result.length ? `${result.join('\n')}\n` : '');
+}

package/scripts/release-loop/gate-triage-node.ts

@@ -1,6 +1,8 @@
 #!/usr/bin/env bun
-import { readFileSync } from 'node:fs';
+import { readFileSync, readdirSync, statSync } from 'node:fs';
+import { join } from 'node:path';
 import { debug } from '../../src/debug/log.js';
+import { effectiveInstanceRoot } from '../../src/instance/resolve.js';
 import { registerStandaloneLogSink } from '../../src/domains/standalone-log-sink.js';
 import type { GateResult } from './gate-node.js';
 import { emitNodeResult, readGraphContext, type GraphContext } from './node-verdict.js';
@@ -39,8 +41,32 @@ export interface GateTriageShadow {
     loadFlaky: string[] | null;
     stalledEnv: Array<{ file: string; local: string }>;
   };
-  /** Shadow always goes to the existing failure exit. */
-  outcome: 'fail';
+  /** Classification succeeded; the release graph still routes to its failure exit. */
+  outcome: 'ok' | 'error';
+}
+
+/** A prescription is selected from observed gate facts. Execution waits until the failed run is terminal. */
+export type GatePrescription =
+  | { action: 'rerun-pod'; reason: string; runId: string | null; commit: string }
+  | { action: 'repair-card'; reason: string; runId: string | null; commit: string; failureIds: string[] }
+  | { action: 'decision-card'; reason: string; runId: string | null; commit: string }
+  | { action: 'none'; reason: string; runId: string | null; commit: string };
+
+export function chooseGatePrescription(context: GraphContext, triage: GateTriageShadow): GatePrescription {
+  const runId = context.runId ?? null;
+  const common = { runId, commit: triage.commit };
+  const gate = context.outputs.gate;
+  const unattributed = (Array.isArray(gate?.stalledShards)
+    && gate.stalledShards.some((shard: unknown) => shard && typeof shard === 'object'
+      && (shard as { reason?: unknown }).reason === 'unattributed'))
+    || (typeof gate?.error === 'string' && /failure attribution (?:incomplete|summary=)|unattributed/.test(gate.error));
+  if (triage.classification === 'exhausted')
+    return { ...common, action: 'decision-card', reason: 'same gate symptom recurred twice' };
+  if (unattributed && triage.introduced.length === 0 && context.input.gateExecutor === 'host')
+    return { ...common, action: 'rerun-pod', reason: 'host gate attribution incomplete' };
+  if (triage.classification === 'regression' && triage.introduced.length > 0)
+    return { ...common, action: 'repair-card', reason: 'introduced gate regressions', failureIds: [...triage.introduced] };
+  return { ...common, action: 'none', reason: `no evidence-backed automatic remedy for ${triage.classification}` };
 }
 
 function timeoutShape(gate: GateResult): boolean {
@@ -104,11 +130,12 @@ export function classifyGateShadow(gate: GateResult, evidence: GateTriageEvidenc
       environmentFailureIds: evidence.environmentFailureIds ? [...evidence.environmentFailureIds] : null,
       loadFlaky: gate.loadFlaky ? [...gate.loadFlaky] : null,
       stalledEnv: gate.stalledEnv.map((item) => ({ file: item.file, local: item.local })) },
-    outcome: 'fail' };
+    outcome: 'ok' };
 }
 
 /** Read only the completed gate output. The graph wiring is a separate change after M3. */
-export function gateTriageShadowFromContext(context: GraphContext, evidence?: GateTriageEvidence): GateTriageShadow {
+export function gateTriageShadowFromContext(context: GraphContext, evidence?: GateTriageEvidence,
+  ledgerHistory: NonNullable<GateTriageEvidence['priorDecisions']> = []): GateTriageShadow {
   const value = context.outputs.gate;
   if (!value || typeof value.commit !== 'string' || !Array.isArray(value.introduced)
     || !value.introduced.every((item) => typeof item === 'string')
@@ -140,6 +167,7 @@ export function gateTriageShadowFromContext(context: GraphContext, evidence?: Ga
       && item.failureIds.every((id: unknown) => typeof id === 'string'))
       ? { priorDecisions: supplied.priorDecisions as GateTriageEvidence['priorDecisions'] } : {}),
   };
+  if (!observed.priorDecisions && ledgerHistory.length) observed.priorDecisions = ledgerHistory;
   // The graph adapter maps a measured gate regression to `fail` for edge routing.
   // Restore the gate verdict before classifying the shadow; keep the context untouched.
   const gate = value.outcome === 'fail' ? { ...value, outcome: 'regression' } : value;
@@ -150,6 +178,34 @@ export function logGateTriageShadow(result: GateTriageShadow, version: string):
   debug.log('graph.ground', 'gate-triage-shadow', { version, ...result });
 }
 
+/** Last two completed triage facts from distinct release runs on this exact cut commit. */
+export function priorGateDecisionsFromLedger(context: GraphContext, root = effectiveInstanceRoot()): NonNullable<GateTriageEvidence['priorDecisions']> {
+  const commit = context.outputs.gate?.commit;
+  if (typeof commit !== 'string' || !commit) return [];
+  const dir = join(root, 'graph-runs', 'release-loop');
+  try {
+    const files = readdirSync(dir).filter(name => name.endsWith('.json'))
+      .map(name => ({ name, mtime: statSync(join(dir, name)).mtimeMs }))
+      .sort((a, b) => b.mtime - a.mtime).slice(0, 200);
+    const prior: NonNullable<GateTriageEvidence['priorDecisions']> = [];
+    for (const { name } of files) {
+      try {
+        const state = JSON.parse(readFileSync(join(dir, name), 'utf8')) as Record<string, unknown>;
+        if (state.runId === context.runId || (state.input as Record<string, unknown> | undefined)?.version !== context.input.version) continue;
+        const nodes = Array.isArray(state.nodes) ? state.nodes as Array<Record<string, unknown>> : [];
+        const node = [...nodes].reverse().find(item => item.nodeId === 'gate-triage' && item.executed === true);
+        const raw = node?.output;
+        const output = typeof raw === 'string' ? JSON.parse(raw) as Record<string, unknown> : raw as Record<string, unknown> | undefined;
+        if (output?.commit !== commit || !['unmeasured', 'load', 'env', 'regression', 'clean'].includes(String(output.classification))
+          || !Array.isArray(output.failureIds) || !output.failureIds.every(id => typeof id === 'string')) continue;
+        prior.push({ classification: output.classification as GateTriageClass, commit, failureIds: output.failureIds as string[] });
+        if (prior.length === 2) break;
+      } catch { /* Other runs may be incomplete while their ledger is being written. */ }
+    }
+    return prior.reverse();
+  } catch { return []; }
+}
+
 /** A saved verdict has fewer fields than the live gate output. Keep the absent evidence absent. */
 export function gateTriageShadowFromSavedVerdict(value: unknown): { version: string; result: GateTriageShadow } {
   if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('invalid saved gate verdict');
@@ -174,20 +230,24 @@ if (import.meta.main) {
   try {
     const savedPath = process.argv[2] === '--gate-verdict' ? process.argv[3] : undefined;
     if (process.argv[2] && !savedPath) throw new Error('usage: gate-triage-node.ts [--gate-verdict <path>]');
+    const context = savedPath ? undefined : readGraphContext();
+    const history = context ? priorGateDecisionsFromLedger(context) : [];
     const { version, result } = savedPath
       ? gateTriageShadowFromSavedVerdict(JSON.parse(readFileSync(savedPath, 'utf8')))
-      : (() => { const context = readGraphContext(); return { version: context.input.version,
-        result: gateTriageShadowFromContext(context) }; })();
+      : { version: context!.input.version, result: gateTriageShadowFromContext(context!, undefined, history) };
     logGateTriageShadow(result, version);
-    emitNodeResult({ ...result, verdict: 'fail', summary: `gate-triage shadow: ${result.classification} — ${result.reason}` });
+    const prescription = context ? chooseGatePrescription(context, result) : undefined;
+    if (prescription) debug.log('release-loop.heal', 'prescribed-shadow', { version, ...prescription });
+    emitNodeResult({ ...result, ...(prescription ? { prescription } : {}), verdict: 'ok',
+      summary: `gate-triage shadow: ${result.classification} — ${result.reason}` });
   } catch (error) {
     const reason = error instanceof Error ? error.message : String(error);
-    const result: GateTriageShadow = { outcome: 'fail', classification: 'unmeasured', reason,
+    const result: GateTriageShadow = { outcome: 'error', classification: 'unmeasured', reason,
       commit: 'unknown', introduced: [], stalledFiles: [], failureIds: [],
       evidence: { highLoad: null, isolatedReproductions: null, isolatedCheckedFailureIds: null,
         isolatedPassedFiles: null, environmentFailureIds: null, loadFlaky: null, stalledEnv: [] } };
     logGateTriageShadow(result, 'unknown');
     emitNodeResult({ ...result, verdict: 'fail', summary: `gate-triage shadow: ${reason}` });
+    process.exitCode = 1;
   }
-  process.exitCode = 1;
 }