@adia-ai/a2ui-mcp 0.0.5 → 0.1.1

package/scripts/semantic-stats.mjs

@@ -0,0 +1,113 @@
+#!/usr/bin/env node
+/**
+ * Compare two eval-diff run artifacts (JSON) and surface verdict-distribution
+ * + pass-rate deltas. Used to verify "no unexplained regressions" between
+ * baseline and a candidate gating-mode run, per Phase 2 exit criteria of
+ * `docs/specs/semantic-validator.md`.
+ *
+ * Usage:
+ *   node packages/a2ui/mcp/scripts/semantic-stats.mjs <baseline.json> <candidate.json>
+ *
+ * Examples:
+ *   # Compare structural-only baseline to combined-gating candidate
+ *   node packages/a2ui/mcp/scripts/semantic-stats.mjs \
+ *     evals/mcp/runs/<baseline-stamp>/zettel.json \
+ *     evals/mcp/runs/<candidate-stamp>/zettel.json
+ *
+ * The script is read-only; it writes nothing to disk. Output is markdown
+ * to stdout — pipe into a file or a journal entry as needed.
+ */
+import { readFile } from 'node:fs/promises';
+
+const args = process.argv.slice(2);
+if (args.length !== 2) {
+  console.error('Usage: semantic-stats.mjs <baseline.json> <candidate.json>');
+  process.exit(2);
+}
+
+const [basePath, candPath] = args;
+const base = JSON.parse(await readFile(basePath, 'utf8'));
+const cand = JSON.parse(await readFile(candPath, 'utf8'));
+
+const fmt = (v) => v == null ? '—' : String(v);
+const delta = (a, b) => {
+  if (a == null || b == null) return '—';
+  const d = Math.round((b - a) * 10) / 10;
+  return d > 0 ? `+${d}` : String(d);
+};
+
+console.log(`# Semantic stats — baseline vs candidate\n`);
+console.log(`- Baseline:  \`${basePath}\` (label=${fmt(base.label)})`);
+console.log(`- Candidate: \`${candPath}\` (label=${fmt(cand.label)})\n`);
+
+console.log(`## Aggregate deltas\n`);
+console.log(`| metric | baseline | candidate | delta |`);
+console.log(`|---|---:|---:|---:|`);
+console.log(`| total | ${base.total} | ${cand.total} | ${delta(base.total, cand.total)} |`);
+console.log(`| coverage % | ${base.coverage} | ${cand.coverage} | ${delta(base.coverage, cand.coverage)} |`);
+console.log(`| emitted | ${base.emitted}/${base.total} | ${cand.emitted}/${cand.total} | ${delta(base.emitted, cand.emitted)} |`);
+console.log(`| avgScoreWhenEmitted | ${base.avgScoreWhenEmitted} | ${cand.avgScoreWhenEmitted} | ${delta(base.avgScoreWhenEmitted, cand.avgScoreWhenEmitted)} |`);
+console.log(`| avgF1WhenEmitted | ${base.avgF1WhenEmitted} | ${cand.avgF1WhenEmitted} | ${delta(base.avgF1WhenEmitted, cand.avgF1WhenEmitted)} |`);
+console.log(`| pass rate % | ${base.passRate} | ${cand.passRate} | ${delta(base.passRate, cand.passRate)} |`);
+
+if (cand.passRateStructural != null) {
+  console.log(`| pass rate % (structural-only on candidate) | ${base.passRate} | ${cand.passRateStructural} | ${delta(base.passRate, cand.passRateStructural)} |`);
+}
+
+if (base.semantic || cand.semantic) {
+  console.log(`| avgSemanticScore | ${fmt(base.semantic?.avgSemanticScore)} | ${fmt(cand.semantic?.avgSemanticScore)} | ${delta(base.semantic?.avgSemanticScore, cand.semantic?.avgSemanticScore)} |`);
+  console.log(`| avgCombinedScore | ${fmt(base.semantic?.avgCombinedScore)} | ${fmt(cand.semantic?.avgCombinedScore)} | ${delta(base.semantic?.avgCombinedScore, cand.semantic?.avgCombinedScore)} |`);
+}
+console.log();
+
+// Verdict distribution (when --semantic was used in either run)
+function verdicts(run) {
+  return run.semantic?.verdictBreakdown || null;
+}
+const bv = verdicts(base);
+const cv = verdicts(cand);
+if (bv || cv) {
+  console.log(`## Verdict distribution\n`);
+  const allKeys = new Set([...(bv ? Object.keys(bv) : []), ...(cv ? Object.keys(cv) : [])]);
+  console.log(`| verdict | baseline | candidate | delta |`);
+  console.log(`|---|---:|---:|---:|`);
+  for (const k of [...allKeys].sort()) {
+    console.log(`| ${k} | ${fmt(bv?.[k] ?? 0)} | ${fmt(cv?.[k] ?? 0)} | ${delta(bv?.[k] ?? 0, cv?.[k] ?? 0)} |`);
+  }
+  console.log();
+}
+
+// Per-row pass-flip surface — which intents flipped pass/fail between the two runs?
+function indexById(run) {
+  return new Map((run.results || []).map((r) => [r.id, r]));
+}
+const baseById = indexById(base);
+const candById = indexById(cand);
+const flips = { pass_to_fail: [], fail_to_pass: [] };
+for (const [id, b] of baseById) {
+  const c = candById.get(id);
+  if (!c) continue;
+  if (b.pass && !c.pass) flips.pass_to_fail.push({ id, intent: b.intent, baseScore: b.validationScore, candCombined: c.combinedScore, candSemantic: c.semanticScore, candVerdict: c.semanticVerdict });
+  else if (!b.pass && c.pass) flips.fail_to_pass.push({ id, intent: b.intent, baseScore: b.validationScore, candCombined: c.combinedScore, candSemantic: c.semanticScore, candVerdict: c.semanticVerdict });
+}
+
+console.log(`## Pass-flip diagnostics\n`);
+console.log(`- pass → fail (regressions to investigate): **${flips.pass_to_fail.length}**`);
+console.log(`- fail → pass (improvements): **${flips.fail_to_pass.length}**\n`);
+
+function flipTable(rows, header) {
+  if (!rows.length) return;
+  console.log(`### ${header}\n`);
+  console.log(`| intent | base validationScore | cand combinedScore | cand semanticScore | cand verdict |`);
+  console.log(`|---|---:|---:|---:|---|`);
+  for (const r of rows.slice(0, 20)) {
+    console.log(`| ${r.intent} | ${fmt(r.baseScore)} | ${fmt(r.candCombined)} | ${fmt(r.candSemantic)} | ${fmt(r.candVerdict)} |`);
+  }
+  if (rows.length > 20) console.log(`\n_(${rows.length - 20} more rows omitted)_`);
+  console.log();
+}
+flipTable(flips.pass_to_fail, 'Regressions (pass → fail)');
+flipTable(flips.fail_to_pass, 'Improvements (fail → pass)');
+
+console.log(`---\n`);
+console.log(`_Generated by \`packages/a2ui/mcp/scripts/semantic-stats.mjs\`_`);

package/scripts/smoke-issues.mjs

@@ -0,0 +1,266 @@
+#!/usr/bin/env node
+// Smoke test: issue-reporter (write, trace attach, auto-fire, coalesce, evalMode).
+// Spec: docs/specs/genui-multiturn-architecture.md §3.5 + §4.6 + §6.4 + §11.
+
+import {
+  reportIssue,
+  autoReport,
+  attachTrace,
+  createIssueAccumulator,
+  AUTO_FIRE_POLICY,
+} from '../../compose/engines/zettel/issue-reporter.js';
+import { StateCache } from '../../compose/engines/zettel/state-cache.js';
+import { mkdtemp, readFile, rm, stat } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+let pass = 0, fail = 0;
+const t = (label, ok, detail = '') => {
+  if (ok) { console.log(`  ✓ ${label}`); pass++; }
+  else    { console.log(`  ✗ ${label}  ${detail}`); fail++; }
+};
+
+const TMP = await mkdtemp(join(tmpdir(), 'a2ui-issues-'));
+const ctx = {
+  storageRoot: TMP,
+  versionInfo: { mcp: '0.1.0', corpus: '0.0.6', engine: 'zettel', llm_adapter: 'stub', model: 'test' },
+};
+
+console.log(`Storage root: ${TMP}`);
+console.log('\n=== reportIssue: basic write ===');
+
+const r1 = await reportIssue({
+  type: 'bug',
+  severity: 'drift',
+  title: 'Test issue with simple title',
+  body: 'Test body content.',
+}, { ...ctx, reporter: 'user' });
+
+t('returns issue_id', !!r1.issue_id);
+t('issue-id format YYYY-MM-DD-slug-rand4', /^\d{4}-\d{2}-\d{2}-[a-z0-9-]+-[a-f0-9]{4}$/.test(r1.issue_id));
+t('returns ack: logged', r1.ack === 'logged');
+t('returns absolute path', r1.path.startsWith(TMP));
+
+const file1 = JSON.parse(await readFile(r1.path, 'utf8'));
+t('written file has expected type', file1.type === 'bug');
+t('written file has expected severity', file1.severity === 'drift');
+t('written file has status open', file1.status === 'open');
+t('written file has reporter.kind', file1.reporter.kind === 'user');
+t('written file has environment', file1.environment?.mcp === '0.1.0');
+t('written file has linked_specs default', Array.isArray(file1.linked_specs) && file1.linked_specs.length > 0);
+t('related_issue_ids defaults to empty array', Array.isArray(file1.related_issue_ids) && file1.related_issue_ids.length === 0);
+t('tags defaults to empty array', Array.isArray(file1.tags));
+t('suggested_owner defaults to "unknown"', file1.suggested_owner === 'unknown');
+
+console.log('\n=== validation guards ===');
+
+let threw = false;
+try { await reportIssue({ type: 'invalid', severity: 'drift', title: 't', body: 'b' }, ctx); }
+catch (e) { threw = /type must be/.test(e.message); }
+t('rejects invalid type', threw);
+
+threw = false;
+try { await reportIssue({ type: 'bug', severity: 'oops', title: 't', body: 'b' }, ctx); }
+catch (e) { threw = /severity must be/.test(e.message); }
+t('rejects invalid severity', threw);
+
+threw = false;
+try { await reportIssue({ type: 'bug', severity: 'drift', title: 'x'.repeat(81), body: 'b' }, ctx); }
+catch (e) { threw = /≤ 80 chars/.test(e.message); }
+t('rejects title > 80 chars', threw);
+
+threw = false;
+try { await reportIssue({ type: 'bug', severity: 'drift', title: 't', body: 'b', trace: 'partial' }, ctx); }
+catch (e) { threw = /trace must be/.test(e.message); }
+t('rejects invalid trace depth', threw);
+
+threw = false;
+try { await reportIssue({ type: 'bug', severity: 'drift', title: 't', body: 'b', tags: 'not-an-array' }, ctx); }
+catch (e) { threw = /tags must be an array/.test(e.message); }
+t('rejects non-array tags', threw);
+
+console.log('\n=== state_id trace attachment ===');
+
+const cache = new StateCache({ maxSize: 10 });
+cache.set('dash-3f9a-v1-26042817', {
+  state_id: 'dash-3f9a-v1-26042817',
+  intent: 'admin dashboard',
+  tool: 'compose_from_chunks',
+  input: { intent: 'admin dashboard' },
+  output: { html: '<dashboard/>', plan: { page: 'dashboard-admin-page' } },
+  ops_history: [
+    { type: 'createSurface', surfaceId: 'main' },
+    { type: 'updateComponents', surfaceId: 'main', components: [] },
+  ],
+  delta_summary: 'Created admin dashboard',
+  warnings: [],
+  duration_ms: 1234,
+  internal: {
+    locator_prompt: 'PROMPT_LOCATOR_v1',
+    locator_response: 'RESPONSE_LOCATOR_v1',
+    modifier_prompt: 'PROMPT_MODIFIER_v1',
+    modifier_response: 'RESPONSE_MODIFIER_v1',
+    validator_results: [{ ok: true }],
+    retries: 0,
+  },
+});
+
+const traceSummary = await attachTrace('dash-3f9a-v1-26042817', 'summary', cache);
+t('summary trace populates state_id', traceSummary?.state_id === 'dash-3f9a-v1-26042817');
+t('summary trace populates input', traceSummary?.input?.intent === 'admin dashboard');
+t('summary trace populates output.ops', Array.isArray(traceSummary?.output?.ops));
+t('summary trace omits internal field', traceSummary?.internal === undefined);
+
+const traceFull = await attachTrace('dash-3f9a-v1-26042817', 'full', cache);
+t('full trace includes internal.locator_prompt', traceFull?.internal?.locator_prompt === 'PROMPT_LOCATOR_v1');
+t('full trace includes internal.modifier_response', traceFull?.internal?.modifier_response === 'RESPONSE_MODIFIER_v1');
+t('full trace includes validator_results', Array.isArray(traceFull?.internal?.validator_results));
+
+const traceMiss = await attachTrace('not-a-real-id', 'summary', cache);
+t('attachTrace returns null on cache miss', traceMiss === null);
+
+const traceNoCache = await attachTrace('dash-3f9a-v1-26042817', 'summary', null);
+t('attachTrace returns null when no cache', traceNoCache === null);
+
+// reportIssue with state_id integrates trace
+const r2 = await reportIssue({
+  type: 'bug',
+  severity: 'drift',
+  title: 'Issue tied to a state',
+  body: 'Reproducible on dashboard generation.',
+  state_id: 'dash-3f9a-v1-26042817',
+  trace: 'full',
+}, { ...ctx, cache });
+const file2 = JSON.parse(await readFile(r2.path, 'utf8'));
+t('reportIssue with state_id+full attaches trace.input', file2.trace?.input?.intent === 'admin dashboard');
+t('reportIssue with full trace attaches internal', file2.trace?.internal?.locator_prompt === 'PROMPT_LOCATOR_v1');
+
+// peek does not touch recency — verify the cache state didn't bump dash- to most-recent.
+// Insert a few more entries, ensure dash- stays the LRU candidate.
+// (This is exercised indirectly: state-cache smoke covers peek-recency directly.)
+
+console.log('\n=== oversized trace spills to sidecar ===');
+
+cache.set('big-state', {
+  state_id: 'big-state',
+  intent: 'big',
+  tool: 'compose',
+  input: {},
+  ops_history: [],
+  internal: { huge_dump: 'x'.repeat(300 * 1024) },
+});
+const r3 = await reportIssue({
+  type: 'bug',
+  severity: 'drift',
+  title: 'Big trace test',
+  body: 'Has an oversized trace.',
+  state_id: 'big-state',
+  trace: 'full',
+}, { ...ctx, cache });
+const file3 = JSON.parse(await readFile(r3.path, 'utf8'));
+t('oversized trace replaced by sidecar pointer', !!file3.trace?.sidecar);
+t('sidecar path has expected shape', /^traces\/.+\.trace\.json$/.test(file3.trace.sidecar));
+const sidecar = await stat(join(TMP, file3.trace.sidecar));
+t('sidecar file exists with non-zero size', sidecar.size > 0);
+
+console.log('\n=== autoReport: policy lookup ===');
+
+const a1 = await autoReport('validator-exhausted', { tool: 'refine_composition' }, ctx);
+const file_a1 = JSON.parse(await readFile(a1.path, 'utf8'));
+t('validator-exhausted: type=bug', file_a1.type === 'bug');
+t('validator-exhausted: severity=blocker', file_a1.severity === 'blocker');
+t('validator-exhausted: suggested_owner=validator', file_a1.suggested_owner === 'validator');
+t('validator-exhausted: reporter.kind=auto', file_a1.reporter.kind === 'auto');
+t('validator-exhausted: reporter.context=validator-exhausted', file_a1.reporter.context === 'validator-exhausted');
+t('validator-exhausted: tags include "auto-fire"', file_a1.tags.includes('auto-fire'));
+
+const a2 = await autoReport('retrieval-zero-then-synthesis-fail', { intent: 'pricing page' }, ctx);
+const file_a2 = JSON.parse(await readFile(a2.path, 'utf8'));
+t('retrieval-zero-then-synthesis-fail: type=training-gap', file_a2.type === 'training-gap');
+t('retrieval-zero-then-synthesis-fail: suggested_owner=chunk-corpus', file_a2.suggested_owner === 'chunk-corpus');
+t('retrieval-zero-then-synthesis-fail: title carries intent', /pricing page/.test(file_a2.title));
+
+const a3 = await autoReport('cache-miss-on-known-state', { state_id: 'gone-12ab-v1-0' }, ctx);
+const file_a3 = JSON.parse(await readFile(a3.path, 'utf8'));
+t('cache-miss-on-known-state: severity=nit', file_a3.severity === 'nit');
+
+threw = false;
+try { await autoReport('unknown-reason', {}, ctx); }
+catch (e) { threw = /unknown reason/.test(e.message); }
+t('autoReport rejects unknown reason', threw);
+
+console.log('\n=== evalMode suppresses auto-fire ===');
+
+const evalCtx = { ...ctx, evalMode: true };
+const aSuppressed = await autoReport('validator-exhausted', { tool: 'refine_composition' }, evalCtx);
+t('autoReport returns null when evalMode=true', aSuppressed === null);
+
+// Manual reportIssue still writes during evalMode (eval-suppression is auto-fire only)
+const aManual = await reportIssue({
+  type: 'bug',
+  severity: 'blocker',
+  title: 'Manual call during evalMode',
+  body: 'should still write',
+}, evalCtx);
+t('manual reportIssue ignores evalMode', !!aManual.issue_id);
+const file_manual = JSON.parse(await readFile(aManual.path, 'utf8'));
+t('manual call during evalMode writes file', file_manual.title === 'Manual call during evalMode');
+
+console.log('\n=== coalescing accumulator ===');
+
+const acc = createIssueAccumulator();
+t('empty accumulator size 0', acc.size() === 0);
+const flushEmpty = await acc.flush(ctx);
+t('empty accumulator flush returns null', flushEmpty === null);
+
+acc.add('locator-empty-targets', { intent: 'change title' });
+t('single-entry accumulator size 1', acc.size() === 1);
+const flushOne = await acc.flush(ctx);
+const file_flush_one = JSON.parse(await readFile(flushOne.path, 'utf8'));
+t('single-entry flush writes normal auto-issue', file_flush_one.reporter.kind === 'auto' && file_flush_one.reporter.context === 'locator-empty-targets');
+t('single-entry flush resets accumulator', acc.size() === 0);
+
+acc.add('locator-empty-targets', { intent: 'change title' });
+acc.add('validator-exhausted', { tool: 'refine_composition' });
+acc.add('ops-failed-after-apply', {});
+t('three-entry accumulator size 3', acc.size() === 3);
+const flushThree = await acc.flush(ctx);
+const file_flush_three = JSON.parse(await readFile(flushThree.path, 'utf8'));
+t('coalesced issue: severity=blocker (highest of three)', file_flush_three.severity === 'blocker');
+t('coalesced issue: type=bug', file_flush_three.type === 'bug');
+t('coalesced issue: reporter.context=coalesced', file_flush_three.reporter.context === 'coalesced');
+t('coalesced issue: tags include "coalesced"', file_flush_three.tags.includes('coalesced'));
+t('coalesced issue: tags include all reasons',
+  ['locator-empty-targets', 'validator-exhausted', 'ops-failed-after-apply'].every((r) => file_flush_three.tags.includes(r))
+);
+t('coalesced issue: body lists every reason',
+  file_flush_three.body.includes('locator-empty-targets') &&
+  file_flush_three.body.includes('validator-exhausted') &&
+  file_flush_three.body.includes('ops-failed-after-apply')
+);
+t('coalesced flush resets accumulator', acc.size() === 0);
+
+// evalMode + coalesce → no write
+const accEval = createIssueAccumulator();
+accEval.add('validator-exhausted', {});
+accEval.add('ops-failed-after-apply', {});
+const flushEval = await accEval.flush({ ...ctx, evalMode: true });
+t('coalesce flush returns null when evalMode=true', flushEval === null);
+
+threw = false;
+try { acc.add('not-a-real-reason'); }
+catch (e) { threw = /unknown reason/.test(e.message); }
+t('accumulator.add rejects unknown reason', threw);
+
+console.log('\n=== AUTO_FIRE_POLICY exported ===');
+
+t('AUTO_FIRE_POLICY exports expected reasons',
+  ['synthesizer-exhausted', 'validator-exhausted', 'locator-empty-targets',
+   'retrieval-zero-then-synthesis-fail', 'cache-miss-on-known-state',
+   'ops-failed-after-apply'].every((r) => AUTO_FIRE_POLICY[r])
+);
+
+await rm(TMP, { recursive: true, force: true });
+
+console.log(`\n${pass} passed, ${fail} failed`);
+process.exit(fail ? 1 : 0);