kenaz 0.0.0-stage → 0.3.2

package/core/dev/jev-eval/review.mjs

@@ -0,0 +1,78 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { readFile, stat, writeFile, link, unlink } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { check, keys, identity, stateHash, readJsonl, validateDataset, MAX_FILE_BYTES } from './schema.mjs';
+
+export const REVIEW_VERSION = 'jev-review-v1';
+const canonical = value => JSON.stringify(value, function (key, item) {
+  return item && typeof item === 'object' && !Array.isArray(item) ? Object.fromEntries(Object.keys(item).sort().map(k => [k, item[k]])) : item;
+});
+export const datasetHash = rows => createHash('sha256').update(canonical(rows)).digest('hex');
+export function exportReview(rows) {
+  validateDataset(rows);
+  return { schema_version: 1, review_version: REVIEW_VERSION, dataset_hash: datasetHash(rows), items: rows.map(row => ({
+    ...identity(row), state_hash: stateHash(row), purpose: row.purpose, language: row.language, context: structuredClone(row.state),
+    ...(row.purpose === 'relevance' ? { candidates: row.candidates.map(({ id, text }) => ({ id, text })) } : {}),
+    decision: row.purpose === 'correction' ? { correction: null } : { grades: row.candidates.map(c => ({ id: c.id, grade: null })) },
+    reviewed: false, reviewer: null, reviewed_at: null,
+  })) };
+}
+function validTime(value) {
+  if (typeof value !== 'string' || !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z$/.test(value)) return false;
+  const time = Date.parse(value);
+  return Number.isFinite(time) && new Date(time).toISOString() === value.replace(/(?<!\.\d{3})Z$/, '.000Z');
+}
+export function applyReview(rows, review) {
+  const expected = exportReview(rows);
+  keys(review, ['schema_version', 'review_version', 'dataset_hash', 'items']);
+  check(review.schema_version === 1 && review.review_version === REVIEW_VERSION, 'invalid_review_version');
+  check(review.dataset_hash === expected.dataset_hash, 'stale_dataset');
+  check(Array.isArray(review.items) && review.items.length === rows.length, 'incomplete_review');
+  const byId = new Map();
+  for (const item of review.items) { check(item && !byId.has(item.id), 'duplicate_review'); byId.set(item.id, item); }
+  const result = rows.map((row, index) => {
+    const item = byId.get(row.id), base = expected.items[index]; keys(item, Object.keys(base));
+    for (const key of Object.keys(base).filter(k => !['decision', 'reviewed', 'reviewer', 'reviewed_at'].includes(k))) check(canonical(item[key]) === canonical(base[key]), 'review_identity_mismatch');
+    check(item.reviewed === true, 'unreviewed_item');
+    check(typeof item.reviewer === 'string' && item.reviewer.trim().length > 0 && item.reviewer.length <= 160, 'invalid_reviewer');
+    check(validTime(item.reviewed_at), 'invalid_review_date');
+    const out = structuredClone(row);
+    if (row.purpose === 'correction') { keys(item.decision, ['correction']); check(typeof item.decision.correction === 'boolean', 'invalid_review_decision'); out.labels.correction = item.decision.correction; }
+    else {
+      keys(item.decision, ['grades']);
+      check(Array.isArray(item.decision.grades) && item.decision.grades.length === row.candidates.length, 'incomplete_grades');
+      const grades = new Map();
+      for (const c of item.decision.grades) { keys(c, ['id', 'grade']); check(!grades.has(c.id) && row.candidates.some(x => x.id === c.id) && Number.isInteger(c.grade) && c.grade >= 0 && c.grade <= 3, 'invalid_review_grade'); grades.set(c.id, c.grade); }
+      for (const c of out.candidates) c.grade = grades.get(c.id);
+    }
+    out.provenance = { ...out.provenance, annotation: 'human_verified', reviewer: item.reviewer, reviewed_at: item.reviewed_at };
+    return out;
+  });
+  return validateDataset(result);
+}
+export async function main(args = process.argv.slice(2)) {
+  const [command, ...rest] = args; check(['export', 'apply'].includes(command), 'invalid_command');
+  const options = {};
+  for (let i = 0; i < rest.length; i += 2) { check(['--dataset', '--out', ...(command === 'apply' ? ['--reviews'] : [])].includes(rest[i]) && typeof rest[i + 1] === 'string' && !rest[i + 1].startsWith('--') && !Object.hasOwn(options, rest[i]), 'invalid_arguments'); options[rest[i]] = rest[i + 1]; }
+  check(options['--dataset'] && options['--out'] && (command !== 'apply' || options['--reviews']), 'missing_argument');
+  check(resolve(options['--dataset']).toLowerCase() !== resolve(options['--out']).toLowerCase() && (!options['--reviews'] || resolve(options['--reviews']).toLowerCase() !== resolve(options['--out']).toLowerCase()), 'overwrite_forbidden');
+  const rows = validateDataset(await readJsonl(options['--dataset']));
+  let output;
+  if (command === 'export') output = JSON.stringify(exportReview(rows), null, 2) + '\n';
+  else {
+    check((await stat(options['--reviews'])).size <= MAX_FILE_BYTES, 'file_budget_exceeded');
+    const text = await readFile(options['--reviews'], 'utf8'); check(Buffer.byteLength(text) <= MAX_FILE_BYTES, 'file_budget_exceeded');
+    let review; try { review = JSON.parse(text.replace(/^\uFEFF/, '')); } catch { throw new Error('invalid_review_json'); }
+    output = applyReview(rows, review).map(r => JSON.stringify(r)).join('\n') + '\n';
+  }
+  check(Buffer.byteLength(output) <= MAX_FILE_BYTES, 'file_budget_exceeded');
+  // A same-directory hard link publishes only the complete file, and fails if out exists.
+  const temporary = `${options['--out']}.${randomUUID()}.tmp`;
+  try { await writeFile(temporary, output, { flag: 'wx', encoding: 'utf8' }); await link(temporary, options['--out']); }
+  finally { await unlink(temporary).catch(() => {}); }
+  return { schema_version: 1, operation: command, count: rows.length, promotion_eligible: false, human_review_is_attestation_not_proof: true };
+}
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) main().then(r => console.log(JSON.stringify(r, null, 2))).catch(error => {
+  console.error(JSON.stringify({ error: /^[a-z_]{1,80}$/.test(error.message) ? error.message : error.code === 'EEXIST' ? 'output_exists' : 'operation_failed' })); process.exitCode = 1;
+});

package/core/dev/jev-eval/schema.mjs

@@ -0,0 +1,87 @@
+import { readFile, stat } from 'node:fs/promises';
+import { createHash } from 'node:crypto';
+
+export const MAX_FILE_BYTES = 8 * 1024 * 1024;
+export const MAX_RECORD_BYTES = 64 * 1024;
+export const SPLITS = ['development', 'calibration', 'held_out'];
+export function check(condition, code = 'invalid_data') { if (!condition) throw new Error(code); }
+export const object = x => x !== null && typeof x === 'object' && !Array.isArray(x);
+export const number = (x, min, max) => typeof x === 'number' && Number.isFinite(x) && x >= min && x <= max;
+const string = x => typeof x === 'string' && x.trim().length > 0 && x.length <= 16384;
+const identifier = x => typeof x === 'string' && /^[\w.:/-]{1,160}$/.test(x);
+export function keys(x, names) {
+  check(object(x) && Object.keys(x).length === names.length && names.every(k => Object.hasOwn(x, k)), 'invalid_fields');
+}
+export async function readJsonl(path) {
+  check((await stat(path)).size <= MAX_FILE_BYTES, 'file_budget_exceeded');
+  const text = await readFile(path, 'utf8');
+  check(Buffer.byteLength(text) <= MAX_FILE_BYTES, 'file_budget_exceeded');
+  try { return text.replace(/^\uFEFF/, '').split(/\r?\n/).filter(x => x.trim()).map(x => {
+    check(Buffer.byteLength(x) <= MAX_RECORD_BYTES, 'record_budget_exceeded');
+    return JSON.parse(x);
+  }); } catch { throw new Error('invalid_jsonl'); }
+}
+export function inputState(row) {
+  return { purpose: row.purpose, ...row.state, ...(row.purpose === 'relevance' ? { candidates: row.candidates.map(({ id, text }) => ({ id, text })) } : {}) };
+}
+export const stateHash = row => createHash('sha256').update(JSON.stringify(inputState(row))).digest('hex');
+export function identity(row) {
+  return Object.fromEntries(['id', 'project_id', 'session_id', 'task_id', 'event_id', 'event_seq', 'task_revision'].map(k => [k, row[k]]));
+}
+export function validateDataset(rows) {
+  check(Array.isArray(rows) && rows.length > 0 && rows.length <= 10000, 'invalid_dataset_size');
+  const ids = new Set(), events = new Set(), splits = new Map(), texts = new Map();
+  for (const row of rows) {
+    check(Buffer.byteLength(JSON.stringify(row)) <= MAX_RECORD_BYTES, 'record_budget_exceeded');
+    keys(row, ['schema_version', 'id', 'project_id', 'session_id', 'group_id', 'task_id', 'event_id', 'event_seq', 'task_revision', 'purpose', 'language', 'split', 'provenance', 'state', row.purpose === 'correction' ? 'labels' : 'candidates']);
+    check(row.schema_version === 1, 'invalid_schema_version');
+    for (const k of ['id', 'project_id', 'session_id', 'group_id', 'task_id', 'event_id']) check(identifier(row[k]), 'invalid_identifier');
+    for (const k of ['event_seq', 'task_revision']) check(Number.isSafeInteger(row[k]) && row[k] >= 0, 'invalid_revision');
+    check(['correction', 'relevance'].includes(row.purpose) && ['en', 'zh', 'mixed'].includes(row.language) && SPLITS.includes(row.split), 'invalid_enum');
+    keys(row.provenance, ['source', 'annotation', 'reviewer', 'reviewed_at']);
+    check(['synthetic', 'real'].includes(row.provenance.source) && ['agent_proposed', 'human_verified'].includes(row.provenance.annotation), 'invalid_provenance');
+    check(row.provenance.annotation === 'human_verified' ? string(row.provenance.reviewer) : row.provenance.reviewer === null, 'invalid_reviewer');
+    check(row.provenance.annotation === 'human_verified' ? typeof row.provenance.reviewed_at === 'string' && /^\d{4}-\d{2}-\d{2}T/.test(row.provenance.reviewed_at) && Number.isFinite(Date.parse(row.provenance.reviewed_at)) : row.provenance.reviewed_at === null, 'invalid_review_date');
+    keys(row.state, ['goal', 'assumptions', 'message']);
+    check(string(row.state.goal) && string(row.state.message) && Array.isArray(row.state.assumptions) && row.state.assumptions.length <= 20 && row.state.assumptions.every(string), 'invalid_state');
+    if (row.purpose === 'correction') { keys(row.labels, ['correction']); check(typeof row.labels.correction === 'boolean', 'invalid_label'); }
+    else {
+      check(Array.isArray(row.candidates) && row.candidates.length >= 2 && row.candidates.length <= 12, 'invalid_candidates');
+      const candidateIds = new Set();
+      for (const c of row.candidates) { keys(c, ['id', 'text', 'grade']); check(identifier(c.id) && string(c.text) && Number.isInteger(c.grade) && number(c.grade, 0, 3) && !candidateIds.has(c.id), 'invalid_candidate'); candidateIds.add(c.id); }
+    }
+    check(!ids.has(row.id), 'duplicate_id'); ids.add(row.id);
+    const event = JSON.stringify([row.project_id, row.session_id, row.event_id, row.purpose]);
+    check(!events.has(event), 'duplicate_event'); events.add(event);
+    for (const group of [`session:${row.project_id}:${row.session_id}`, `group:${row.group_id}`]) {
+      check(!splits.has(group) || splits.get(group) === row.split, 'split_leakage'); splits.set(group, row.split);
+    }
+    const hash = stateHash(row);
+    check(!texts.has(hash) || texts.get(hash) === row.split, 'content_split_leakage'); texts.set(hash, row.split);
+  }
+  return rows;
+}
+export function validatePrediction(row, prediction) {
+  keys(prediction, [...Object.keys(identity(row)), 'state_hash', 'model', 'question_set_version', 'status', 'failure_reason', 'http_status', 'usage', 'latency_ms', row.purpose === 'correction' ? 'correction_probability' : 'scores']);
+  for (const [k, v] of Object.entries(identity(row))) check(prediction[k] === v, 'prediction_identity_mismatch');
+  check(prediction.state_hash === stateHash(row), 'stale_prediction');
+  check(['jev-1.13.0', 'rules-v1'].includes(prediction.model) && prediction.question_set_version === 'jev-eval-v1', 'invalid_prediction_version');
+  check(['ok', 'abstain', 'timeout', 'unavailable', 'invalid'].includes(prediction.status), 'invalid_status');
+  check(number(prediction.latency_ms, 0, 3600000), 'invalid_latency');
+  check(prediction.http_status === null || Number.isInteger(prediction.http_status) && number(prediction.http_status, 100, 599), 'invalid_http_status');
+  check(prediction.status === 'ok' ? prediction.failure_reason === null : ['timeout', 'http_failure', 'transport_failure', 'invalid_response', 'abstain'].includes(prediction.failure_reason), 'invalid_failure_reason');
+  if (prediction.usage !== null) { keys(prediction.usage, ['input_tokens', 'output_tokens']); for (const n of Object.values(prediction.usage)) check(Number.isSafeInteger(n) && n >= 0, 'invalid_usage'); }
+  if (row.purpose === 'correction') {
+    check(prediction.status === 'ok' ? number(prediction.correction_probability, 0, 1) : prediction.correction_probability === null, 'invalid_probability');
+  } else {
+    check(Array.isArray(prediction.scores) && prediction.scores.length === row.candidates.length, 'invalid_scores');
+    const expected = new Set(row.candidates.map(c => c.id));
+    for (const item of prediction.scores) {
+      keys(item, ['id', 'score']); check(expected.delete(item.id), 'invalid_score_id');
+      check(item.score === null || number(item.score, 0, 3), 'invalid_score');
+      if (prediction.status !== 'ok') check(item.score === null, 'failure_with_score');
+    }
+    if (prediction.status === 'ok') check(prediction.scores.some(s => s.score !== null), 'empty_ok_prediction');
+  }
+  return prediction;
+}

package/core/dev/jev-eval/test/context.test.mjs

@@ -0,0 +1,95 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtemp, writeFile, symlink, readdir, unlink, rmdir, lstat } from 'node:fs/promises';
+import { createHash } from 'node:crypto';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { enrich, resolveSource, renderHtml } from '../context.mjs';
+import { normalizeMessage, redactHomePaths, credentialBearing } from '../corpus.mjs';
+
+const hash = text => createHash('sha256').update(text).digest('hex'), cwd = 'C:/Users/test/Documents/GitHub/KenazAI';
+const c = (uuid, parentUuid, role, text, extra = {}) => ({ uuid, parentUuid, sessionId: 's1', cwd, type: role, message: { role, content: [{ type: 'text', text }] }, ...extra });
+async function fixture(t, kind, records, targetLine, overrides = {}) {
+  const dir = await mkdtemp(join(tmpdir(), 'jev-context-test-'));
+  t.after(async () => { for (const name of await readdir(dir)) { const p = join(dir, name), s = await lstat(p); if (s.isSymbolicLink() && process.platform === 'win32') await rmdir(p); else await unlink(p); } await rmdir(dir); });
+  const path = join(dir, 'source.jsonl'), bytes = records.map(r => JSON.stringify(r) + '\n').join(''); await writeFile(path, bytes);
+  const source = { kind, relative_path: 'source.jsonl', source_id: hash(`${kind}:source.jsonl`) };
+  const target = records[targetLine - 1], message = normalizeMessage(redactHomePaths(kind === 'claude' ? target.message.content[0].text : target.payload.message));
+  const row = { kind: 'pending_historical_review', id: hash(`${source.source_id}:${targetLine}:${hash(message.toLowerCase())}`), source_id: source.source_id, session_id: hash(`${kind}:s1`), group_id: 'group', message, review_status: 'pending', provenance: { source: 'real', annotation: 'unreviewed' }, ...overrides };
+  return { dir, path, bytes, row, source, manifest: { schema_version: 1, sources: [source], items: [{ id: row.id, source_id: source.source_id, line: targetLine }] }, options: { sourceRoots: { claude: dir, codex: dir }, cwdRoots: [cwd] } };
+}
+test('Claude ancestry follows parent UUID rather than neighboring branch, skips thinking/tools/future', async t => {
+  const records = [c('u1', null, 'user', 'Start'), c('a1', 'u1', 'assistant', 'Correct branch'), c('wrong', 'u1', 'assistant', 'Other branch'), c('tool', 'a1', 'user', '', { message: { role: 'user', content: [{ type: 'tool_result', content: 'secret tool output' }] } }), c('thinking', 'tool', 'assistant', '', { message: { role: 'assistant', content: [{ type: 'thinking', thinking: 'hidden reasoning' }] } }), c('target', 'thinking', 'user', 'Please revise'), c('future', 'target', 'assistant', 'Future answer')];
+  const f = await fixture(t, 'claude', records, 6), before = JSON.stringify(f.row), result = await enrich([f.row], f.manifest, f.options), row = result.rows[0];
+  assert.equal(row.status, 'restored'); assert.deepEqual(row.observed_context.map(c => c.text), ['Start', 'Correct branch']);
+  assert.ok(row.observed_context.every(c => c.source_line < row.target_line)); assert.equal(JSON.stringify(row).includes('hidden reasoning'), false); assert.equal(JSON.stringify(row).includes('Other branch'), false);
+  assert.equal(row.source_prefix_sha256, hash(records.slice(0, 6).map(r => JSON.stringify(r) + '\n').join(''))); assert.equal(JSON.stringify(f.row), before); assert.equal(row.decision.correction, null);
+});
+test('stale targets and wrong session rejected without dropping IDs', async t => {
+  const f = await fixture(t, 'claude', [c('a', null, 'assistant', 'Context'), c('u', 'a', 'user', 'Target')], 2);
+  for (const change of [r => r.message = 'Changed', r => r.session_id = hash('other'), r => r.review_status = 'human_verified']) {
+    const row = structuredClone(f.row); change(row); const result = await enrich([row], f.manifest, f.options); assert.equal(result.rows[0].id, row.id); assert.equal(result.rows[0].status, 'rejected'); assert.equal(result.rows[0].observed_context, undefined);
+  }
+});
+test('source paths reject traversal absolute paths hash mismatch and symlink escape', async t => {
+  const f = await fixture(t, 'claude', [c('u', null, 'user', 'Target')], 1);
+  for (const relative_path of ['../outside', '/absolute', 'C:\\absolute', 'nested/../../outside']) await assert.rejects(resolveSource({ kind: 'claude', relative_path, source_id: hash(`claude:${relative_path}`) }, f.options.sourceRoots), /unsafe_source_path/);
+  await assert.rejects(resolveSource({ ...f.source, source_id: 'wrong' }, f.options.sourceRoots), /source_id_mismatch/);
+  const outside = await fixture(t, 'claude', [c('u', null, 'user', 'Other')], 1), link = join(f.dir, 'escape');
+  await symlink(outside.dir, link, process.platform === 'win32' ? 'junction' : 'dir');
+  await assert.rejects(resolveSource({ kind: 'claude', relative_path: 'escape/source.jsonl', source_id: hash('claude:escape/source.jsonl') }, f.options.sourceRoots), /source_path_escape/);
+});
+test('credential context dropped as whole message and gaps remain explicit', async t => {
+  const f = await fixture(t, 'claude', [c('a1', null, 'assistant', 'Safe context'), c('a2', 'a1', 'assistant', 'Use apikey_abcdefgh12345 now'), c('u', 'a2', 'user', 'Continue')], 3);
+  const row = (await enrich([f.row], f.manifest, f.options)).rows[0]; assert.equal(row.status, 'partial'); assert.deepEqual(row.observed_context.map(c => c.text), ['Safe context']); assert.ok(row.context_gaps.includes('credential_context')); assert.equal(JSON.stringify(row).includes('apikey_'), false);
+});
+test('normalization and JSON quote escaping cannot hide credential context', async t => {
+  const escaped = 'API_KEY="abcd1234"'; assert.equal(credentialBearing(escaped), true); assert.equal(credentialBearing(JSON.stringify(escaped)), false);
+  for (const text of [escaped, 'The config is {"API_KEY":"abc12345"}', 'Use ＡＰＩ＿ＫＥＹ＝ａｂｃ１２３４５ locally', 'ＡＰＩＫＥＹ＿ａｂｃｄｅｆｇｈ１２３４５']) {
+    assert.equal(credentialBearing(normalizeMessage(text)), true);
+    const f = await fixture(t, 'claude', [c('safe', null, 'assistant', 'Safe earlier text'), c('secret', 'safe', 'assistant', text), c('target', 'secret', 'user', 'Continue')], 3);
+    const row = (await enrich([f.row], f.manifest, f.options)).rows[0];
+    assert.deepEqual(row.observed_context.map(c => c.text), ['Safe earlier text']);
+    assert.ok(row.context_gaps.includes('credential_context')); assert.ok(row.observed_context.every(c => !credentialBearing(c.text)));
+  }
+  const target = await fixture(t, 'claude', [c('user', null, 'user', 'ＡＰＩ＿ＫＥＹ＝ａｂｃ１２３４５')], 1);
+  assert.equal((await enrich([target.row], target.manifest, target.options)).rows[0].status, 'rejected');
+});
+const meta = { type: 'session_meta', payload: { id: 's1', cwd, source: 'cli' } };
+const assistant = text => ({ type: 'response_item', payload: { type: 'message', role: 'assistant', phase: 'final_answer', content: [{ type: 'output_text', text }] } });
+const user = text => ({ type: 'event_msg', payload: { type: 'user_message', message: text } });
+test('Codex compaction/scope resets context and event assistant duplicates are ignored', async t => {
+  const list = [meta, assistant('Before reset'), { type: 'compacted', payload: {} }, assistant('After reset'), { type: 'event_msg', payload: { type: 'agent_message', message: 'Event duplicate' } }, user('Target'), assistant('Future')];
+  const f = await fixture(t, 'codex', list, 6), row = (await enrich([f.row], f.manifest, f.options)).rows[0];
+  assert.deepEqual(row.observed_context.map(c => c.text), ['After reset']); assert.ok(row.context_gaps.includes('compaction_or_rollback'));
+  const scoped = await fixture(t, 'codex', [meta, assistant('Original cwd'), { type: 'turn_context', payload: { cwd: 'C:/Other' } }, assistant('Foreign'), { type: 'turn_context', payload: { cwd } }, user('Return')], 6);
+  assert.equal((await enrich([scoped.row], scoped.manifest, scoped.options)).rows[0].status, 'missing');
+});
+test('bounded context marks truncation, missing ancestry never uses nearest neighbor', async t => {
+  const long = await fixture(t, 'claude', [c('a', null, 'assistant', 'Long visible text '.repeat(500)), c('u', 'a', 'user', 'Target')], 2);
+  const row = (await enrich([long.row], long.manifest, long.options)).rows[0]; assert.equal(row.status, 'partial'); assert.equal(row.observed_context[0].text.length, 3000); assert.equal(row.observed_context[0].truncated, true);
+  const missing = await fixture(t, 'claude', [c('a', null, 'assistant', 'Not ancestor'), c('u', 'unknown', 'user', 'Target')], 2);
+  const m = (await enrich([missing.row], missing.manifest, missing.options)).rows[0]; assert.equal(m.status, 'missing'); assert.ok(m.context_gaps.includes('missing_ancestor'));
+});
+test('static HTML escapes raw content and has no scripts or external resources', () => {
+  const html = renderHtml([{ id: 'case', status: 'restored', message: '<script>fetch("https://evil")</script>', observed_context: [{ role: 'assistant', source_line: 1, text: '<img src=x onerror=alert(1)>' }] }]);
+  assert.equal(html.includes('<script>'), false); assert.equal(html.includes('<img'), false); assert.ok(html.includes('&lt;script&gt;')); assert.ok(html.includes("default-src 'none'"));
+});
+test('metadata-only duplicate UUID tolerated; conflicting text identity rejected', async t => {
+  const first = c('a', null, 'assistant', 'Actual context');
+  const attachment = { uuid: 'attachment', type: 'attachment', parentUuid: 'a', sessionId: 's1', cwd, arbitraryMetadata: 1 };
+  const f = await fixture(t, 'claude', [first, attachment, { ...attachment, arbitraryMetadata: 2 }, c('u', 'a', 'user', 'Target')], 4);
+  assert.equal((await enrich([f.row], f.manifest, f.options)).rows[0].status, 'restored');
+  const bad = await fixture(t, 'claude', [first, c('a', null, 'assistant', 'Conflicting context'), c('u', 'a', 'user', 'Target')], 3);
+  assert.equal((await enrich([bad.row], bad.manifest, bad.options)).rows[0].status, 'rejected');
+});
+test('ancestry walk and Codex history window are bounded with explicit gaps', async t => {
+  const list = [c('a', null, 'assistant', 'Ancient context')];
+  for (let i = 0; i < 6; i++) list.push({ uuid: `tool${i}`, parentUuid: i ? `tool${i - 1}` : 'a', sessionId: 's1', cwd, type: 'user', message: { role: 'user', content: [{ type: 'tool_result', content: 'omitted' }] } });
+  list.push(c('u', 'tool5', 'user', 'Target'));
+  const f = await fixture(t, 'claude', list, list.length); f.options.limits = { maxAncestorSteps: 3 };
+  const row = (await enrich([f.row], f.manifest, f.options)).rows[0]; assert.ok(row.context_gaps.includes('ancestry_budget')); assert.equal(row.status, 'missing');
+  const history = [meta, ...Array.from({ length: 25 }, (_, i) => assistant(`Turn ${i}`)), user('Target')];
+  const g = await fixture(t, 'codex', history, history.length), restored = (await enrich([g.row], g.manifest, g.options)).rows[0];
+  assert.equal(restored.observed_context.length, 6); assert.ok(restored.context_gaps.includes('history_window')); assert.ok(renderHtml([restored]).includes('history_window'));
+});

package/core/dev/jev-eval/test/corpus.test.mjs

@@ -0,0 +1,102 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtemp, mkdir, writeFile, readdir, unlink, rmdir } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { credentialBearing, generatedMessage, automationSeed, redactHomePaths, userText, withinRoots, extractFile, sampleCandidates, records, discover, inventory, main } from '../corpus.mjs';
+
+async function temp(t) {
+  const dir = await mkdtemp(join(tmpdir(), 'jev-corpus-test-'));
+  async function cleanup(path) { for (const entry of await readdir(path, { withFileTypes: true })) { const child = join(path, entry.name); if (entry.isDirectory()) await cleanup(child); else await unlink(child); } await rmdir(path); }
+  t.after(() => cleanup(dir)); return dir;
+}
+const roots = ['C:/Users/test/Documents/GitHub/KenazAI'];
+const claude = (text, extra = {}) => ({ type: 'user', sessionId: 's1', cwd: roots[0], message: { role: 'user', content: text }, ...extra });
+const meta = (extra = {}) => ({ type: 'session_meta', payload: { id: 's1', cwd: roots[0], source: 'cli', ...extra } });
+const event = text => ({ type: 'event_msg', payload: { type: 'user_message', message: text } });
+const response = text => ({ type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text }] } });
+async function extract(t, items, kind = 'claude', limits = {}) {
+  const dir = await temp(t), path = join(dir, 's.jsonl'); await writeFile(path, items.map(r => JSON.stringify(r)).join('\n'));
+  const stats = {}, candidates = await extractFile({ path, kind, relative_path: 's.jsonl' }, roots, stats, limits); return { candidates, stats };
+}
+test('credential detection quarantines whole message and home paths are removed', () => {
+  for (const text of ['apikey_abcdefgh123456', 'sk-abcdefgh-123456', 'Authorization: Bearer abcdefgh123', '$env:TYPESAFE_API_KEY="abc123456"', 'export API_KEY=abc12345', 'password: hunter22', 'https://user:secret@site.test', '-----BEGIN PRIVATE KEY-----', 'github_pat_abcde12345678']) assert.equal(credentialBearing(text), true);
+  assert.equal(credentialBearing('Please configure the API key locally.'), false);
+  const clean = redactHomePaths('C:\\Users\\someone\\repo /home/someone/repo /Users/someone/file'); assert.equal(clean.includes('someone'), false);
+});
+test('generated and non-user records are excluded without fabricated labels', async t => {
+  for (const text of ['# AGENTS.md instructions for repo', '<environment_context>cwd</environment_context>', '[CANARY:v2]', '<system-reminder>generated</system-reminder>', '<subagent_notification>done</subagent_notification>']) assert.equal(generatedMessage(text), true);
+  assert.equal(userText(claude([{ type: 'tool_result', content: 'not user' }]), 'claude'), null);
+  const { candidates } = await extract(t, [claude('Real request'), claude('internal', { isMeta: true }), claude('render-only', { isVisibleInTranscriptOnly: true }), { type: 'assistant', message: { role: 'assistant', content: 'not user' } }, claude('<environment_context>private</environment_context>'), claude('Another request')]);
+  assert.equal(candidates.length, 2);
+  for (const { row } of candidates) { for (const field of ['labels', 'goal', 'assumptions', 'grade']) assert.equal(Object.hasOwn(row, field), false); assert.equal(row.review_status, 'pending'); assert.equal(row.incomplete_context_for_correction, true); }
+});
+test('secret message cannot survive as later preceding context', async t => {
+  const { candidates, stats } = await extract(t, [claude('First'), claude('Use apikey_abcdefgh1234 now'), claude('Next request')]);
+  assert.equal(candidates.length, 2); assert.equal(stats.credential_message, 1); assert.equal(candidates[1].row.context_missing, true);
+  assert.equal(JSON.stringify(candidates).includes('apikey_'), false);
+});
+test('Codex exact cwd boundaries and subagent metadata fail closed', async t => {
+  assert.equal(withinRoots(roots[0] + '/dashboard', roots), true); assert.equal(withinRoots(roots[0] + 'Else', roots), false); assert.equal(withinRoots(roots[0] + '/../Other', roots), false);
+  for (const metadata of [meta({ cwd: 'C:/Other' }), meta({ source: { subagent: { parent_thread_id: 'parent' } } })]) assert.equal((await extract(t, [metadata, event('Request')], 'codex')).candidates.length, 0);
+  assert.equal((await extract(t, [event('No metadata')], 'codex')).candidates.length, 0);
+  const proper = await extract(t, [meta(), response('Replayed old text'), event('Actual text'), response('Actual text')], 'codex');
+  assert.equal(proper.candidates.length, 1); assert.equal(proper.candidates[0].row.message, 'Actual text'); assert.equal(proper.candidates[0].row.context_missing, true);
+  const fallback = await extract(t, [meta(), response('Fallback text')], 'codex'); assert.equal(fallback.candidates.length, 1); assert.equal(fallback.stats.response_fallback_candidates, 1);
+});
+test('whole Claude sidechain file rejected even after first candidate', async t => {
+  const { candidates } = await extract(t, [claude('Earlier request'), claude('Subagent request', { isSidechain: true })]); assert.equal(candidates.length, 0);
+});
+test('automation seeds exclude entire session while genuine TASK references remain', async t => {
+  const seeds = ['- ## Automated Dispatch Context\nThis prompt was machine-assembled by the Kenaz orchestration system.', 'This prompt was machine-assembled by the Kenaz orchestration system.', '- Extract key technical insights from this conversation chunk. Output in English only. Be concise, 1-3 sentences.', '[HEADLESS MODE] 你已被 Formation Lead 指派任務', 'You have access to MCP tools from kenaz-dashboard. Call the synapse_status tool now.'];
+  for (const seed of seeds) {
+    assert.equal(automationSeed(seed), true);
+    const result = await extract(t, [claude('Prior automated followup'), claude(seed), claude('Continue automatically')]);
+    assert.equal(result.candidates.length, 0); assert.equal(result.stats.automation_seed, 1);
+  }
+  assert.equal(automationSeed('Please investigate TASK_35 and explain the Automated Dispatch Context header.'), false);
+  const genuine = await extract(t, [claude('Please investigate TASK_35'), claude('[Request interrupted by user for tool use]'), claude('<turn_aborted>The user interrupted the previous turn</turn_aborted>'), claude('Continue my investigation')]);
+  assert.equal(genuine.candidates.length, 2);
+  const smoke = await extract(t, [meta(), event(seeds.at(-1)), event('Continue')], 'codex'); assert.equal(smoke.candidates.length, 0);
+});
+test('automation discovered in one source excludes same session in another source', async t => {
+  const dir = await temp(t), claudeRoot = join(dir, 'claude'), codexRoot = join(dir, 'codex'), project = join(claudeRoot, 'C-KenazAI');
+  await mkdir(project, { recursive: true }); await mkdir(codexRoot);
+  await writeFile(join(project, 'a.jsonl'), JSON.stringify(claude('Earlier inherited automated turn')));
+  await writeFile(join(project, 'b.jsonl'), JSON.stringify(claude('[HEADLESS MODE] 你已被 Formation Lead 指派任務')));
+  const result = await inventory({ claudeRoot, codexRoot, roots }); assert.equal(result.rows.length, 0); assert.equal(result.summary.exclusions.cross_file_automation_candidates, 1);
+});
+test('Claude foreign/missing cwd excluded; changing session identity rejects entire file', async t => {
+  const foreign = await extract(t, [claude('Allowed first'), claude('Foreign', { cwd: 'C:/Other' }), claude('Allowed return'), claude('Missing', { cwd: undefined })]);
+  assert.equal(foreign.candidates.length, 2); assert.equal(foreign.candidates[1].row.context_missing, true);
+  const mixed = await extract(t, [claude('First'), claude('Other session', { sessionId: 's2' })]); assert.equal(mixed.candidates.length, 0); assert.equal(mixed.stats.session_identity_change, 1);
+  const reentry = await extract(t, [meta(), event('First'), { type: 'turn_context', payload: { cwd: 'C:/Other' } }, { type: 'turn_context', payload: { cwd: roots[0] } }, event('Return')], 'codex');
+  assert.equal(reentry.candidates[1].row.context_missing, true);
+});
+test('dedup and balanced sampling deterministic; generic duplicates do not merge families', async t => {
+  const { candidates: a } = await extract(t, [claude('Continue'), claude('Unique A1'), claude('Unique A2')]);
+  const { candidates: b } = await extract(t, [claude(' continue ', { sessionId: 's2' }), claude('Unique B1', { sessionId: 's2' }), claude('Unique B2', { sessionId: 's2' })]);
+  const input = [...a, ...b], sample = sampleCandidates(input, 4);
+  assert.equal(sample.unique_count, 5); assert.equal(sample.duplicate_count, 1); assert.equal(sample.group_count, 2);
+  assert.deepEqual(sampleCandidates([...input].reverse(), 4), sample); assert.equal(new Set(sample.selected.map(c => c.row.session_id)).size, 2);
+  const lengthy = 'Copied historical context '.repeat(10);
+  const c = await extract(t, [claude(lengthy)]), d = await extract(t, [claude(lengthy, { sessionId: 's2' })]);
+  assert.equal(sampleCandidates([...c.candidates, ...d.candidates]).group_count, 1);
+});
+test('stream bounds discard oversized lines then recover next JSON record', async t => {
+  const dir = await temp(t), path = join(dir, 'big.jsonl'); await writeFile(path, 'x'.repeat(200000) + '\n' + JSON.stringify({ ok: true }) + '\n');
+  const stats = {}, output = []; for await (const item of records(path, stats, { maxLineBytes: 100 })) output.push(item);
+  assert.equal(stats.oversize_line, 1); assert.deepEqual(output, [{ value: { ok: true }, line: 2 }]);
+});
+test('discovery excludes non-Kenaz Claude and nested subagents; summary only dry output', async t => {
+  const dir = await temp(t), claudeRoot = join(dir, 'claude'), codexRoot = join(dir, 'codex');
+  for (const path of [join(claudeRoot, 'C-KenazAI'), join(claudeRoot, 'C-Other'), join(claudeRoot, 'C-KenazAI', 'subagents'), codexRoot]) await mkdir(path, { recursive: true });
+  await writeFile(join(claudeRoot, 'C-KenazAI', 's.jsonl'), JSON.stringify(claude('Kept real user text')));
+  await writeFile(join(claudeRoot, 'C-KenazAI', 'agent-1.jsonl'), JSON.stringify(claude('Agent')));
+  await writeFile(join(claudeRoot, 'C-Other', 's.jsonl'), JSON.stringify(claude('Other')));
+  await writeFile(join(claudeRoot, 'C-KenazAI', 'subagents', 'a.jsonl'), JSON.stringify(claude('Nested')));
+  assert.equal((await discover({ claudeRoot, codexRoot })).length, 1);
+  const result = await inventory({ claudeRoot, codexRoot, roots, target: 500 }); assert.equal(result.summary.shortfall, 499);
+  const report = await main(['--claude-root', claudeRoot, '--codex-root', codexRoot]); assert.equal(report.written, false); assert.equal(JSON.stringify(report).includes('Kept real'), false);
+  await assert.rejects(main(['--target', 'NaN']), /invalid_target/);
+});

package/core/dev/jev-eval/test/eval.test.mjs

@@ -0,0 +1,146 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { createServer } from 'node:http';
+import { once } from 'node:events';
+import { spawnSync } from 'node:child_process';
+import { template, main } from '../cli.mjs';
+import { readJsonl, validateDataset, validatePrediction, stateHash, inputState } from '../schema.mjs';
+import { MODEL, RUBRIC, prediction, rules, replayProvider, liveProvider, buildRequest, parseAnswers } from '../providers.mjs';
+import { report } from '../metrics.mjs';
+
+const fixture = new URL('../fixtures/synthetic.jsonl', import.meta.url);
+const rows = validateDataset(await readJsonl(fixture));
+const clone = x => structuredClone(x);
+const row = () => template();
+function correctionResponse(noul = 0.9) { return { model: MODEL, answers: { correction: { type: 'noul', noul } }, usage: { input_tokens: 30, output_tokens: 2 } }; }
+async function server(t, handler) {
+  const http = createServer(handler); http.listen(0, '127.0.0.1'); await once(http, 'listening');
+  t.after(() => { http.closeAllConnections(); http.close(); });
+  return `http://127.0.0.1:${http.address().port}/v1/systemone`;
+}
+const live = (endpoint, extra = {}) => liveProvider({ apiKey: 'test-only-secret', endpoint, allowLoopbackForTests: true, timeoutMs: 2000, ...extra });
+
+test('varied synthetic fixtures are strict, immutable and never promotion eligible', async () => {
+  const before = JSON.stringify(rows);
+  assert.equal(rows.length, 11);
+  const result = report(rows, rows.map(rules), { provider: 'rules' });
+  assert.equal(result.promotion_eligible, false);
+  assert.equal(result.provenance.annotation.agent_proposed, rows.length);
+  assert.equal(result.metrics.relevance.zero_relevance_queries_excluded, 1);
+  assert.equal(JSON.stringify(rows), before);
+  assert.equal(JSON.stringify(result).includes(rows[0].state.message), false);
+  assert.equal((await main(['validate', '--dataset', fixture.pathname.replace(/^\/([A-Z]:)/i, '$1')])).valid, true);
+});
+test('schema rejects unknown fields, invalid labels, duplicates and dishonest review metadata', () => {
+  for (const mutate of [r => r.extra = true, r => r.labels.correction = 1, r => r.provenance.annotation = 'human_verified', r => r.state.message = '', r => r.event_seq = -1]) {
+    const r = row(); mutate(r); assert.throws(() => validateDataset([r]));
+  }
+  const valid = row(); valid.provenance = { source: 'real', annotation: 'human_verified', reviewer: 'reviewer-a', reviewed_at: '2026-09-20T00:00:00Z' };
+  assert.doesNotThrow(() => validateDataset([valid]));
+  assert.throws(() => validateDataset([row(), row()]));
+});
+test('group, session and exact-content split leakage rejected', () => {
+  for (const kind of ['group', 'session', 'content']) {
+    const a = row(), b = row(); b.id = 'other'; b.event_id = 'other'; b.split = 'held_out';
+    if (kind !== 'session') b.session_id = 'other';
+    if (kind !== 'group') b.group_id = 'other';
+    if (kind !== 'content') b.state.message = 'Different message';
+    assert.throws(() => validateDataset([a, b]), /leakage/);
+  }
+});
+test('model input and hash exclude ground truth and provenance', () => {
+  const r = clone(rows.find(r => r.purpose === 'relevance')), hash = stateHash(r);
+  r.candidates[0].grade = 0; r.provenance.reviewer = 'changed'; r.split = 'held_out';
+  assert.equal(stateHash(r), hash);
+  const input = JSON.stringify(inputState(r));
+  for (const forbidden of ['grade', 'provenance', 'reviewer', 'split', 'labels']) assert.equal(input.includes(`"${forbidden}"`), false);
+  r.state.message += ' revised'; assert.notEqual(stateHash(r), hash);
+});
+test('replay rejects wrong project/session/task, stale revision/hash, unknown IDs and aliases', () => {
+  const r = row(), p = prediction(r, 'ok', 0.8);
+  for (const field of ['project_id', 'session_id', 'task_id', 'event_id', 'task_revision', 'state_hash', 'model', 'question_set_version']) {
+    const bad = clone(p); bad[field] = 'wrong'; assert.throws(() => replayProvider([r], [bad]));
+  }
+  assert.throws(() => replayProvider([r], []));
+  assert.throws(() => replayProvider([r], [p, p]));
+  assert.equal(replayProvider([r], [p])(r), p);
+});
+test('correction metrics include failed positives, preserve unanswered Brier denominator', () => {
+  const data = [true, true, false, false].map((label, i) => ({ ...row(), id: `m${i}`, labels: { correction: label } }));
+  const preds = [prediction(data[0], 'ok', 0.8), prediction(data[1], 'timeout'), prediction(data[2], 'ok', 0.6), prediction(data[3], 'ok', 0.1)];
+  const result = report(data, preds, { provider: 'replay' }), m = result.metrics.correction;
+  assert.deepEqual(m.confusion, { tp: 1, fp: 1, tn: 1, fn: 0 });
+  assert.equal(m.coverage, 0.75); assert.equal(m.recall_including_abstentions, 0.5); assert.equal(m.recall_answered_only, 1);
+  assert.equal(m.precision, 0.5); assert.equal(m.accuracy_including_abstentions, 0.5);
+  assert.ok(Math.abs(m.brier_answered_only - 0.41 / 3) < 1e-12); assert.equal(m.brier_count, 3);
+  const empty = report([data[1]], [prediction(data[1], 'abstain')], { provider: 'replay' }).metrics.correction;
+  assert.equal(empty.precision, null); assert.equal(empty.brier_answered_only, null); assert.equal(empty.recall_including_abstentions, 0);
+});
+test('ranking uses full ideal, tracks partial scores and handles zero relevance', () => {
+  const r = clone(rows.find(r => r.purpose === 'relevance'));
+  const partial = prediction(r, 'ok', { [r.candidates[1].id]: 3 });
+  const m = report([r], [partial], { provider: 'replay', k: 1 }).metrics.relevance;
+  assert.equal(m.ndcg_at_k, 1 / 7); assert.equal(m.recall_at_k, 0.5); assert.equal(m.query_coverage, 0); assert.equal(m.candidate_coverage, 0.25); assert.equal(m.ndcg_complete_queries_only, null);
+  const failed = report([r], [prediction(r, 'timeout')], { provider: 'replay' }).metrics.relevance;
+  assert.equal(failed.ndcg_at_k, 0); assert.equal(failed.recall_at_k, 0);
+  const zero = rows.at(-1), z = report([zero], [rules(zero)], { provider: 'rules' }).metrics.relevance;
+  assert.equal(z.ndcg_at_k, null); assert.equal(z.zero_relevance_queries_excluded, 1);
+  partial.scores[0].id = 'wrong'; assert.throws(() => validatePrediction(r, partial));
+});
+test('missing/blank key and unsafe URL rejected before any network', () => {
+  let calls = 0; const fetchImpl = () => { calls++; throw new Error('unexpected'); };
+  for (const apiKey of ['', ' ', '\r\n']) assert.throws(() => liveProvider({ apiKey, fetchImpl }), /api_key/);
+  for (const endpoint of ['http://evil.example', 'https://api.typesafe.ai.evil.example/v1/systemone', 'https://api.typesafe.ai/v1/systemone?key=x', 'http://localhost:1234']) assert.throws(() => liveProvider({ apiKey: 'fake', endpoint, fetchImpl }), /unsafe_endpoint/);
+  assert.equal(calls, 0);
+  const child = spawnSync(process.execPath, ['dev/jev-eval/cli.mjs', 'evaluate', '--dataset', 'dev/jev-eval/fixtures/synthetic.jsonl', '--provider', 'live'], { env: { ...process.env, TYPESAFE_API_KEY: '' }, encoding: 'utf8' });
+  assert.equal(child.status, 1); assert.deepEqual(JSON.parse(child.stderr), { error: 'missing_or_invalid_api_key' });
+});
+test('successful HTTP batches by row and tracks usage without gold leakage', async t => {
+  const endpoint = await server(t, async (req, res) => {
+    const parts = []; for await (const chunk of req) parts.push(chunk);
+    const request = JSON.parse(Buffer.concat(parts));
+    assert.equal(request.model, MODEL); assert.equal(request.state.labels, undefined); assert.equal(req.headers.authorization, 'Bearer test-only-secret');
+    res.setHeader('Content-Type', 'application/json'); res.end(JSON.stringify(correctionResponse()));
+  });
+  const p = await live(endpoint)(row()); assert.equal(p.status, 'ok'); assert.equal(p.correction_probability, 0.9); assert.equal(p.usage.input_tokens, 30); validatePrediction(row(), p);
+});
+test('HTTP error statuses abstain without leaking raw provider errors', async t => {
+  for (const status of [401, 422, 429, 529]) {
+    const endpoint = await server(t, (req, res) => { res.writeHead(status); res.end('secret-body test-only-secret'); });
+    const p = await live(endpoint)(row()); assert.equal(p.status, 'unavailable'); assert.equal(p.http_status, status); assert.equal(p.correction_probability, null); assert.equal(JSON.stringify(p).includes('secret'), false);
+  }
+});
+test('redirect rejected without contacting target', async t => {
+  let targetCalls = 0;
+  const target = await server(t, (req, res) => { targetCalls++; res.end('{}'); });
+  const endpoint = await server(t, (req, res) => { res.writeHead(302, { Location: target }); res.end(); });
+  const p = await live(endpoint)(row()); assert.equal(p.status, 'unavailable'); assert.equal(targetCalls, 0);
+});
+test('timeout remains active across response body and also covers stalled transport', async t => {
+  const endpoint = await server(t, (req, res) => { res.writeHead(200, { 'Content-Type': 'application/json' }); res.flushHeaders(); res.write('{'); });
+  const start = Date.now(), p = await live(endpoint, { timeoutMs: 80 })(row());
+  assert.equal(p.status, 'timeout'); assert.ok(Date.now() - start < 3000);
+  const stalled = await liveProvider({ apiKey: 'fake', timeoutMs: 20, fetchImpl: () => new Promise(() => {}) })(row());
+  assert.equal(stalled.status, 'timeout');
+});
+test('invalid JSON, oversized response and malformed answers fail closed', async t => {
+  const invalids = ['{', 'x'.repeat(65537), JSON.stringify(correctionResponse(1.1)), JSON.stringify({ ...correctionResponse(), answers: { wrong: { type: 'noul', noul: 0.9 } } }), JSON.stringify({ ...correctionResponse(), usage: null })];
+  for (const body of invalids) {
+    const endpoint = await server(t, (req, res) => res.end(body));
+    const p = await live(endpoint)(row()); assert.equal(p.status, 'invalid'); assert.equal(p.correction_probability, null);
+  }
+});
+test('score answers enforce distributions, legends, bounds, confidence and exact IDs', () => {
+  const r = rows.find(r => r.purpose === 'relevance'), request = buildRequest(r);
+  const answer = { type: 'score', score: 2, legend: Object.fromEntries(RUBRIC.map((v, i) => [i, v])), probabilities: { 0: 0, 1: 0, 2: 1, 3: 0 }, confidence: 1 };
+  const body = { model: MODEL, answers: Object.fromEntries(Object.keys(request.questions).map(k => [k, clone(answer)])), usage: { input_tokens: 100, output_tokens: 8 } };
+  assert.equal(Object.keys(parseAnswers(r, body, request)).length, r.candidates.length);
+  for (const mutate of [a => a.score = NaN, a => a.probabilities[2] = 0.3, a => a.legend[0] = 'wrong', a => a.confidence = 2, a => a.probabilities[4] = 0, a => a.type = 'noul']) {
+    const bad = clone(body); mutate(bad.answers.candidate_0); assert.throws(() => parseAnswers(r, bad, request));
+  }
+});
+test('CLI defaults offline, rejects ambiguous options, and preflight prints no key', async () => {
+  assert.equal((await main(['preflight'])).network_attempted, false);
+  assert.equal((await main(['template'])).schema_version, 1);
+  for (const args of [['evaluate', '--endpoint', 'http://evil'], ['evaluate', '--dataset', 'a', '--dataset', 'b'], ['preflight', '--provider', 'live']]) await assert.rejects(main(args));
+});

package/core/dev/jev-eval/test/muninn-baseline.test.mjs

@@ -0,0 +1,71 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { readFile } from 'node:fs/promises';
+import { createHash } from 'node:crypto';
+import { resolve } from 'node:path';
+import { readJsonl } from '../schema.mjs';
+import { runBaseline, strictAnswers, safeInvocation, nativeTransport, main } from '../muninn-baseline.mjs';
+
+const source = await readFile(new URL('../../../.formation/tools/muninn.js', import.meta.url), 'utf8');
+const rows = await readJsonl(new URL('../fixtures/synthetic.jsonl', import.meta.url));
+const corrections = rows.filter(r => r.purpose === 'correction');
+const answers = () => corrections.map((r, idx) => ({ idx, translated: `Synthetic translation ${idx}`, labels: r.labels.correction ? ['correction'] : [] }));
+const response = items => ({ status: 0, stdout: JSON.stringify({ result: JSON.stringify(items), modelUsage: { 'claude-haiku-test': { inputTokens: 20 } } }) });
+
+test('offline default captures actual source prompt/parser without model quality claim', async () => {
+  const result = await main([]);
+  assert.equal(result.live, false); assert.equal(result.promotion_eligible, false); assert.equal(result.status, 'ok');
+  assert.equal(result.batch_count, 8); assert.equal(result.translation_present, 8); assert.equal(result.correction_confusion, null);
+  assert.equal(result.source_sha256, createHash('sha256').update(source).digest('hex'));
+  assert.equal(result.source_requested_permission_bypass, true); assert.equal(result.actual_permission_bypass, false);
+});
+test('isolated source passes exact indexed messages, single transport and stable confusion', () => {
+  let calls = 0; const before = JSON.stringify(rows);
+  const result = runBaseline(rows, { source, live: true, transport: input => {
+    calls++; assert.ok(input.startsWith('For each numbered message, output a JSON array'));
+    assert.ok(input.includes('length_too_short')); assert.ok(!input.includes('Development startup also fails'));
+    corrections.forEach((r, idx) => assert.ok(input.includes(`[${idx}] ${r.state.message.replace(/\r?\n/g, ' ')}`)));
+    return response(answers());
+  } });
+  assert.equal(calls, 1); assert.equal(result.status, 'ok'); assert.equal(JSON.stringify(rows), before);
+  assert.deepEqual(result.correction_confusion, { tp: 4, fp: 0, tn: 4, fn: 0, precision: 1, recall: 1 });
+  assert.deepEqual(result.observed_model_metadata, ['claude-haiku-test']);
+  assert.equal(JSON.stringify(result).includes(corrections[0].state.message), false);
+});
+test('strict raw checking rejects identities and labels before permissive source filters them', () => {
+  const changes = [a => a.pop(), a => a[1].idx = a[0].idx, a => a.reverse(), a => a[0].idx = 0.5, a => a[0].translated = ' ', a => a[0].labels = ['invented'], a => a[0].labels = ['correction', 'correction'], a => a[0].extra = true];
+  for (const change of changes) {
+    const items = answers(); change(items); assert.throws(() => strictAnswers(JSON.stringify(items), corrections.length));
+    const result = runBaseline(rows, { source, live: true, transport: () => response(items) });
+    assert.notEqual(result.status, 'ok'); assert.equal(result.correction_confusion, null); assert.equal(result.translation_present, 0);
+  }
+});
+test('failures timeouts malformed envelopes and oversize output remain sanitized failures', () => {
+  for (const transport of [() => ({ status: 1, stdout: 'private-error' }), () => ({ status: null, error: { code: 'ETIMEDOUT' } }), () => ({ status: 0, stdout: '{' }), () => ({ status: 0, stdout: 'x'.repeat(1048577) }), () => { throw 'private-error'; }, () => ({ status: 0, stdout: JSON.stringify({ is_error: true, result: 'private-error' }) })]) {
+    const result = runBaseline(rows, { source, live: true, transport });
+    assert.notEqual(result.status, 'ok'); assert.equal(result.correction_confusion, null); assert.equal(JSON.stringify(result).includes('private-error'), false);
+  }
+});
+test('real data rejected before transport including irrelevant purpose rows', () => {
+  let calls = 0; const real = structuredClone(rows); real.at(-1).provenance.source = 'real';
+  assert.throws(() => runBaseline(real, { source, live: true, transport: () => { calls++; } }), /synthetic_only/); assert.equal(calls, 0);
+});
+test('VM denies filesystem/network and unrelated module calls; empty environment', () => {
+  for (const expression of ["require('fs').readFileSync('x')", "require('http').request('x')", "require('os').homedir()", "require('net')", "require('./log-reader').findTaskLog()"])
+    assert.throws(() => runBaseline(rows, { source: source + '\n' + expression }), /blocked_vm/);
+  const checked = source + '\nif (Object.keys(process.env).length || require.main === module) throw new Error("unexpected_environment");';
+  assert.equal(runBaseline(rows, { source: checked }).status, 'ok');
+});
+test('safe subprocess uses explicit executable argv no shell and bounded output/time', () => {
+  const exe = resolve('test-claude.exe'), invocation = safeInvocation(exe);
+  for (const flag of ['--safe-mode', '--tools', '--disable-slash-commands', '--no-session-persistence']) assert.ok(invocation.args.includes(flag));
+  assert.equal(invocation.args[invocation.args.indexOf('--tools') + 1], '');
+  assert.equal(invocation.args.includes('--dangerously-skip-permissions'), false);
+  let calls = 0;
+  nativeTransport(exe, '/isolated', (command, args, options) => {
+    calls++; assert.equal(command, exe); assert.deepEqual(args, invocation.args); assert.equal(options.shell, false);
+    assert.equal(options.timeout, 30000); assert.equal(options.maxBuffer, 1048576); assert.equal(options.windowsHide, true);
+    assert.equal(Object.hasOwn(options.env, 'TYPESAFE_API_KEY'), false); return { status: 0, stdout: '{}' };
+  })('synthetic prompt'); assert.equal(calls, 1);
+  for (const bad of ['claude', resolve('claude.cmd'), resolve('claude.bat')]) assert.throws(() => safeInvocation(bad), /invalid_claude_cli/);
+});