kenaz 0.3.22 → 0.3.23

package/core/dev/jev-eval/review.mjs

@@ -1,78 +0,0 @@
-import { createHash, randomUUID } from 'node:crypto';
-import { readFile, stat, writeFile, link, unlink } from 'node:fs/promises';
-import { resolve } from 'node:path';
-import { fileURLToPath } from 'node:url';
-import { check, keys, identity, stateHash, readJsonl, validateDataset, MAX_FILE_BYTES } from './schema.mjs';
-
-export const REVIEW_VERSION = 'jev-review-v1';
-const canonical = value => JSON.stringify(value, function (key, item) {
-  return item && typeof item === 'object' && !Array.isArray(item) ? Object.fromEntries(Object.keys(item).sort().map(k => [k, item[k]])) : item;
-});
-export const datasetHash = rows => createHash('sha256').update(canonical(rows)).digest('hex');
-export function exportReview(rows) {
-  validateDataset(rows);
-  return { schema_version: 1, review_version: REVIEW_VERSION, dataset_hash: datasetHash(rows), items: rows.map(row => ({
-    ...identity(row), state_hash: stateHash(row), purpose: row.purpose, language: row.language, context: structuredClone(row.state),
-    ...(row.purpose === 'relevance' ? { candidates: row.candidates.map(({ id, text }) => ({ id, text })) } : {}),
-    decision: row.purpose === 'correction' ? { correction: null } : { grades: row.candidates.map(c => ({ id: c.id, grade: null })) },
-    reviewed: false, reviewer: null, reviewed_at: null,
-  })) };
-}
-function validTime(value) {
-  if (typeof value !== 'string' || !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z$/.test(value)) return false;
-  const time = Date.parse(value);
-  return Number.isFinite(time) && new Date(time).toISOString() === value.replace(/(?<!\.\d{3})Z$/, '.000Z');
-}
-export function applyReview(rows, review) {
-  const expected = exportReview(rows);
-  keys(review, ['schema_version', 'review_version', 'dataset_hash', 'items']);
-  check(review.schema_version === 1 && review.review_version === REVIEW_VERSION, 'invalid_review_version');
-  check(review.dataset_hash === expected.dataset_hash, 'stale_dataset');
-  check(Array.isArray(review.items) && review.items.length === rows.length, 'incomplete_review');
-  const byId = new Map();
-  for (const item of review.items) { check(item && !byId.has(item.id), 'duplicate_review'); byId.set(item.id, item); }
-  const result = rows.map((row, index) => {
-    const item = byId.get(row.id), base = expected.items[index]; keys(item, Object.keys(base));
-    for (const key of Object.keys(base).filter(k => !['decision', 'reviewed', 'reviewer', 'reviewed_at'].includes(k))) check(canonical(item[key]) === canonical(base[key]), 'review_identity_mismatch');
-    check(item.reviewed === true, 'unreviewed_item');
-    check(typeof item.reviewer === 'string' && item.reviewer.trim().length > 0 && item.reviewer.length <= 160, 'invalid_reviewer');
-    check(validTime(item.reviewed_at), 'invalid_review_date');
-    const out = structuredClone(row);
-    if (row.purpose === 'correction') { keys(item.decision, ['correction']); check(typeof item.decision.correction === 'boolean', 'invalid_review_decision'); out.labels.correction = item.decision.correction; }
-    else {
-      keys(item.decision, ['grades']);
-      check(Array.isArray(item.decision.grades) && item.decision.grades.length === row.candidates.length, 'incomplete_grades');
-      const grades = new Map();
-      for (const c of item.decision.grades) { keys(c, ['id', 'grade']); check(!grades.has(c.id) && row.candidates.some(x => x.id === c.id) && Number.isInteger(c.grade) && c.grade >= 0 && c.grade <= 3, 'invalid_review_grade'); grades.set(c.id, c.grade); }
-      for (const c of out.candidates) c.grade = grades.get(c.id);
-    }
-    out.provenance = { ...out.provenance, annotation: 'human_verified', reviewer: item.reviewer, reviewed_at: item.reviewed_at };
-    return out;
-  });
-  return validateDataset(result);
-}
-export async function main(args = process.argv.slice(2)) {
-  const [command, ...rest] = args; check(['export', 'apply'].includes(command), 'invalid_command');
-  const options = {};
-  for (let i = 0; i < rest.length; i += 2) { check(['--dataset', '--out', ...(command === 'apply' ? ['--reviews'] : [])].includes(rest[i]) && typeof rest[i + 1] === 'string' && !rest[i + 1].startsWith('--') && !Object.hasOwn(options, rest[i]), 'invalid_arguments'); options[rest[i]] = rest[i + 1]; }
-  check(options['--dataset'] && options['--out'] && (command !== 'apply' || options['--reviews']), 'missing_argument');
-  check(resolve(options['--dataset']).toLowerCase() !== resolve(options['--out']).toLowerCase() && (!options['--reviews'] || resolve(options['--reviews']).toLowerCase() !== resolve(options['--out']).toLowerCase()), 'overwrite_forbidden');
-  const rows = validateDataset(await readJsonl(options['--dataset']));
-  let output;
-  if (command === 'export') output = JSON.stringify(exportReview(rows), null, 2) + '\n';
-  else {
-    check((await stat(options['--reviews'])).size <= MAX_FILE_BYTES, 'file_budget_exceeded');
-    const text = await readFile(options['--reviews'], 'utf8'); check(Buffer.byteLength(text) <= MAX_FILE_BYTES, 'file_budget_exceeded');
-    let review; try { review = JSON.parse(text.replace(/^\uFEFF/, '')); } catch { throw new Error('invalid_review_json'); }
-    output = applyReview(rows, review).map(r => JSON.stringify(r)).join('\n') + '\n';
-  }
-  check(Buffer.byteLength(output) <= MAX_FILE_BYTES, 'file_budget_exceeded');
-  // A same-directory hard link publishes only the complete file, and fails if out exists.
-  const temporary = `${options['--out']}.${randomUUID()}.tmp`;
-  try { await writeFile(temporary, output, { flag: 'wx', encoding: 'utf8' }); await link(temporary, options['--out']); }
-  finally { await unlink(temporary).catch(() => {}); }
-  return { schema_version: 1, operation: command, count: rows.length, promotion_eligible: false, human_review_is_attestation_not_proof: true };
-}
-if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) main().then(r => console.log(JSON.stringify(r, null, 2))).catch(error => {
-  console.error(JSON.stringify({ error: /^[a-z_]{1,80}$/.test(error.message) ? error.message : error.code === 'EEXIST' ? 'output_exists' : 'operation_failed' })); process.exitCode = 1;
-});

package/core/dev/jev-eval/schema.mjs

@@ -1,87 +0,0 @@
-import { readFile, stat } from 'node:fs/promises';
-import { createHash } from 'node:crypto';
-
-export const MAX_FILE_BYTES = 8 * 1024 * 1024;
-export const MAX_RECORD_BYTES = 64 * 1024;
-export const SPLITS = ['development', 'calibration', 'held_out'];
-export function check(condition, code = 'invalid_data') { if (!condition) throw new Error(code); }
-export const object = x => x !== null && typeof x === 'object' && !Array.isArray(x);
-export const number = (x, min, max) => typeof x === 'number' && Number.isFinite(x) && x >= min && x <= max;
-const string = x => typeof x === 'string' && x.trim().length > 0 && x.length <= 16384;
-const identifier = x => typeof x === 'string' && /^[\w.:/-]{1,160}$/.test(x);
-export function keys(x, names) {
-  check(object(x) && Object.keys(x).length === names.length && names.every(k => Object.hasOwn(x, k)), 'invalid_fields');
-}
-export async function readJsonl(path) {
-  check((await stat(path)).size <= MAX_FILE_BYTES, 'file_budget_exceeded');
-  const text = await readFile(path, 'utf8');
-  check(Buffer.byteLength(text) <= MAX_FILE_BYTES, 'file_budget_exceeded');
-  try { return text.replace(/^\uFEFF/, '').split(/\r?\n/).filter(x => x.trim()).map(x => {
-    check(Buffer.byteLength(x) <= MAX_RECORD_BYTES, 'record_budget_exceeded');
-    return JSON.parse(x);
-  }); } catch { throw new Error('invalid_jsonl'); }
-}
-export function inputState(row) {
-  return { purpose: row.purpose, ...row.state, ...(row.purpose === 'relevance' ? { candidates: row.candidates.map(({ id, text }) => ({ id, text })) } : {}) };
-}
-export const stateHash = row => createHash('sha256').update(JSON.stringify(inputState(row))).digest('hex');
-export function identity(row) {
-  return Object.fromEntries(['id', 'project_id', 'session_id', 'task_id', 'event_id', 'event_seq', 'task_revision'].map(k => [k, row[k]]));
-}
-export function validateDataset(rows) {
-  check(Array.isArray(rows) && rows.length > 0 && rows.length <= 10000, 'invalid_dataset_size');
-  const ids = new Set(), events = new Set(), splits = new Map(), texts = new Map();
-  for (const row of rows) {
-    check(Buffer.byteLength(JSON.stringify(row)) <= MAX_RECORD_BYTES, 'record_budget_exceeded');
-    keys(row, ['schema_version', 'id', 'project_id', 'session_id', 'group_id', 'task_id', 'event_id', 'event_seq', 'task_revision', 'purpose', 'language', 'split', 'provenance', 'state', row.purpose === 'correction' ? 'labels' : 'candidates']);
-    check(row.schema_version === 1, 'invalid_schema_version');
-    for (const k of ['id', 'project_id', 'session_id', 'group_id', 'task_id', 'event_id']) check(identifier(row[k]), 'invalid_identifier');
-    for (const k of ['event_seq', 'task_revision']) check(Number.isSafeInteger(row[k]) && row[k] >= 0, 'invalid_revision');
-    check(['correction', 'relevance'].includes(row.purpose) && ['en', 'zh', 'mixed'].includes(row.language) && SPLITS.includes(row.split), 'invalid_enum');
-    keys(row.provenance, ['source', 'annotation', 'reviewer', 'reviewed_at']);
-    check(['synthetic', 'real'].includes(row.provenance.source) && ['agent_proposed', 'human_verified'].includes(row.provenance.annotation), 'invalid_provenance');
-    check(row.provenance.annotation === 'human_verified' ? string(row.provenance.reviewer) : row.provenance.reviewer === null, 'invalid_reviewer');
-    check(row.provenance.annotation === 'human_verified' ? typeof row.provenance.reviewed_at === 'string' && /^\d{4}-\d{2}-\d{2}T/.test(row.provenance.reviewed_at) && Number.isFinite(Date.parse(row.provenance.reviewed_at)) : row.provenance.reviewed_at === null, 'invalid_review_date');
-    keys(row.state, ['goal', 'assumptions', 'message']);
-    check(string(row.state.goal) && string(row.state.message) && Array.isArray(row.state.assumptions) && row.state.assumptions.length <= 20 && row.state.assumptions.every(string), 'invalid_state');
-    if (row.purpose === 'correction') { keys(row.labels, ['correction']); check(typeof row.labels.correction === 'boolean', 'invalid_label'); }
-    else {
-      check(Array.isArray(row.candidates) && row.candidates.length >= 2 && row.candidates.length <= 12, 'invalid_candidates');
-      const candidateIds = new Set();
-      for (const c of row.candidates) { keys(c, ['id', 'text', 'grade']); check(identifier(c.id) && string(c.text) && Number.isInteger(c.grade) && number(c.grade, 0, 3) && !candidateIds.has(c.id), 'invalid_candidate'); candidateIds.add(c.id); }
-    }
-    check(!ids.has(row.id), 'duplicate_id'); ids.add(row.id);
-    const event = JSON.stringify([row.project_id, row.session_id, row.event_id, row.purpose]);
-    check(!events.has(event), 'duplicate_event'); events.add(event);
-    for (const group of [`session:${row.project_id}:${row.session_id}`, `group:${row.group_id}`]) {
-      check(!splits.has(group) || splits.get(group) === row.split, 'split_leakage'); splits.set(group, row.split);
-    }
-    const hash = stateHash(row);
-    check(!texts.has(hash) || texts.get(hash) === row.split, 'content_split_leakage'); texts.set(hash, row.split);
-  }
-  return rows;
-}
-export function validatePrediction(row, prediction) {
-  keys(prediction, [...Object.keys(identity(row)), 'state_hash', 'model', 'question_set_version', 'status', 'failure_reason', 'http_status', 'usage', 'latency_ms', row.purpose === 'correction' ? 'correction_probability' : 'scores']);
-  for (const [k, v] of Object.entries(identity(row))) check(prediction[k] === v, 'prediction_identity_mismatch');
-  check(prediction.state_hash === stateHash(row), 'stale_prediction');
-  check(['jev-1.13.0', 'rules-v1'].includes(prediction.model) && prediction.question_set_version === 'jev-eval-v1', 'invalid_prediction_version');
-  check(['ok', 'abstain', 'timeout', 'unavailable', 'invalid'].includes(prediction.status), 'invalid_status');
-  check(number(prediction.latency_ms, 0, 3600000), 'invalid_latency');
-  check(prediction.http_status === null || Number.isInteger(prediction.http_status) && number(prediction.http_status, 100, 599), 'invalid_http_status');
-  check(prediction.status === 'ok' ? prediction.failure_reason === null : ['timeout', 'http_failure', 'transport_failure', 'invalid_response', 'abstain'].includes(prediction.failure_reason), 'invalid_failure_reason');
-  if (prediction.usage !== null) { keys(prediction.usage, ['input_tokens', 'output_tokens']); for (const n of Object.values(prediction.usage)) check(Number.isSafeInteger(n) && n >= 0, 'invalid_usage'); }
-  if (row.purpose === 'correction') {
-    check(prediction.status === 'ok' ? number(prediction.correction_probability, 0, 1) : prediction.correction_probability === null, 'invalid_probability');
-  } else {
-    check(Array.isArray(prediction.scores) && prediction.scores.length === row.candidates.length, 'invalid_scores');
-    const expected = new Set(row.candidates.map(c => c.id));
-    for (const item of prediction.scores) {
-      keys(item, ['id', 'score']); check(expected.delete(item.id), 'invalid_score_id');
-      check(item.score === null || number(item.score, 0, 3), 'invalid_score');
-      if (prediction.status !== 'ok') check(item.score === null, 'failure_with_score');
-    }
-    if (prediction.status === 'ok') check(prediction.scores.some(s => s.score !== null), 'empty_ok_prediction');
-  }
-  return prediction;
-}

package/core/dev/jev-eval/test/context.test.mjs

@@ -1,95 +0,0 @@
-import test from 'node:test';
-import assert from 'node:assert/strict';
-import { mkdtemp, writeFile, symlink, readdir, unlink, rmdir, lstat } from 'node:fs/promises';
-import { createHash } from 'node:crypto';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { enrich, resolveSource, renderHtml } from '../context.mjs';
-import { normalizeMessage, redactHomePaths, credentialBearing } from '../corpus.mjs';
-
-const hash = text => createHash('sha256').update(text).digest('hex'), cwd = 'C:/Users/test/Documents/GitHub/KenazAI';
-const c = (uuid, parentUuid, role, text, extra = {}) => ({ uuid, parentUuid, sessionId: 's1', cwd, type: role, message: { role, content: [{ type: 'text', text }] }, ...extra });
-async function fixture(t, kind, records, targetLine, overrides = {}) {
-  const dir = await mkdtemp(join(tmpdir(), 'jev-context-test-'));
-  t.after(async () => { for (const name of await readdir(dir)) { const p = join(dir, name), s = await lstat(p); if (s.isSymbolicLink() && process.platform === 'win32') await rmdir(p); else await unlink(p); } await rmdir(dir); });
-  const path = join(dir, 'source.jsonl'), bytes = records.map(r => JSON.stringify(r) + '\n').join(''); await writeFile(path, bytes);
-  const source = { kind, relative_path: 'source.jsonl', source_id: hash(`${kind}:source.jsonl`) };
-  const target = records[targetLine - 1], message = normalizeMessage(redactHomePaths(kind === 'claude' ? target.message.content[0].text : target.payload.message));
-  const row = { kind: 'pending_historical_review', id: hash(`${source.source_id}:${targetLine}:${hash(message.toLowerCase())}`), source_id: source.source_id, session_id: hash(`${kind}:s1`), group_id: 'group', message, review_status: 'pending', provenance: { source: 'real', annotation: 'unreviewed' }, ...overrides };
-  return { dir, path, bytes, row, source, manifest: { schema_version: 1, sources: [source], items: [{ id: row.id, source_id: source.source_id, line: targetLine }] }, options: { sourceRoots: { claude: dir, codex: dir }, cwdRoots: [cwd] } };
-}
-test('Claude ancestry follows parent UUID rather than neighboring branch, skips thinking/tools/future', async t => {
-  const records = [c('u1', null, 'user', 'Start'), c('a1', 'u1', 'assistant', 'Correct branch'), c('wrong', 'u1', 'assistant', 'Other branch'), c('tool', 'a1', 'user', '', { message: { role: 'user', content: [{ type: 'tool_result', content: 'secret tool output' }] } }), c('thinking', 'tool', 'assistant', '', { message: { role: 'assistant', content: [{ type: 'thinking', thinking: 'hidden reasoning' }] } }), c('target', 'thinking', 'user', 'Please revise'), c('future', 'target', 'assistant', 'Future answer')];
-  const f = await fixture(t, 'claude', records, 6), before = JSON.stringify(f.row), result = await enrich([f.row], f.manifest, f.options), row = result.rows[0];
-  assert.equal(row.status, 'restored'); assert.deepEqual(row.observed_context.map(c => c.text), ['Start', 'Correct branch']);
-  assert.ok(row.observed_context.every(c => c.source_line < row.target_line)); assert.equal(JSON.stringify(row).includes('hidden reasoning'), false); assert.equal(JSON.stringify(row).includes('Other branch'), false);
-  assert.equal(row.source_prefix_sha256, hash(records.slice(0, 6).map(r => JSON.stringify(r) + '\n').join(''))); assert.equal(JSON.stringify(f.row), before); assert.equal(row.decision.correction, null);
-});
-test('stale targets and wrong session rejected without dropping IDs', async t => {
-  const f = await fixture(t, 'claude', [c('a', null, 'assistant', 'Context'), c('u', 'a', 'user', 'Target')], 2);
-  for (const change of [r => r.message = 'Changed', r => r.session_id = hash('other'), r => r.review_status = 'human_verified']) {
-    const row = structuredClone(f.row); change(row); const result = await enrich([row], f.manifest, f.options); assert.equal(result.rows[0].id, row.id); assert.equal(result.rows[0].status, 'rejected'); assert.equal(result.rows[0].observed_context, undefined);
-  }
-});
-test('source paths reject traversal absolute paths hash mismatch and symlink escape', async t => {
-  const f = await fixture(t, 'claude', [c('u', null, 'user', 'Target')], 1);
-  for (const relative_path of ['../outside', '/absolute', 'C:\\absolute', 'nested/../../outside']) await assert.rejects(resolveSource({ kind: 'claude', relative_path, source_id: hash(`claude:${relative_path}`) }, f.options.sourceRoots), /unsafe_source_path/);
-  await assert.rejects(resolveSource({ ...f.source, source_id: 'wrong' }, f.options.sourceRoots), /source_id_mismatch/);
-  const outside = await fixture(t, 'claude', [c('u', null, 'user', 'Other')], 1), link = join(f.dir, 'escape');
-  await symlink(outside.dir, link, process.platform === 'win32' ? 'junction' : 'dir');
-  await assert.rejects(resolveSource({ kind: 'claude', relative_path: 'escape/source.jsonl', source_id: hash('claude:escape/source.jsonl') }, f.options.sourceRoots), /source_path_escape/);
-});
-test('credential context dropped as whole message and gaps remain explicit', async t => {
-  const f = await fixture(t, 'claude', [c('a1', null, 'assistant', 'Safe context'), c('a2', 'a1', 'assistant', 'Use apikey_abcdefgh12345 now'), c('u', 'a2', 'user', 'Continue')], 3);
-  const row = (await enrich([f.row], f.manifest, f.options)).rows[0]; assert.equal(row.status, 'partial'); assert.deepEqual(row.observed_context.map(c => c.text), ['Safe context']); assert.ok(row.context_gaps.includes('credential_context')); assert.equal(JSON.stringify(row).includes('apikey_'), false);
-});
-test('normalization and JSON quote escaping cannot hide credential context', async t => {
-  const escaped = 'API_KEY="abcd1234"'; assert.equal(credentialBearing(escaped), true); assert.equal(credentialBearing(JSON.stringify(escaped)), false);
-  for (const text of [escaped, 'The config is {"API_KEY":"abc12345"}', 'Use ＡＰＩ＿ＫＥＹ＝ａｂｃ１２３４５ locally', 'ＡＰＩＫＥＹ＿ａｂｃｄｅｆｇｈ１２３４５']) {
-    assert.equal(credentialBearing(normalizeMessage(text)), true);
-    const f = await fixture(t, 'claude', [c('safe', null, 'assistant', 'Safe earlier text'), c('secret', 'safe', 'assistant', text), c('target', 'secret', 'user', 'Continue')], 3);
-    const row = (await enrich([f.row], f.manifest, f.options)).rows[0];
-    assert.deepEqual(row.observed_context.map(c => c.text), ['Safe earlier text']);
-    assert.ok(row.context_gaps.includes('credential_context')); assert.ok(row.observed_context.every(c => !credentialBearing(c.text)));
-  }
-  const target = await fixture(t, 'claude', [c('user', null, 'user', 'ＡＰＩ＿ＫＥＹ＝ａｂｃ１２３４５')], 1);
-  assert.equal((await enrich([target.row], target.manifest, target.options)).rows[0].status, 'rejected');
-});
-const meta = { type: 'session_meta', payload: { id: 's1', cwd, source: 'cli' } };
-const assistant = text => ({ type: 'response_item', payload: { type: 'message', role: 'assistant', phase: 'final_answer', content: [{ type: 'output_text', text }] } });
-const user = text => ({ type: 'event_msg', payload: { type: 'user_message', message: text } });
-test('Codex compaction/scope resets context and event assistant duplicates are ignored', async t => {
-  const list = [meta, assistant('Before reset'), { type: 'compacted', payload: {} }, assistant('After reset'), { type: 'event_msg', payload: { type: 'agent_message', message: 'Event duplicate' } }, user('Target'), assistant('Future')];
-  const f = await fixture(t, 'codex', list, 6), row = (await enrich([f.row], f.manifest, f.options)).rows[0];
-  assert.deepEqual(row.observed_context.map(c => c.text), ['After reset']); assert.ok(row.context_gaps.includes('compaction_or_rollback'));
-  const scoped = await fixture(t, 'codex', [meta, assistant('Original cwd'), { type: 'turn_context', payload: { cwd: 'C:/Other' } }, assistant('Foreign'), { type: 'turn_context', payload: { cwd } }, user('Return')], 6);
-  assert.equal((await enrich([scoped.row], scoped.manifest, scoped.options)).rows[0].status, 'missing');
-});
-test('bounded context marks truncation, missing ancestry never uses nearest neighbor', async t => {
-  const long = await fixture(t, 'claude', [c('a', null, 'assistant', 'Long visible text '.repeat(500)), c('u', 'a', 'user', 'Target')], 2);
-  const row = (await enrich([long.row], long.manifest, long.options)).rows[0]; assert.equal(row.status, 'partial'); assert.equal(row.observed_context[0].text.length, 3000); assert.equal(row.observed_context[0].truncated, true);
-  const missing = await fixture(t, 'claude', [c('a', null, 'assistant', 'Not ancestor'), c('u', 'unknown', 'user', 'Target')], 2);
-  const m = (await enrich([missing.row], missing.manifest, missing.options)).rows[0]; assert.equal(m.status, 'missing'); assert.ok(m.context_gaps.includes('missing_ancestor'));
-});
-test('static HTML escapes raw content and has no scripts or external resources', () => {
-  const html = renderHtml([{ id: 'case', status: 'restored', message: '<script>fetch("https://evil")</script>', observed_context: [{ role: 'assistant', source_line: 1, text: '<img src=x onerror=alert(1)>' }] }]);
-  assert.equal(html.includes('<script>'), false); assert.equal(html.includes('<img'), false); assert.ok(html.includes('&lt;script&gt;')); assert.ok(html.includes("default-src 'none'"));
-});
-test('metadata-only duplicate UUID tolerated; conflicting text identity rejected', async t => {
-  const first = c('a', null, 'assistant', 'Actual context');
-  const attachment = { uuid: 'attachment', type: 'attachment', parentUuid: 'a', sessionId: 's1', cwd, arbitraryMetadata: 1 };
-  const f = await fixture(t, 'claude', [first, attachment, { ...attachment, arbitraryMetadata: 2 }, c('u', 'a', 'user', 'Target')], 4);
-  assert.equal((await enrich([f.row], f.manifest, f.options)).rows[0].status, 'restored');
-  const bad = await fixture(t, 'claude', [first, c('a', null, 'assistant', 'Conflicting context'), c('u', 'a', 'user', 'Target')], 3);
-  assert.equal((await enrich([bad.row], bad.manifest, bad.options)).rows[0].status, 'rejected');
-});
-test('ancestry walk and Codex history window are bounded with explicit gaps', async t => {
-  const list = [c('a', null, 'assistant', 'Ancient context')];
-  for (let i = 0; i < 6; i++) list.push({ uuid: `tool${i}`, parentUuid: i ? `tool${i - 1}` : 'a', sessionId: 's1', cwd, type: 'user', message: { role: 'user', content: [{ type: 'tool_result', content: 'omitted' }] } });
-  list.push(c('u', 'tool5', 'user', 'Target'));
-  const f = await fixture(t, 'claude', list, list.length); f.options.limits = { maxAncestorSteps: 3 };
-  const row = (await enrich([f.row], f.manifest, f.options)).rows[0]; assert.ok(row.context_gaps.includes('ancestry_budget')); assert.equal(row.status, 'missing');
-  const history = [meta, ...Array.from({ length: 25 }, (_, i) => assistant(`Turn ${i}`)), user('Target')];
-  const g = await fixture(t, 'codex', history, history.length), restored = (await enrich([g.row], g.manifest, g.options)).rows[0];
-  assert.equal(restored.observed_context.length, 6); assert.ok(restored.context_gaps.includes('history_window')); assert.ok(renderHtml([restored]).includes('history_window'));
-});

package/core/dev/jev-eval/test/corpus.test.mjs

@@ -1,102 +0,0 @@
-import test from 'node:test';
-import assert from 'node:assert/strict';
-import { mkdtemp, mkdir, writeFile, readdir, unlink, rmdir } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { credentialBearing, generatedMessage, automationSeed, redactHomePaths, userText, withinRoots, extractFile, sampleCandidates, records, discover, inventory, main } from '../corpus.mjs';
-
-async function temp(t) {
-  const dir = await mkdtemp(join(tmpdir(), 'jev-corpus-test-'));
-  async function cleanup(path) { for (const entry of await readdir(path, { withFileTypes: true })) { const child = join(path, entry.name); if (entry.isDirectory()) await cleanup(child); else await unlink(child); } await rmdir(path); }
-  t.after(() => cleanup(dir)); return dir;
-}
-const roots = ['C:/Users/test/Documents/GitHub/KenazAI'];
-const claude = (text, extra = {}) => ({ type: 'user', sessionId: 's1', cwd: roots[0], message: { role: 'user', content: text }, ...extra });
-const meta = (extra = {}) => ({ type: 'session_meta', payload: { id: 's1', cwd: roots[0], source: 'cli', ...extra } });
-const event = text => ({ type: 'event_msg', payload: { type: 'user_message', message: text } });
-const response = text => ({ type: 'response_item', payload: { type: 'message', role: 'user', content: [{ type: 'input_text', text }] } });
-async function extract(t, items, kind = 'claude', limits = {}) {
-  const dir = await temp(t), path = join(dir, 's.jsonl'); await writeFile(path, items.map(r => JSON.stringify(r)).join('\n'));
-  const stats = {}, candidates = await extractFile({ path, kind, relative_path: 's.jsonl' }, roots, stats, limits); return { candidates, stats };
-}
-test('credential detection quarantines whole message and home paths are removed', () => {
-  for (const text of ['apikey_abcdefgh123456', 'sk-abcdefgh-123456', 'Authorization: Bearer abcdefgh123', '$env:TYPESAFE_API_KEY="abc123456"', 'export API_KEY=abc12345', 'password: hunter22', 'https://user:secret@site.test', '-----BEGIN PRIVATE KEY-----', 'github_pat_abcde12345678']) assert.equal(credentialBearing(text), true);
-  assert.equal(credentialBearing('Please configure the API key locally.'), false);
-  const clean = redactHomePaths('C:\\Users\\someone\\repo /home/someone/repo /Users/someone/file'); assert.equal(clean.includes('someone'), false);
-});
-test('generated and non-user records are excluded without fabricated labels', async t => {
-  for (const text of ['# AGENTS.md instructions for repo', '<environment_context>cwd</environment_context>', '[CANARY:v2]', '<system-reminder>generated</system-reminder>', '<subagent_notification>done</subagent_notification>']) assert.equal(generatedMessage(text), true);
-  assert.equal(userText(claude([{ type: 'tool_result', content: 'not user' }]), 'claude'), null);
-  const { candidates } = await extract(t, [claude('Real request'), claude('internal', { isMeta: true }), claude('render-only', { isVisibleInTranscriptOnly: true }), { type: 'assistant', message: { role: 'assistant', content: 'not user' } }, claude('<environment_context>private</environment_context>'), claude('Another request')]);
-  assert.equal(candidates.length, 2);
-  for (const { row } of candidates) { for (const field of ['labels', 'goal', 'assumptions', 'grade']) assert.equal(Object.hasOwn(row, field), false); assert.equal(row.review_status, 'pending'); assert.equal(row.incomplete_context_for_correction, true); }
-});
-test('secret message cannot survive as later preceding context', async t => {
-  const { candidates, stats } = await extract(t, [claude('First'), claude('Use apikey_abcdefgh1234 now'), claude('Next request')]);
-  assert.equal(candidates.length, 2); assert.equal(stats.credential_message, 1); assert.equal(candidates[1].row.context_missing, true);
-  assert.equal(JSON.stringify(candidates).includes('apikey_'), false);
-});
-test('Codex exact cwd boundaries and subagent metadata fail closed', async t => {
-  assert.equal(withinRoots(roots[0] + '/dashboard', roots), true); assert.equal(withinRoots(roots[0] + 'Else', roots), false); assert.equal(withinRoots(roots[0] + '/../Other', roots), false);
-  for (const metadata of [meta({ cwd: 'C:/Other' }), meta({ source: { subagent: { parent_thread_id: 'parent' } } })]) assert.equal((await extract(t, [metadata, event('Request')], 'codex')).candidates.length, 0);
-  assert.equal((await extract(t, [event('No metadata')], 'codex')).candidates.length, 0);
-  const proper = await extract(t, [meta(), response('Replayed old text'), event('Actual text'), response('Actual text')], 'codex');
-  assert.equal(proper.candidates.length, 1); assert.equal(proper.candidates[0].row.message, 'Actual text'); assert.equal(proper.candidates[0].row.context_missing, true);
-  const fallback = await extract(t, [meta(), response('Fallback text')], 'codex'); assert.equal(fallback.candidates.length, 1); assert.equal(fallback.stats.response_fallback_candidates, 1);
-});
-test('whole Claude sidechain file rejected even after first candidate', async t => {
-  const { candidates } = await extract(t, [claude('Earlier request'), claude('Subagent request', { isSidechain: true })]); assert.equal(candidates.length, 0);
-});
-test('automation seeds exclude entire session while genuine TASK references remain', async t => {
-  const seeds = ['- ## Automated Dispatch Context\nThis prompt was machine-assembled by the Kenaz orchestration system.', 'This prompt was machine-assembled by the Kenaz orchestration system.', '- Extract key technical insights from this conversation chunk. Output in English only. Be concise, 1-3 sentences.', '[HEADLESS MODE] 你已被 Formation Lead 指派任務', 'You have access to MCP tools from kenaz-dashboard. Call the synapse_status tool now.'];
-  for (const seed of seeds) {
-    assert.equal(automationSeed(seed), true);
-    const result = await extract(t, [claude('Prior automated followup'), claude(seed), claude('Continue automatically')]);
-    assert.equal(result.candidates.length, 0); assert.equal(result.stats.automation_seed, 1);
-  }
-  assert.equal(automationSeed('Please investigate TASK_35 and explain the Automated Dispatch Context header.'), false);
-  const genuine = await extract(t, [claude('Please investigate TASK_35'), claude('[Request interrupted by user for tool use]'), claude('<turn_aborted>The user interrupted the previous turn</turn_aborted>'), claude('Continue my investigation')]);
-  assert.equal(genuine.candidates.length, 2);
-  const smoke = await extract(t, [meta(), event(seeds.at(-1)), event('Continue')], 'codex'); assert.equal(smoke.candidates.length, 0);
-});
-test('automation discovered in one source excludes same session in another source', async t => {
-  const dir = await temp(t), claudeRoot = join(dir, 'claude'), codexRoot = join(dir, 'codex'), project = join(claudeRoot, 'C-KenazAI');
-  await mkdir(project, { recursive: true }); await mkdir(codexRoot);
-  await writeFile(join(project, 'a.jsonl'), JSON.stringify(claude('Earlier inherited automated turn')));
-  await writeFile(join(project, 'b.jsonl'), JSON.stringify(claude('[HEADLESS MODE] 你已被 Formation Lead 指派任務')));
-  const result = await inventory({ claudeRoot, codexRoot, roots }); assert.equal(result.rows.length, 0); assert.equal(result.summary.exclusions.cross_file_automation_candidates, 1);
-});
-test('Claude foreign/missing cwd excluded; changing session identity rejects entire file', async t => {
-  const foreign = await extract(t, [claude('Allowed first'), claude('Foreign', { cwd: 'C:/Other' }), claude('Allowed return'), claude('Missing', { cwd: undefined })]);
-  assert.equal(foreign.candidates.length, 2); assert.equal(foreign.candidates[1].row.context_missing, true);
-  const mixed = await extract(t, [claude('First'), claude('Other session', { sessionId: 's2' })]); assert.equal(mixed.candidates.length, 0); assert.equal(mixed.stats.session_identity_change, 1);
-  const reentry = await extract(t, [meta(), event('First'), { type: 'turn_context', payload: { cwd: 'C:/Other' } }, { type: 'turn_context', payload: { cwd: roots[0] } }, event('Return')], 'codex');
-  assert.equal(reentry.candidates[1].row.context_missing, true);
-});
-test('dedup and balanced sampling deterministic; generic duplicates do not merge families', async t => {
-  const { candidates: a } = await extract(t, [claude('Continue'), claude('Unique A1'), claude('Unique A2')]);
-  const { candidates: b } = await extract(t, [claude(' continue ', { sessionId: 's2' }), claude('Unique B1', { sessionId: 's2' }), claude('Unique B2', { sessionId: 's2' })]);
-  const input = [...a, ...b], sample = sampleCandidates(input, 4);
-  assert.equal(sample.unique_count, 5); assert.equal(sample.duplicate_count, 1); assert.equal(sample.group_count, 2);
-  assert.deepEqual(sampleCandidates([...input].reverse(), 4), sample); assert.equal(new Set(sample.selected.map(c => c.row.session_id)).size, 2);
-  const lengthy = 'Copied historical context '.repeat(10);
-  const c = await extract(t, [claude(lengthy)]), d = await extract(t, [claude(lengthy, { sessionId: 's2' })]);
-  assert.equal(sampleCandidates([...c.candidates, ...d.candidates]).group_count, 1);
-});
-test('stream bounds discard oversized lines then recover next JSON record', async t => {
-  const dir = await temp(t), path = join(dir, 'big.jsonl'); await writeFile(path, 'x'.repeat(200000) + '\n' + JSON.stringify({ ok: true }) + '\n');
-  const stats = {}, output = []; for await (const item of records(path, stats, { maxLineBytes: 100 })) output.push(item);
-  assert.equal(stats.oversize_line, 1); assert.deepEqual(output, [{ value: { ok: true }, line: 2 }]);
-});
-test('discovery excludes non-Kenaz Claude and nested subagents; summary only dry output', async t => {
-  const dir = await temp(t), claudeRoot = join(dir, 'claude'), codexRoot = join(dir, 'codex');
-  for (const path of [join(claudeRoot, 'C-KenazAI'), join(claudeRoot, 'C-Other'), join(claudeRoot, 'C-KenazAI', 'subagents'), codexRoot]) await mkdir(path, { recursive: true });
-  await writeFile(join(claudeRoot, 'C-KenazAI', 's.jsonl'), JSON.stringify(claude('Kept real user text')));
-  await writeFile(join(claudeRoot, 'C-KenazAI', 'agent-1.jsonl'), JSON.stringify(claude('Agent')));
-  await writeFile(join(claudeRoot, 'C-Other', 's.jsonl'), JSON.stringify(claude('Other')));
-  await writeFile(join(claudeRoot, 'C-KenazAI', 'subagents', 'a.jsonl'), JSON.stringify(claude('Nested')));
-  assert.equal((await discover({ claudeRoot, codexRoot })).length, 1);
-  const result = await inventory({ claudeRoot, codexRoot, roots, target: 500 }); assert.equal(result.summary.shortfall, 499);
-  const report = await main(['--claude-root', claudeRoot, '--codex-root', codexRoot]); assert.equal(report.written, false); assert.equal(JSON.stringify(report).includes('Kept real'), false);
-  await assert.rejects(main(['--target', 'NaN']), /invalid_target/);
-});

package/core/dev/jev-eval/test/eval.test.mjs

@@ -1,146 +0,0 @@
-import test from 'node:test';
-import assert from 'node:assert/strict';
-import { createServer } from 'node:http';
-import { once } from 'node:events';
-import { spawnSync } from 'node:child_process';
-import { template, main } from '../cli.mjs';
-import { readJsonl, validateDataset, validatePrediction, stateHash, inputState } from '../schema.mjs';
-import { MODEL, RUBRIC, prediction, rules, replayProvider, liveProvider, buildRequest, parseAnswers } from '../providers.mjs';
-import { report } from '../metrics.mjs';
-
-const fixture = new URL('../fixtures/synthetic.jsonl', import.meta.url);
-const rows = validateDataset(await readJsonl(fixture));
-const clone = x => structuredClone(x);
-const row = () => template();
-function correctionResponse(noul = 0.9) { return { model: MODEL, answers: { correction: { type: 'noul', noul } }, usage: { input_tokens: 30, output_tokens: 2 } }; }
-async function server(t, handler) {
-  const http = createServer(handler); http.listen(0, '127.0.0.1'); await once(http, 'listening');
-  t.after(() => { http.closeAllConnections(); http.close(); });
-  return `http://127.0.0.1:${http.address().port}/v1/systemone`;
-}
-const live = (endpoint, extra = {}) => liveProvider({ apiKey: 'test-only-secret', endpoint, allowLoopbackForTests: true, timeoutMs: 2000, ...extra });
-
-test('varied synthetic fixtures are strict, immutable and never promotion eligible', async () => {
-  const before = JSON.stringify(rows);
-  assert.equal(rows.length, 11);
-  const result = report(rows, rows.map(rules), { provider: 'rules' });
-  assert.equal(result.promotion_eligible, false);
-  assert.equal(result.provenance.annotation.agent_proposed, rows.length);
-  assert.equal(result.metrics.relevance.zero_relevance_queries_excluded, 1);
-  assert.equal(JSON.stringify(rows), before);
-  assert.equal(JSON.stringify(result).includes(rows[0].state.message), false);
-  assert.equal((await main(['validate', '--dataset', fixture.pathname.replace(/^\/([A-Z]:)/i, '$1')])).valid, true);
-});
-test('schema rejects unknown fields, invalid labels, duplicates and dishonest review metadata', () => {
-  for (const mutate of [r => r.extra = true, r => r.labels.correction = 1, r => r.provenance.annotation = 'human_verified', r => r.state.message = '', r => r.event_seq = -1]) {
-    const r = row(); mutate(r); assert.throws(() => validateDataset([r]));
-  }
-  const valid = row(); valid.provenance = { source: 'real', annotation: 'human_verified', reviewer: 'reviewer-a', reviewed_at: '2026-09-20T00:00:00Z' };
-  assert.doesNotThrow(() => validateDataset([valid]));
-  assert.throws(() => validateDataset([row(), row()]));
-});
-test('group, session and exact-content split leakage rejected', () => {
-  for (const kind of ['group', 'session', 'content']) {
-    const a = row(), b = row(); b.id = 'other'; b.event_id = 'other'; b.split = 'held_out';
-    if (kind !== 'session') b.session_id = 'other';
-    if (kind !== 'group') b.group_id = 'other';
-    if (kind !== 'content') b.state.message = 'Different message';
-    assert.throws(() => validateDataset([a, b]), /leakage/);
-  }
-});
-test('model input and hash exclude ground truth and provenance', () => {
-  const r = clone(rows.find(r => r.purpose === 'relevance')), hash = stateHash(r);
-  r.candidates[0].grade = 0; r.provenance.reviewer = 'changed'; r.split = 'held_out';
-  assert.equal(stateHash(r), hash);
-  const input = JSON.stringify(inputState(r));
-  for (const forbidden of ['grade', 'provenance', 'reviewer', 'split', 'labels']) assert.equal(input.includes(`"${forbidden}"`), false);
-  r.state.message += ' revised'; assert.notEqual(stateHash(r), hash);
-});
-test('replay rejects wrong project/session/task, stale revision/hash, unknown IDs and aliases', () => {
-  const r = row(), p = prediction(r, 'ok', 0.8);
-  for (const field of ['project_id', 'session_id', 'task_id', 'event_id', 'task_revision', 'state_hash', 'model', 'question_set_version']) {
-    const bad = clone(p); bad[field] = 'wrong'; assert.throws(() => replayProvider([r], [bad]));
-  }
-  assert.throws(() => replayProvider([r], []));
-  assert.throws(() => replayProvider([r], [p, p]));
-  assert.equal(replayProvider([r], [p])(r), p);
-});
-test('correction metrics include failed positives, preserve unanswered Brier denominator', () => {
-  const data = [true, true, false, false].map((label, i) => ({ ...row(), id: `m${i}`, labels: { correction: label } }));
-  const preds = [prediction(data[0], 'ok', 0.8), prediction(data[1], 'timeout'), prediction(data[2], 'ok', 0.6), prediction(data[3], 'ok', 0.1)];
-  const result = report(data, preds, { provider: 'replay' }), m = result.metrics.correction;
-  assert.deepEqual(m.confusion, { tp: 1, fp: 1, tn: 1, fn: 0 });
-  assert.equal(m.coverage, 0.75); assert.equal(m.recall_including_abstentions, 0.5); assert.equal(m.recall_answered_only, 1);
-  assert.equal(m.precision, 0.5); assert.equal(m.accuracy_including_abstentions, 0.5);
-  assert.ok(Math.abs(m.brier_answered_only - 0.41 / 3) < 1e-12); assert.equal(m.brier_count, 3);
-  const empty = report([data[1]], [prediction(data[1], 'abstain')], { provider: 'replay' }).metrics.correction;
-  assert.equal(empty.precision, null); assert.equal(empty.brier_answered_only, null); assert.equal(empty.recall_including_abstentions, 0);
-});
-test('ranking uses full ideal, tracks partial scores and handles zero relevance', () => {
-  const r = clone(rows.find(r => r.purpose === 'relevance'));
-  const partial = prediction(r, 'ok', { [r.candidates[1].id]: 3 });
-  const m = report([r], [partial], { provider: 'replay', k: 1 }).metrics.relevance;
-  assert.equal(m.ndcg_at_k, 1 / 7); assert.equal(m.recall_at_k, 0.5); assert.equal(m.query_coverage, 0); assert.equal(m.candidate_coverage, 0.25); assert.equal(m.ndcg_complete_queries_only, null);
-  const failed = report([r], [prediction(r, 'timeout')], { provider: 'replay' }).metrics.relevance;
-  assert.equal(failed.ndcg_at_k, 0); assert.equal(failed.recall_at_k, 0);
-  const zero = rows.at(-1), z = report([zero], [rules(zero)], { provider: 'rules' }).metrics.relevance;
-  assert.equal(z.ndcg_at_k, null); assert.equal(z.zero_relevance_queries_excluded, 1);
-  partial.scores[0].id = 'wrong'; assert.throws(() => validatePrediction(r, partial));
-});
-test('missing/blank key and unsafe URL rejected before any network', () => {
-  let calls = 0; const fetchImpl = () => { calls++; throw new Error('unexpected'); };
-  for (const apiKey of ['', ' ', '\r\n']) assert.throws(() => liveProvider({ apiKey, fetchImpl }), /api_key/);
-  for (const endpoint of ['http://evil.example', 'https://api.typesafe.ai.evil.example/v1/systemone', 'https://api.typesafe.ai/v1/systemone?key=x', 'http://localhost:1234']) assert.throws(() => liveProvider({ apiKey: 'fake', endpoint, fetchImpl }), /unsafe_endpoint/);
-  assert.equal(calls, 0);
-  const child = spawnSync(process.execPath, ['dev/jev-eval/cli.mjs', 'evaluate', '--dataset', 'dev/jev-eval/fixtures/synthetic.jsonl', '--provider', 'live'], { env: { ...process.env, TYPESAFE_API_KEY: '' }, encoding: 'utf8' });
-  assert.equal(child.status, 1); assert.deepEqual(JSON.parse(child.stderr), { error: 'missing_or_invalid_api_key' });
-});
-test('successful HTTP batches by row and tracks usage without gold leakage', async t => {
-  const endpoint = await server(t, async (req, res) => {
-    const parts = []; for await (const chunk of req) parts.push(chunk);
-    const request = JSON.parse(Buffer.concat(parts));
-    assert.equal(request.model, MODEL); assert.equal(request.state.labels, undefined); assert.equal(req.headers.authorization, 'Bearer test-only-secret');
-    res.setHeader('Content-Type', 'application/json'); res.end(JSON.stringify(correctionResponse()));
-  });
-  const p = await live(endpoint)(row()); assert.equal(p.status, 'ok'); assert.equal(p.correction_probability, 0.9); assert.equal(p.usage.input_tokens, 30); validatePrediction(row(), p);
-});
-test('HTTP error statuses abstain without leaking raw provider errors', async t => {
-  for (const status of [401, 422, 429, 529]) {
-    const endpoint = await server(t, (req, res) => { res.writeHead(status); res.end('secret-body test-only-secret'); });
-    const p = await live(endpoint)(row()); assert.equal(p.status, 'unavailable'); assert.equal(p.http_status, status); assert.equal(p.correction_probability, null); assert.equal(JSON.stringify(p).includes('secret'), false);
-  }
-});
-test('redirect rejected without contacting target', async t => {
-  let targetCalls = 0;
-  const target = await server(t, (req, res) => { targetCalls++; res.end('{}'); });
-  const endpoint = await server(t, (req, res) => { res.writeHead(302, { Location: target }); res.end(); });
-  const p = await live(endpoint)(row()); assert.equal(p.status, 'unavailable'); assert.equal(targetCalls, 0);
-});
-test('timeout remains active across response body and also covers stalled transport', async t => {
-  const endpoint = await server(t, (req, res) => { res.writeHead(200, { 'Content-Type': 'application/json' }); res.flushHeaders(); res.write('{'); });
-  const start = Date.now(), p = await live(endpoint, { timeoutMs: 80 })(row());
-  assert.equal(p.status, 'timeout'); assert.ok(Date.now() - start < 3000);
-  const stalled = await liveProvider({ apiKey: 'fake', timeoutMs: 20, fetchImpl: () => new Promise(() => {}) })(row());
-  assert.equal(stalled.status, 'timeout');
-});
-test('invalid JSON, oversized response and malformed answers fail closed', async t => {
-  const invalids = ['{', 'x'.repeat(65537), JSON.stringify(correctionResponse(1.1)), JSON.stringify({ ...correctionResponse(), answers: { wrong: { type: 'noul', noul: 0.9 } } }), JSON.stringify({ ...correctionResponse(), usage: null })];
-  for (const body of invalids) {
-    const endpoint = await server(t, (req, res) => res.end(body));
-    const p = await live(endpoint)(row()); assert.equal(p.status, 'invalid'); assert.equal(p.correction_probability, null);
-  }
-});
-test('score answers enforce distributions, legends, bounds, confidence and exact IDs', () => {
-  const r = rows.find(r => r.purpose === 'relevance'), request = buildRequest(r);
-  const answer = { type: 'score', score: 2, legend: Object.fromEntries(RUBRIC.map((v, i) => [i, v])), probabilities: { 0: 0, 1: 0, 2: 1, 3: 0 }, confidence: 1 };
-  const body = { model: MODEL, answers: Object.fromEntries(Object.keys(request.questions).map(k => [k, clone(answer)])), usage: { input_tokens: 100, output_tokens: 8 } };
-  assert.equal(Object.keys(parseAnswers(r, body, request)).length, r.candidates.length);
-  for (const mutate of [a => a.score = NaN, a => a.probabilities[2] = 0.3, a => a.legend[0] = 'wrong', a => a.confidence = 2, a => a.probabilities[4] = 0, a => a.type = 'noul']) {
-    const bad = clone(body); mutate(bad.answers.candidate_0); assert.throws(() => parseAnswers(r, bad, request));
-  }
-});
-test('CLI defaults offline, rejects ambiguous options, and preflight prints no key', async () => {
-  assert.equal((await main(['preflight'])).network_attempted, false);
-  assert.equal((await main(['template'])).schema_version, 1);
-  for (const args of [['evaluate', '--endpoint', 'http://evil'], ['evaluate', '--dataset', 'a', '--dataset', 'b'], ['preflight', '--provider', 'live']]) await assert.rejects(main(args));
-});

package/core/dev/jev-eval/test/muninn-baseline.test.mjs

@@ -1,71 +0,0 @@
-import test from 'node:test';
-import assert from 'node:assert/strict';
-import { readFile } from 'node:fs/promises';
-import { createHash } from 'node:crypto';
-import { resolve } from 'node:path';
-import { readJsonl } from '../schema.mjs';
-import { runBaseline, strictAnswers, safeInvocation, nativeTransport, main } from '../muninn-baseline.mjs';
-
-const source = await readFile(new URL('../../../.formation/tools/muninn.js', import.meta.url), 'utf8');
-const rows = await readJsonl(new URL('../fixtures/synthetic.jsonl', import.meta.url));
-const corrections = rows.filter(r => r.purpose === 'correction');
-const answers = () => corrections.map((r, idx) => ({ idx, translated: `Synthetic translation ${idx}`, labels: r.labels.correction ? ['correction'] : [] }));
-const response = items => ({ status: 0, stdout: JSON.stringify({ result: JSON.stringify(items), modelUsage: { 'claude-haiku-test': { inputTokens: 20 } } }) });
-
-test('offline default captures actual source prompt/parser without model quality claim', async () => {
-  const result = await main([]);
-  assert.equal(result.live, false); assert.equal(result.promotion_eligible, false); assert.equal(result.status, 'ok');
-  assert.equal(result.batch_count, 8); assert.equal(result.translation_present, 8); assert.equal(result.correction_confusion, null);
-  assert.equal(result.source_sha256, createHash('sha256').update(source).digest('hex'));
-  assert.equal(result.source_requested_permission_bypass, true); assert.equal(result.actual_permission_bypass, false);
-});
-test('isolated source passes exact indexed messages, single transport and stable confusion', () => {
-  let calls = 0; const before = JSON.stringify(rows);
-  const result = runBaseline(rows, { source, live: true, transport: input => {
-    calls++; assert.ok(input.startsWith('For each numbered message, output a JSON array'));
-    assert.ok(input.includes('length_too_short')); assert.ok(!input.includes('Development startup also fails'));
-    corrections.forEach((r, idx) => assert.ok(input.includes(`[${idx}] ${r.state.message.replace(/\r?\n/g, ' ')}`)));
-    return response(answers());
-  } });
-  assert.equal(calls, 1); assert.equal(result.status, 'ok'); assert.equal(JSON.stringify(rows), before);
-  assert.deepEqual(result.correction_confusion, { tp: 4, fp: 0, tn: 4, fn: 0, precision: 1, recall: 1 });
-  assert.deepEqual(result.observed_model_metadata, ['claude-haiku-test']);
-  assert.equal(JSON.stringify(result).includes(corrections[0].state.message), false);
-});
-test('strict raw checking rejects identities and labels before permissive source filters them', () => {
-  const changes = [a => a.pop(), a => a[1].idx = a[0].idx, a => a.reverse(), a => a[0].idx = 0.5, a => a[0].translated = ' ', a => a[0].labels = ['invented'], a => a[0].labels = ['correction', 'correction'], a => a[0].extra = true];
-  for (const change of changes) {
-    const items = answers(); change(items); assert.throws(() => strictAnswers(JSON.stringify(items), corrections.length));
-    const result = runBaseline(rows, { source, live: true, transport: () => response(items) });
-    assert.notEqual(result.status, 'ok'); assert.equal(result.correction_confusion, null); assert.equal(result.translation_present, 0);
-  }
-});
-test('failures timeouts malformed envelopes and oversize output remain sanitized failures', () => {
-  for (const transport of [() => ({ status: 1, stdout: 'private-error' }), () => ({ status: null, error: { code: 'ETIMEDOUT' } }), () => ({ status: 0, stdout: '{' }), () => ({ status: 0, stdout: 'x'.repeat(1048577) }), () => { throw 'private-error'; }, () => ({ status: 0, stdout: JSON.stringify({ is_error: true, result: 'private-error' }) })]) {
-    const result = runBaseline(rows, { source, live: true, transport });
-    assert.notEqual(result.status, 'ok'); assert.equal(result.correction_confusion, null); assert.equal(JSON.stringify(result).includes('private-error'), false);
-  }
-});
-test('real data rejected before transport including irrelevant purpose rows', () => {
-  let calls = 0; const real = structuredClone(rows); real.at(-1).provenance.source = 'real';
-  assert.throws(() => runBaseline(real, { source, live: true, transport: () => { calls++; } }), /synthetic_only/); assert.equal(calls, 0);
-});
-test('VM denies filesystem/network and unrelated module calls; empty environment', () => {
-  for (const expression of ["require('fs').readFileSync('x')", "require('http').request('x')", "require('os').homedir()", "require('net')", "require('./log-reader').findTaskLog()"])
-    assert.throws(() => runBaseline(rows, { source: source + '\n' + expression }), /blocked_vm/);
-  const checked = source + '\nif (Object.keys(process.env).length || require.main === module) throw new Error("unexpected_environment");';
-  assert.equal(runBaseline(rows, { source: checked }).status, 'ok');
-});
-test('safe subprocess uses explicit executable argv no shell and bounded output/time', () => {
-  const exe = resolve('test-claude.exe'), invocation = safeInvocation(exe);
-  for (const flag of ['--safe-mode', '--tools', '--disable-slash-commands', '--no-session-persistence']) assert.ok(invocation.args.includes(flag));
-  assert.equal(invocation.args[invocation.args.indexOf('--tools') + 1], '');
-  assert.equal(invocation.args.includes('--dangerously-skip-permissions'), false);
-  let calls = 0;
-  nativeTransport(exe, '/isolated', (command, args, options) => {
-    calls++; assert.equal(command, exe); assert.deepEqual(args, invocation.args); assert.equal(options.shell, false);
-    assert.equal(options.timeout, 30000); assert.equal(options.maxBuffer, 1048576); assert.equal(options.windowsHide, true);
-    assert.equal(Object.hasOwn(options.env, 'TYPESAFE_API_KEY'), false); return { status: 0, stdout: '{}' };
-  })('synthetic prompt'); assert.equal(calls, 1);
-  for (const bad of ['claude', resolve('claude.cmd'), resolve('claude.bat')]) assert.throws(() => safeInvocation(bad), /invalid_claude_cli/);
-});