@arjunkhera/atlas 0.2.3 → 0.3.0

package/door/lib/onboarding.mjs

@@ -1,130 +0,0 @@
-// `atlas test onboarding`: the answer key, and the fixed script that compares.
-//
-// Milestone M2, slice L2 stage one, done line 5: "The fresh-agent test scores
-// 6 of 6 under the isolation rule."
-//
-// Why this exists. The M2 criterion said a fresh agent "completes the six-step
-// task list from the files alone, scored 6 of 6 by the answer key", and no
-// slice built the key or the scorer. Six of six was therefore a self-grade
-// (panel finding P4). This is the key, and this is the scorer.
-//
-// The scorer runs OUTSIDE the isolated session. The isolated session has the
-// Atlas package off, so it cannot score itself even if it wanted to. It writes
-// its six answers to a file; this reads that file.
-import { readFileSync, existsSync } from 'node:fs';
-import { resolve } from 'node:path';
-import { parseYaml, secretPatterns } from '../../shape/check.mjs';
-
-
-// The isolation rule, in the words of closure R3. The scorer prints it, so the
-// person who runs the test sets the session up the same way every time.
-export const ISOLATION = Object.freeze([
-  'no memory directory',
-  'no global instructions',
-  'the Engram MCP disconnected',
-  'the Atlas package off',
-  'the repo at HEAD',
-]);
-
-// The answer key is read from a path. It never ships in the package and never
-// sits in the repository under test, because the agent under test must not be
-// able to read it (amendment A1.9, the L2 stage one clause).
-export function loadKey(path) {
-  if (!path || path === true) throw new Error('give the answer key with --key <path>. Keys stay outside the package and outside the repo under test.');
-  const full = resolve(String(path));
-  if (!existsSync(full)) throw new Error(`there is no answer key at ${full}`);
-  const key = parseYaml(readFileSync(full, 'utf8'));
-  if (!Array.isArray(key?.tasks) || !key.tasks.length) throw new Error(`${full} carries no tasks`);
-  return { ...key, path: full };
-}
-
-// The task list as the isolated session sees it: the questions, and nothing
-// else. No sources, no expected strings.
-export function questions(key) {
-  return key.tasks.map((task) => ({ id: task.id, number: task.number, ask: task.ask }));
-}
-
-const normalise = (text) => String(text ?? '').toLowerCase().replace(/\s+/g, ' ').trim();
-
-// A phrase must appear as a whole word, not inside another one. Plain substring
-// matching passed "the dependencies file is nonexistent" for the phrase "none",
-// because "none" sits inside "nonexistent" (acceptance review of batch 16,
-// finding S2-5).
-//
-// What this still cannot do: it compares text, not meaning. "do not run
-// scripts/stack.mjs" carries the right words in the wrong order and passes.
-// The frozen text asks for "a fixed script that compares", so that limit is the
-// design, and it is written down here rather than left to be discovered.
-const says = (said, phrase) => {
-  const wanted = normalise(phrase);
-  if (!wanted) return false;
-  // The boundary is a letter, a digit or an underscore. Dots, slashes, commas
-  // and hyphens are boundaries, so "atlas/deploy.yaml." at the end of a
-  // sentence still matches, and "none" inside "nonexistent" still does not.
-  const edge = '[^a-z0-9_]';
-  return new RegExp(`(?:^|${edge})${wanted.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}(?:$|${edge})`).test(said);
-};
-
-// One answer against one task.
-export function scoreTask(task, answer, patterns) {
-  const said = normalise(answer);
-  if (!said) return { id: task.id, number: task.number, pass: false, source: task.source, missing: task.must_say ?? [], why: 'no answer was given' };
-
-  // An answer that carries a credential value fails, whatever else it says.
-  // The file it read holds names only, so a value can only have come from
-  // somewhere it should not have been read from.
-  for (const pattern of patterns) {
-    if (pattern.test(String(answer ?? ''))) {
-      return { id: task.id, number: task.number, pass: false, source: task.source, missing: [], why: 'the answer holds something that looks like a credential value; these files carry names only' };
-    }
-  }
-
-  // The answer must cite a file. The test is whether the seven files answer on
-  // their own, so an answer that names no file proves nothing about them
-  // (acceptance review of batch 16, finding S2-5).
-  const cites = (task.cites ?? [task.source]).some((path) => says(said, path));
-  if (!cites) {
-    return { id: task.id, number: task.number, pass: false, source: task.source, missing: [task.source], why: `the answer names no source file; it should name ${task.source}` };
-  }
-
-  const anyOf = task.accept_any_of ?? [];
-  if (anyOf.length && anyOf.some((phrase) => says(said, phrase))) {
-    return { id: task.id, number: task.number, pass: true, source: task.source, missing: [], why: `the answer says what ${task.source} says, and names it` };
-  }
-  const missing = (task.must_say ?? []).filter((phrase) => !says(said, phrase));
-  return {
-    id: task.id,
-    number: task.number,
-    pass: !missing.length,
-    source: task.source,
-    missing,
-    why: missing.length ? `the answer does not name ${missing.join(', ')}; the file that says it is ${task.source}` : `the answer names what ${task.source} says, and names the file`,
-  };
-}
-
-// The whole run. `answers` is a map of task id to the answer text the isolated
-// session wrote.
-export function score(key, answers, { patterns = secretPatterns() } = {}) {
-  const rows = key.tasks.map((task) => scoreTask(task, answers?.[task.id] ?? answers?.[String(task.number)] ?? null, patterns));
-  const passed = rows.filter((row) => row.pass).length;
-  return {
-    repo: key.repo,
-    key: key.path,
-    isolation: ISOLATION,
-    rows,
-    passed,
-    of: key.tasks.length,
-    pass: passed >= (key.pass ?? key.tasks.length),
-  };
-}
-
-// The answers file the isolated session writes. YAML or JSON, one entry per
-// task id. Anything the key does not name is ignored, and a task with no entry
-// scores zero.
-export function readAnswers(path) {
-  const text = readFileSync(path, 'utf8');
-  const data = text.trimStart().startsWith('{') ? JSON.parse(text) : parseYaml(text);
-  const rows = data?.answers ?? data;
-  if (!rows || typeof rows !== 'object' || Array.isArray(rows)) throw new Error(`${path} is not a map of task id to answer`);
-  return rows;
-}