@polymorphism-tech/morph-spec 8.34.2 → 8.35.0

package/src/lib/tasks/eval-node.js

@@ -0,0 +1,485 @@
+/**
+ * Eval Node — the `runner: "evals"` kind of `project.tests[]`.
+ *
+ * WHY THIS IS A RUNNER KIND AND NOT A SIXTH VERIFY NODE
+ *
+ * An eval IS `dotnet test` with a category filter: one command, one cwd, one
+ * clock, one exit code. Modelling it as something other than a test suite would
+ * mean a second copy of the executor, a second decision for `--filter`, a second
+ * decision for `--linux`, a second place for `expectedCount` — and a new line in
+ * a residency-capped charter that has zero slack. As a runner kind it inherits
+ * all of that for free, and the ONE thing the rollup cannot express (eval red vs
+ * unit-test red) is recovered by the verify stamp, which records `nodes.evals`
+ * separately.
+ *
+ * WHAT THIS MODULE OWNS
+ *
+ *   - the STATES of a cache tree, judged BEFORE `dotnet test` is invoked. Paying
+ *     for a whole suite to discover at the end that the cache did not serve is
+ *     the worst possible ordering;
+ *   - the `MORPH_EVAL_*` variables the CLI DICTATES, so the same xunit suite is
+ *     runnable by `verify` without a second configuration path;
+ *   - reading the summary the fixture writes — where **null is UNKNOWN, never
+ *     zero**.
+ *
+ * WHAT IT DELIBERATELY DOES NOT OWN
+ *
+ * The pin. `readAiPin()` (src/lib/standards/verified-against.js) is the single
+ * reader of `framework/ai-pin.json`; a second parser here is how two truths
+ * about "which version is pinned" begin.
+ *
+ * @module eval-node
+ */
+
+import { existsSync, readFileSync, readdirSync, statSync } from 'fs';
+import { join, resolve, sep } from 'path';
+import { tmpdir } from 'os';
+
+import { isInside } from './test-runner.js';
+import { readAiPin } from '../standards/verified-against.js';
+
+/** The runner kind. One spelling, imported by every reader. */
+export const EVAL_RUNNER = 'evals';
+
+/** Stamp file at the root of a cache tree — the SAME name the .NET fixture writes. */
+export const EVAL_STAMP_FILE = '.morph-eval-cache.json';
+
+/** Default dataset root, relative to the cache root's parent. */
+export const DEFAULT_DATASET_DIR = 'eval-datasets';
+
+/** Default credential file. Gitignored, like `e2e.auth.envFile`; never in the committed config. */
+export const DEFAULT_ENV_FILE = '.env.evals';
+
+/**
+ * The pin entries that decide the FORMAT of a response cache.
+ *
+ * Only the `Microsoft.Extensions.AI` family: the cache key is computed by that
+ * library, and its own docs say the hash is not guaranteed stable across
+ * releases. Bumping an unrelated provider package (OpenAI, Google.GenAI) cannot
+ * invalidate a cache tree, and a stamp that reacted to it would fail for
+ * nothing — a guardrail that cries wolf gets switched off.
+ *
+ * @param {Record<string, {version: string}>} packages
+ * @returns {Record<string, string>} name → exact version
+ */
+export function cacheRelevantPackages(packages) {
+  const out = {};
+  for (const [name, entry] of Object.entries(packages ?? {})) {
+    if (name === 'Microsoft.Extensions.AI' || name.startsWith('Microsoft.Extensions.AI.Evaluation')) {
+      out[name] = entry?.version;
+    }
+  }
+  return out;
+}
+
+/**
+ * Read the pin from a project, wherever the framework lives.
+ *
+ * Two roots, ONE reader: this repository carries `framework/ai-pin.json` at the
+ * top; an installed project carries it under `.morph/framework/`. `readAiPin`
+ * appends `framework/ai-pin.json` to whatever root it is given, so both cases
+ * are the same function with a different argument — never a second parser.
+ *
+ * Returns null when neither exists: a project with no pin simply has no stamp
+ * ruler, and refusing to run there would make the eval node unusable outside
+ * this framework's own stack.
+ *
+ * @param {string} projectPath
+ * @returns {{pinnedAt: string, packages: object}|null}
+ */
+export function readProjectPin(projectPath) {
+  for (const root of [projectPath, join(projectPath, '.morph')]) {
+    try {
+      return readAiPin(root);
+    } catch {
+      // Next candidate. The throw is readAiPin's contract for "no pin here",
+      // and it is the reason this loop exists instead of an existsSync probe
+      // that would duplicate the path knowledge.
+    }
+  }
+  return null;
+}
+
+/**
+ * Resolve every path an eval node names, against the project root.
+ *
+ * @param {import('./test-nodes.js').TestNode & {evals?: object}} node
+ * @param {string} projectPath
+ * @returns {{cacheRoot: string, datasetRoot: string, resultsRoot: string, envFile: string}}
+ */
+export function resolveEvalPaths(node, projectPath) {
+  const root = resolve(projectPath);
+  const evals = node?.evals ?? {};
+  const cacheRoot = resolve(root, evals.cachePath ?? '');
+  const beside = resolve(cacheRoot, '..');
+
+  return {
+    cacheRoot,
+    datasetRoot: evals.datasetPath ? resolve(root, evals.datasetPath) : join(beside, DEFAULT_DATASET_DIR),
+    // Results live OUTSIDE the committed cache tree by construction — they carry
+    // stopwatch latency, token usage and the cache-hit flag, so committing them
+    // is a guaranteed diff on every run with no signal in it.
+    resultsRoot: join(beside, 'eval-results'),
+    envFile: resolve(root, evals.envFile ?? DEFAULT_ENV_FILE),
+  };
+}
+
+/**
+ * First problem with a node's `evals` block, or null when it is well formed.
+ *
+ * Fail-CLOSED, like the rest of `validateTestNodes`: a typo here would make the
+ * eval silently disappear, which is the exact defect this feature exists to
+ * prevent.
+ *
+ * @param {*} raw - the raw `tests[]` entry
+ * @param {string|null} projectPath
+ * @returns {string|null}
+ */
+export function evalBlockProblem(raw, projectPath) {
+  const evals = raw?.evals;
+
+  if (evals === undefined) {
+    return 'runner "evals" requires an evals.cachePath — a default would create a cache tree somewhere nobody declared';
+  }
+  if (typeof evals !== 'object' || evals === null || Array.isArray(evals)) {
+    return 'evals must be an object';
+  }
+  if (typeof evals.cachePath !== 'string' || !evals.cachePath.trim()) {
+    return 'evals.cachePath must be a non-empty string';
+  }
+  for (const key of ['datasetPath', 'envFile']) {
+    if (evals[key] !== undefined && (typeof evals[key] !== 'string' || !evals[key].trim())) {
+      return `evals.${key} must be a non-empty string`;
+    }
+  }
+
+  const root = resolve(projectPath ?? '.');
+  for (const key of ['cachePath', 'datasetPath']) {
+    const value = evals[key];
+    if (typeof value !== 'string' || !value.trim()) continue;
+    // Same rule as `cwd`: a path that leaves the root addresses ANOTHER tree.
+    if (!isInside(resolve(root, value.trim()), root)) {
+      return `evals.${key} "${value}" resolves outside the project root`;
+    }
+  }
+
+  return null;
+}
+
+/** Does this cache tree hold at least one entry, beyond the stamp? */
+function cacheHasEntries(cacheRoot) {
+  const walk = (dir) => {
+    let entries;
+    try {
+      entries = readdirSync(dir, { withFileTypes: true });
+    } catch {
+      return false;
+    }
+    for (const entry of entries) {
+      const full = join(dir, entry.name);
+      if (entry.isDirectory()) {
+        if (walk(full)) return true;
+      } else if (entry.name !== EVAL_STAMP_FILE) {
+        return true;
+      }
+    }
+    return false;
+  };
+  return walk(cacheRoot);
+}
+
+/** Read the stamp the .NET fixture wrote. null for absent OR unreadable — Check tells them apart. */
+export function readEvalStamp(cacheRoot) {
+  const path = join(cacheRoot, EVAL_STAMP_FILE);
+  if (!existsSync(path)) return null;
+  try {
+    const parsed = JSON.parse(readFileSync(path, 'utf8'));
+    return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Decide, BEFORE spending a single second of suite clock, whether this eval node
+ * can run.
+ *
+ * The order is the order of the spec, and it is not arbitrary: the three cache
+ * states are judged before the command is invoked, because discovering at the
+ * END of a suite that the cache did not serve costs the whole clock and tells
+ * the reader nothing they could not have been told first.
+ *
+ * **There is no `skip` for a bad cache.** A skip here would recreate the hole
+ * this harness closed twice: a green gate over a verification that never
+ * happened. The only skip is scope — an eval suite does not belong in the
+ * per-task loop.
+ *
+ * @param {string} projectPath
+ * @param {object} node - a resolved test node with `runner === 'evals'`
+ * @param {{taskId?: string|null, refresh?: boolean}} [options]
+ * @returns {{status: 'skip'|'fail'|null, reason: string|null}} status null means RUN
+ */
+export function evalPreflight(projectPath, node, options = {}) {
+  const { taskId = null, refresh = false } = options;
+
+  if (node?.runner !== EVAL_RUNNER) return { status: null, reason: null };
+
+  // Scope first: an eval suite does not pay for itself in a per-task loop, and
+  // the result Gate 3 reads is the feature-scoped one. Same treatment `e2e: true`
+  // already gets.
+  if (taskId) {
+    return { status: 'skip', reason: 'eval node runs at feature scope only' };
+  }
+
+  const { cacheRoot, envFile } = resolveEvalPaths(node, projectPath);
+
+  // In refresh the tree is being (re)written; demanding a populated, stamped
+  // cache would make the FIRST cache impossible to create.
+  if (refresh) {
+    if (!existsSync(envFile)) {
+      return {
+        status: 'fail',
+        reason: `eval-refresh-no-credential: --refresh-cache exige o arquivo de credencial "${envFile}", que não existe`,
+      };
+    }
+    return { status: null, reason: null };
+  }
+
+  if (!existsSync(cacheRoot)) {
+    return { status: 'fail', reason: `eval-cache-missing: ${cacheRoot}` };
+  }
+
+  if (!cacheHasEntries(cacheRoot)) {
+    // Distinct from `missing` on purpose: an empty COMMITTED directory is far
+    // more likely a `.gitignore` mistake than a deliberate absence, and the two
+    // ask for different fixes.
+    return { status: 'fail', reason: `eval-cache-empty: ${cacheRoot} não contém nenhuma entrada de cache` };
+  }
+
+  const pin = readProjectPin(projectPath);
+  if (!pin) {
+    // No pin, no ruler — and a project without one is a legitimate case, not a
+    // failure: the stamp check simply does not apply. Saying so is the point;
+    // silently treating it as "stamp ok" would be the same thing said badly.
+    return { status: null, reason: null };
+  }
+
+  const stamp = readEvalStamp(cacheRoot);
+  if (!stamp) {
+    return {
+      status: 'fail',
+      reason: `eval-cache-unstamped: ${join(cacheRoot, EVAL_STAMP_FILE)} ausente ou ilegível`
+            + ' — a árvore não diz sob quais versões foi gravada; rode verify --refresh-cache',
+    };
+  }
+
+  const expected = cacheRelevantPackages(pin.packages);
+  const recorded = stamp.packages ?? {};
+  const divergent = Object.entries(expected)
+    .filter(([name, version]) => recorded[name] !== version)
+    .map(([name, version]) => `${name}: gravada ${recorded[name] ?? '(ausente)'} vs pin ${version}`);
+
+  if (divergent.length > 0) {
+    return {
+      status: 'fail',
+      reason: `eval-cache-stale-pin: ${divergent.join('; ')} — rode verify --refresh-cache`,
+    };
+  }
+
+  return { status: null, reason: null };
+}
+
+/**
+ * The `MORPH_EVAL_*` variables the CLI dictates for one eval node.
+ *
+ * The summary path is chosen HERE, by the CLI, and not by the fixture: two eval
+ * nodes in the same `verify` must not collide, and the process that reads the
+ * file is the one that should decide where it lives.
+ *
+ * @param {object} node
+ * @param {string} projectPath
+ * @param {{refresh?: boolean}} [options]
+ * @returns {{env: Record<string,string>, summaryPath: string}}
+ */
+export function evalNodeEnv(node, projectPath, options = {}) {
+  const { refresh = false } = options;
+  const { cacheRoot, datasetRoot, resultsRoot, envFile } = resolveEvalPaths(node, projectPath);
+  const summaryPath = join(tmpdir(), `morph-eval-${node.id}-${process.pid}.json`);
+
+  const env = {
+    MORPH_EVAL_CACHE: cacheRoot,
+    MORPH_EVAL_DATASET: datasetRoot,
+    MORPH_EVAL_RESULTS: resultsRoot,
+    MORPH_EVAL_MODE: refresh ? 'refresh' : 'offline',
+    MORPH_EVAL_SUMMARY: summaryPath,
+    MORPH_EVAL_ENVFILE: envFile,
+  };
+
+  const pin = readProjectPin(projectPath);
+  if (pin) {
+    // Compact, and only the packages that decide the cache format. The fixture
+    // stamps from THIS, so it never has to read (or parse) the pin itself.
+    env.MORPH_EVAL_PIN = JSON.stringify({
+      pinnedAt: pin.pinnedAt,
+      packages: cacheRelevantPackages(pin.packages),
+    });
+  }
+
+  // The credential file goes on top, same pattern as `e2e.auth.envFile`. Absent
+  // is NOT an error offline: that is the whole point of a warm cache.
+  Object.assign(env, readEnvFile(envFile));
+
+  return { env, summaryPath };
+}
+
+/**
+ * Parse a `KEY=value` file. Blank lines and `#` comments ignored; surrounding
+ * quotes stripped; the first `=` splits.
+ *
+ * Fail-open to `{}`: an unreadable credential file must not take down a run that
+ * did not need a credential in the first place.
+ *
+ * @param {string} path
+ * @returns {Record<string,string>}
+ */
+export function readEnvFile(path) {
+  if (!path || !existsSync(path)) return {};
+  let text;
+  try {
+    text = readFileSync(path, 'utf8');
+  } catch {
+    return {};
+  }
+
+  const out = {};
+  for (const line of text.split(/\r?\n/)) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith('#')) continue;
+    const eq = trimmed.indexOf('=');
+    if (eq <= 0) continue;
+    const key = trimmed.slice(0, eq).trim();
+    let value = trimmed.slice(eq + 1).trim();
+    if (value.length >= 2 && ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'")))) {
+      value = value.slice(1, -1);
+    }
+    if (key) out[key] = value;
+  }
+  return out;
+}
+
+/**
+ * Read the summary the fixture wrote.
+ *
+ * @param {string} path
+ * @returns {object|null} **null is UNKNOWN, never zero.** A summary nobody could
+ *   read is not a suite that evaluated zero scenarios: collapsing the two would
+ *   manufacture a count drop against `expectedCount` out of a missing file.
+ */
+export function readEvalSummary(path) {
+  if (!path || !existsSync(path)) return null;
+  try {
+    const parsed = JSON.parse(readFileSync(path, 'utf8'));
+    if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return null;
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * How many scenarios an eval run evaluated.
+ *
+ * The summary wins when it is readable; otherwise the caller falls back to the
+ * `dotnet` extractor, because an eval IS `dotnet test`. **No new extractor is
+ * invented**, and an unreadable summary yields null, never 0.
+ *
+ * @param {object|null} summary
+ * @returns {number|null}
+ */
+export function evalScenarioCount(summary) {
+  const n = summary?.scenarios;
+  return Number.isInteger(n) ? n : null;
+}
+
+/**
+ * The NAMED failures of an eval run, one readable line each.
+ *
+ * The summary publishes `failures[]` (scenario, assertion, kind, reason) and
+ * without this reader the field would be written and never surfaced — a
+ * published field with no consumer, the same shape #91 closed on `superseded`.
+ * Worse here than usual: the whole point of `failures[]` is that a red eval
+ * names what broke, and a name nobody prints is not a name.
+ *
+ * Defensive by construction, because this parses a file produced by ANOTHER
+ * process: a summary that is absent, unreadable, or carries a malformed entry
+ * yields fewer lines, never a throw. A crash here would turn "the eval failed"
+ * into "the verify crashed", which reports the wrong defect.
+ *
+ * @param {object|null} summary
+ * @returns {string[]} readable lines; empty when there is nothing to name
+ */
+export function evalFailureLines(summary) {
+  const failures = summary?.failures;
+  if (!Array.isArray(failures)) return [];
+
+  return failures
+    .filter((f) => f && typeof f === 'object')
+    .map((f) => {
+      const scenario = typeof f.scenario === 'string' && f.scenario ? f.scenario : '(cenário sem nome)';
+      const assertion = typeof f.assertion === 'string' && f.assertion ? f.assertion : '(asserção sem nome)';
+      const kind = typeof f.kind === 'string' && f.kind ? f.kind : '(régua desconhecida)';
+      const reason = typeof f.reason === 'string' && f.reason ? f.reason : '(sem motivo registrado)';
+      return `eval-fail: ${scenario} — ${assertion} (${kind}): ${reason}`;
+    });
+}
+
+/**
+ * Why an otherwise-green eval run must still be red, or null when it is fine.
+ *
+ * One case, and it is the guardrail-inert shape this repository has already paid
+ * to learn: a suite that evaluated ZERO scenarios and exited 0. The exit code
+ * alone cannot see it, because nothing failed — nothing ran.
+ *
+ * @param {object|null} summary
+ * @returns {string|null}
+ */
+export function evalEmptyRunProblem(summary) {
+  if (summary && summary.scenarios === 0) {
+    return 'eval-empty-run: a suíte de eval avaliou 0 cenários e saiu verde';
+  }
+  return null;
+}
+
+/**
+ * Warning for a run whose cache went cold — the tree changed and must be committed.
+ *
+ * A warning and not a failure, deliberately: spending money nobody asked for is
+ * bad, but lying about the result is worse. Without the warning, a freshly
+ * written cache stays on the disk of whoever ran it and CI misses tomorrow.
+ *
+ * @param {object|null} summary
+ * @returns {string|null}
+ */
+export function evalColdCacheWarning(summary) {
+  if (summary?.cache === 'cold') {
+    const misses = Number.isInteger(summary.misses) ? summary.misses : '?';
+    return `a árvore de cache mudou (${misses} miss(es)) — commite ${EVAL_STAMP_FILE} e as entradas novas`;
+  }
+  return null;
+}
+
+/** Relative form of a path, for messages that should not shout an absolute path. */
+export function relativeToProject(projectPath, target) {
+  const root = resolve(projectPath) + sep;
+  const full = resolve(target);
+  return full.startsWith(root) ? full.slice(root.length).replace(/\\/g, '/') : full;
+}
+
+/** Is `path` a directory? Fail-open to false. */
+export function isDirectory(path) {
+  try {
+    return statSync(path).isDirectory();
+  } catch {
+    return false;
+  }
+}