ai-capability-framework 1.0.3 → 1.1.0

package/examples/12-capability-discovery-selection/evaluation/saved-campaign.mjs

@@ -0,0 +1,385 @@
+import fs from "node:fs";
+import path from "node:path";
+import { createHash } from "node:crypto";
+import {
+  buildCapabilitySearchCorpus,
+  hashDiscoveryContent,
+  fingerprintDiscoveryEvaluationSuite,
+  validateCapabilityEmbeddingIndex,
+  DISCOVERY_STANDARD_PROFILE
+} from "ai-capability-framework/discovery";
+import {
+  campaignBinding,
+  loadSuite,
+  registryFor,
+  trialSchedule,
+  MODES,
+  CAPS
+} from "./campaign.mjs";
+import { QUALIFICATION_WORKFLOWS } from "./qualification-workflows.mjs";
+import { FROZEN_V3, FROZEN_QUALIFICATION_WORKFLOWS } from "./freeze.mjs";
+
+export const byteHash = (bytes) =>
+  `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
+const requireEvidence = (condition) => {
+  if (!condition)
+    throw new Error("Saved campaign evidence is incomplete or incompatible.");
+};
+export function readBounded(filename, maximum) {
+  requireEvidence(
+    fs.statSync(filename).isFile() && fs.statSync(filename).size <= maximum
+  );
+  const bytes = fs.readFileSync(filename);
+  requireEvidence(bytes.length <= maximum);
+  return bytes;
+}
+const parse = (filename, maximum) =>
+  JSON.parse(readBounded(filename, maximum).toString("utf8"));
+const fingerprint = (value) =>
+  typeof value === "string" && /^sha256:[a-f0-9]{64}$/.test(value);
+const quantities = ["providerCalls", "inputTokens", "outputTokens", "estimatedCost"];
+const same = (left, right) =>
+  hashDiscoveryContent(left) === hashDiscoveryContent(right);
+const close = (left, right) =>
+  typeof right === "number" &&
+  Math.abs(left - right) <= Math.max(1e-10, Math.abs(left) * 1e-10);
+
+export function readSavedArtifact(directory, reference) {
+  requireEvidence(
+    reference &&
+      /^[a-zA-Z0-9_.-]+\.json$/.test(reference.name) &&
+      fingerprint(reference.hash)
+  );
+  const filename = path.join(directory, reference.name);
+  requireEvidence(
+    path.dirname(fs.realpathSync(filename)) === fs.realpathSync(directory)
+  );
+  const bytes = readBounded(filename, 16 * 1024 * 1024),
+    value = JSON.parse(bytes.toString("utf8"));
+  requireEvidence(hashDiscoveryContent(value) === reference.hash);
+  return { value, bytesHash: byteHash(bytes) };
+}
+
+export function validateSavedSchedule(records, trials, suite) {
+  const scheduled = [
+    "administrative-index",
+    ...["train", "baseline", "qualify"].flatMap((stage) =>
+      trialSchedule(suite, stage).map((t) => t.key)
+    ),
+    ...QUALIFICATION_WORKFLOWS.flatMap((w) =>
+      MODES.map((mode) => `workflow.${w.id}.${mode.replace("/", ".")}`)
+    )
+  ];
+  requireEvidence(
+    same([...trials.keys()], scheduled) && [...trials.values()].every((t) => !t.pending)
+  );
+  requireEvidence(
+    same(
+      records.filter((r) => r.type === "stage.complete").map((r) => r.data),
+      [
+        ...["train", "baseline", "qualify"].map((stage) => ({
+          stage,
+          hash: hashDiscoveryContent(trialSchedule(suite, stage)),
+          count: trialSchedule(suite, stage).length
+        })),
+        { stage: "workflows", count: 48 }
+      ]
+    ) &&
+      records.filter((r) => r.type === "profile.freeze").length === 1 &&
+      records.find((r) => r.type === "profile.freeze").data.k === 8
+  );
+  return scheduled;
+}
+
+/** Reads immutable evidence only. Never opens a campaign ledger or host configuration. */
+export function readSavedCampaign(inputPath) {
+  const manifest = parse(inputPath, 64_000);
+  requireEvidence(
+    Object.keys(manifest).sort().join() ===
+      "campaignDirectory,campaignFingerprint,journalHash,reportHash,reportPath,schemaVersion"
+  );
+  requireEvidence(
+    manifest.schemaVersion === "1.0" &&
+      [manifest.campaignFingerprint, manifest.journalHash, manifest.reportHash].every(
+        fingerprint
+      ) &&
+      [manifest.campaignDirectory, manifest.reportPath].every(
+        (p) => typeof p === "string" && p.trim()
+      )
+  );
+  const directory = fs.realpathSync(
+    path.resolve(path.dirname(inputPath), manifest.campaignDirectory)
+  );
+  const reportPath = fs.realpathSync(
+    path.resolve(path.dirname(inputPath), manifest.reportPath)
+  );
+  requireEvidence(!fs.existsSync(path.join(directory, "campaign.lock")));
+  const journal = readBounded(path.join(directory, "journal.jsonl"), 64 * 1024 * 1024);
+  const reportBytes = readBounded(reportPath, 16 * 1024 * 1024);
+  requireEvidence(
+    byteHash(journal) === manifest.journalHash &&
+      byteHash(reportBytes) === manifest.reportHash &&
+      journal.at(-1) === 10
+  );
+  const original = JSON.parse(reportBytes.toString("utf8"));
+  requireEvidence(
+    original.status === "evaluation_completed" &&
+      original.report?.complete &&
+      original.campaignFingerprint === manifest.campaignFingerprint
+  );
+  const suite = loadSuite(),
+    registry = registryFor(suite),
+    corpus = buildCapabilitySearchCorpus(registry);
+  requireEvidence(
+    fingerprintDiscoveryEvaluationSuite(suite) === FROZEN_V3 &&
+      hashDiscoveryContent(QUALIFICATION_WORKFLOWS) === FROZEN_QUALIFICATION_WORKFLOWS
+  );
+  const records = journal
+    .toString("utf8")
+    .trimEnd()
+    .split("\n")
+    .map((line) => {
+      requireEvidence(line.length <= 128_000);
+      return JSON.parse(line);
+    });
+  const head = records[0]?.data;
+  requireEvidence(
+    records[0]?.type === "campaign" &&
+      head?.fingerprint === manifest.campaignFingerprint &&
+      hashDiscoveryContent({ binding: head.binding, caps: head.caps }) ===
+        head.fingerprint
+  );
+  const { predecessor, ...binding } = head.binding;
+  requireEvidence(
+    same(
+      binding,
+      campaignBinding(
+        suite,
+        QUALIFICATION_WORKFLOWS,
+        binding.sourceFingerprint,
+        binding.hostIdentity
+      )
+    ) &&
+      fingerprint(binding.sourceFingerprint) &&
+      binding.hostIdentity.mock === false &&
+      predecessor &&
+      Object.keys(CAPS).every(
+        (key) =>
+          Number.isFinite(head.caps[key]) &&
+          head.caps[key] >= 0 &&
+          head.caps[key] <= CAPS[key]
+      )
+  );
+  const artifacts = [];
+  const readArtifact = (reference) => {
+    const { value, bytesHash } = readSavedArtifact(directory, reference);
+    artifacts.push({
+      name: reference.name,
+      hash: reference.hash,
+      bytesHash
+    });
+    return value;
+  };
+  const { values, trials, charged, httpRequests, index } = verifySavedJournal(
+    records,
+    readArtifact
+  );
+  const scheduled = validateSavedSchedule(records, trials, suite);
+  const ledger = original.report.ledger;
+  requireEvidence(
+    !ledger.halted &&
+      ledger.pendingReservations === 0 &&
+      ledger.pendingHttp === 0 &&
+      ledger.pendingTrials === 0 &&
+      ledger.completedTrials === scheduled.length &&
+      ledger.httpRequests === httpRequests &&
+      ledger.fingerprint === head.fingerprint &&
+      same(ledger.caps, head.caps) &&
+      quantities.every(
+        (key) =>
+          close(charged[key], ledger.charged[key]) && charged[key] <= head.caps[key]
+      ) &&
+      httpRequests <= head.caps.httpRequests
+  );
+  const identity = hashDiscoveryContent({
+    execution: binding.execution,
+    tokenizerIdentity: binding.hostIdentity.tokenizerIdentity,
+    pricing: binding.hostIdentity.pricing,
+    mock: false
+  });
+  requireEvidence(
+    index &&
+      index.indexFingerprint === original.indexFingerprint &&
+      validateCapabilityEmbeddingIndex(index, {
+        corpus,
+        configuration: {
+          provider: "openai",
+          model: binding.execution.embeddingModel,
+          dimensions: 1536,
+          configurationId: `evaluation.embedding.${identity.slice(7, 23)}`
+        },
+        mock: false,
+        limits: DISCOVERY_STANDARD_PROFILE.indexLimits
+      }).valid
+  );
+  const expectedTrials = ["train", "baseline", "qualify"].flatMap((stage) =>
+    trialSchedule(suite, stage)
+  );
+  const taskValues = values.filter((v) => v.observation),
+    workflows = values.filter((v) => v.workflowId);
+  requireEvidence(
+    same(
+      taskValues.map((v) => v.trial),
+      expectedTrials
+    ) &&
+      workflows.length === 48 &&
+      values.length === scheduled.length
+  );
+  for (const value of taskValues)
+    requireEvidence(
+      trials.get(value.trial.key).binding ===
+        hashDiscoveryContent({
+          trial: value.trial,
+          campaign: head.fingerprint,
+          index: value.trial.mode.startsWith("hybrid") ? index.indexFingerprint : null
+        })
+    );
+  for (const scenario of QUALIFICATION_WORKFLOWS)
+    for (const mode of MODES) {
+      const matching = workflows.filter(
+        (w) => w.workflowId === scenario.id && w.mode === mode
+      );
+      requireEvidence(
+        matching.length === 1 &&
+          matching[0].kind === scenario.kind &&
+          trials.get(`workflow.${scenario.id}.${mode.replace("/", ".")}`).binding ===
+            hashDiscoveryContent({
+              workflow: hashDiscoveryContent(scenario),
+              mode,
+              k: 8,
+              campaign: head.fingerprint
+            })
+      );
+    }
+  return {
+    manifest,
+    original,
+    suite,
+    registry,
+    taskValues,
+    workflows,
+    ledger,
+    artifacts,
+    directory,
+    reportPath,
+    binding,
+    trialArtifacts: Object.fromEntries(
+      [...trials].map(([key, value]) => [key, value.artifact])
+    ),
+    journalRecords: records.length
+  };
+}
+
+/** Pure journal reconciliation, also used by synthetic integrity regressions. */
+export function verifySavedJournal(records, readArtifact) {
+  requireEvidence(records[0]?.type === "campaign");
+  const values = [],
+    trials = new Map(),
+    reservations = new Map(),
+    settled = new Set(),
+    http = new Set(),
+    httpSeen = new Set();
+  const charged = Object.fromEntries(quantities.map((key) => [key, 0]));
+  let previous = null,
+    httpRequests = 0,
+    index = null;
+  for (const [offset, { hash, ...record }] of records.entries()) {
+    const d = record.data;
+    requireEvidence(
+      record.sequence === offset + 1 &&
+        record.previous === previous &&
+        hashDiscoveryContent(record) === hash
+    );
+    previous = hash;
+    requireEvidence(
+      [
+        "campaign",
+        "capacity.preflight",
+        "reservation",
+        "settlement",
+        "http.start",
+        "http.end",
+        "trial.start",
+        "trial.end",
+        "index.ready",
+        "schedule",
+        "stage.complete",
+        "profile.freeze",
+        "halt"
+      ].includes(record.type)
+    );
+    if (record.type === "campaign") requireEvidence(offset === 0);
+    if (record.type === "halt") requireEvidence(false);
+    if (record.type === "reservation") {
+      requireEvidence(!reservations.has(d.id) && !settled.has(d.id));
+      reservations.set(d.id, d);
+    }
+    if (record.type === "settlement") {
+      const reservation = reservations.get(d.id),
+        attempts = Math.max(d.providerAttempts, d.completedResponses);
+      requireEvidence(
+        reservation &&
+          reservation.runId === d.runId &&
+          [d.providerAttempts, d.completedResponses].every(
+            (n) => Number.isSafeInteger(n) && n >= 0 && n <= 1
+          ) &&
+          Object.values(d.usage).every((n) => Number.isFinite(n) && n >= 0)
+      );
+      for (const key of quantities) {
+        const amount =
+          key === "providerCalls"
+            ? attempts
+            : (d.usage[key] ?? (attempts ? reservation.amounts[key] : 0));
+        requireEvidence(
+          Number.isFinite(amount) &&
+            amount >= 0 &&
+            d.charged[key] === amount &&
+            amount <= reservation.amounts[key]
+        );
+        charged[key] += amount;
+      }
+      reservations.delete(d.id);
+      settled.add(d.id);
+    }
+    if (record.type === "http.start") {
+      requireEvidence(!httpSeen.has(d.id) && Number.isFinite(d.fee) && d.fee >= 0);
+      httpSeen.add(d.id);
+      http.add(d.id);
+      httpRequests++;
+      charged.estimatedCost += d.fee;
+    }
+    if (record.type === "http.end") requireEvidence(http.delete(d.id));
+    if (record.type === "trial.start") {
+      requireEvidence(!trials.has(d.key));
+      trials.set(d.key, { binding: d.binding, pending: true });
+    }
+    if (record.type === "trial.end") {
+      const trial = trials.get(d.key);
+      requireEvidence(trial?.pending && trial.binding === d.binding);
+      trial.pending = false;
+      trial.artifact = d.artifact;
+      const value = readArtifact(d.artifact);
+      values.push(value);
+      if (d.key === "administrative-index") {
+        requireEvidence(value.status === "built");
+        index = value.index;
+      }
+    }
+    if (record.type === "index.ready")
+      requireEvidence(same(readArtifact(d.artifact).index, index));
+  }
+  requireEvidence(
+    !reservations.size && !http.size && [...trials.values()].every((t) => !t.pending)
+  );
+  return { values, trials, charged, httpRequests, index };
+}

package/examples/12-capability-discovery-selection/evaluation/wording.mjs

@@ -0,0 +1,115 @@
+// Bounded English wording support for the synthetic application, not a prose verifier.
+export const WORDING_CHECKER_VERSION = "synthetic-wording.v2";
+
+const months = [
+  "January",
+  "February",
+  "March",
+  "April",
+  "May",
+  "June",
+  "July",
+  "August",
+  "September",
+  "October",
+  "November",
+  "December"
+];
+const monthPattern = months
+  .map((month) => `${month}|${month.slice(0, 3)}\\.?`)
+  .join("|");
+const namedDate = `(?:${monthPattern})\\s+\\d{1,2},?\\s+\\d{4}|\\d{1,2}\\s+(?:${monthPattern}),?\\s+\\d{4}`;
+const values = new RegExp(
+  `(?:[\\w.+-]+@example\\.com|\\b(?:\\d{4}-\\d{2}-\\d{2}(?:T\\d{2}:\\d{2}:\\d{2}Z)?|${namedDate}|\\d{1,2}/\\d{1,2}/\\d{4}|(?:TCK|SKU|WH|USR|ORD|EVT)-[A-Z0-9]+|\\d{2}:\\d{2}|\\d+(?:\\.\\d+)?)\\b)`,
+  "gi"
+);
+
+function dateKey(year, month, day) {
+  if (![year, month, day].every(Number.isInteger)) return null;
+  if (year < 1 || year > 9999 || month < 1 || month > 12 || day < 1) return null;
+  const leap = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0);
+  const days = [31, leap ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
+  if (day > days[month - 1]) return null;
+  return `${String(year).padStart(4, "0")}-${String(month).padStart(2, "0")}-${String(day).padStart(2, "0")}`;
+}
+
+function calendarValue(value) {
+  const iso = /^(\d{4})-(\d{2})-(\d{2})(?:T(\d{2}):(\d{2}):(\d{2})Z)?$/.exec(value);
+  if (iso) {
+    if (iso[4] !== undefined && (+iso[4] > 23 || +iso[5] > 59 || +iso[6] > 59))
+      return null;
+    return dateKey(+iso[1], +iso[2], +iso[3]);
+  }
+  const parts = value.replace(/[.,]/g, "").toLowerCase().split(/\s+/);
+  if (parts.length !== 3) return null;
+  const dayFirst = /^\d+$/.test(parts[0]);
+  const name = parts[dayFirst ? 1 : 0];
+  const month =
+    months.findIndex((entry) =>
+      [entry.toLowerCase(), entry.slice(0, 3).toLowerCase()].includes(name)
+    ) + 1;
+  return dateKey(+parts[2], month, +parts[dayFirst ? 0 : 1]);
+}
+
+export function hasUnsupportedWordingValue(summary, facts) {
+  const permitted = new Set(),
+    dates = new Set();
+  const collect = (value) => {
+    if (value && typeof value === "object") {
+      for (const child of Object.values(value)) collect(child);
+    } else if (typeof value === "string" || typeof value === "number") {
+      const authored = String(value);
+      permitted.add(authored);
+      const date = calendarValue(authored);
+      if (date) {
+        dates.add(date);
+        if (/^\d{4}-\d{2}-\d{2}T/.test(authored)) {
+          permitted.add(date);
+          permitted.add(authored.slice(11, 16));
+        }
+      }
+    }
+  };
+  collect(facts);
+  for (const match of summary.matchAll(values)) {
+    const value = match[0];
+    if (/^\d{4}-\d{2}-\d{2}T/.test(value)) {
+      if (!calendarValue(value) || !permitted.has(value)) return true;
+    } else if (
+      /^\d{4}-\d{2}-\d{2}$/.test(value) ||
+      (/[a-z]/i.test(value) && /\s/.test(value))
+    ) {
+      const date = calendarValue(value);
+      if (!date || !dates.has(date)) return true;
+    } else if (
+      !permitted.has(value) &&
+      !(/^\d+(?:\.\d+)?$/.test(value) && permitted.has(String(Number(value))))
+    )
+      return true;
+  }
+  return false;
+}
+
+export function positiveCompletionText(summary) {
+  const expanded = summary.replace(
+    /\b(has|have|had|was|were|is|are)n['’]t\b/gi,
+    "$1 not"
+  );
+  const actions =
+    /\b(?:sent|paid|refunded|committed|applied|reassigned|transferred|completed|executed|delivered|approved|prepared|drafted|cancelled)\b/gi;
+  return expanded.replace(actions, (word, offset) => {
+    // Only the action span is masked. A later affirmative clause still gets checked.
+    const prefix = expanded
+      .slice(Math.max(0, offset - 120), offset)
+      .split(/[.!?;,\n]|\b(?:but|however|and)\b/i)
+      .at(-1);
+    const negated =
+      /\b(?:not|never)\s+(?:(?:been|be|already|yet)\s+){0,2}$/i.test(prefix) ||
+      /\b(?:no (?:action|refund|payment|adjustment|invitation|invite)|nothing)\s+(?:(?:has|have|had|was|were|is|are)\s+)?(?:been\s+)?$/i.test(
+        prefix
+      );
+    const prospective =
+      /\b(?:can|could|may|might|will|would|shall)\s+(?:only\s+)?be\s+$/i.test(prefix);
+    return negated || prospective ? " ".repeat(word.length) : word;
+  });
+}