@agentproto/eval 0.2.11 → 0.3.0

package/dist/index.mjs

@@ -415,6 +415,550 @@ function toVitest(suite, opts, hooks) {
     }
   });
 }
+var FIRST_PERSON_TERMS = [
+  "je",
+  "j'",
+  "j\u2019",
+  "moi",
+  "mon",
+  "ma",
+  "mes",
+  "me",
+  "m'",
+  "m\u2019",
+  "nous",
+  "notre",
+  "nos",
+  "mien(?:ne)?s?"
+];
+var FIRST_PERSON_RE = new RegExp(
+  FIRST_PERSON_TERMS.map((term) => {
+    const isElision = term.endsWith("'") || term.endsWith("\u2019");
+    return `(?<![\\p{L}\\p{N}_])${term}${isElision ? "" : "(?![\\p{L}\\p{N}_])"}`;
+  }).join("|"),
+  "iu"
+);
+var ABBREVIATIONS = /* @__PURE__ */ new Set([
+  "m",
+  "mme",
+  "mlle",
+  "dr",
+  "st",
+  "ste",
+  "etc",
+  "cf",
+  "vs",
+  "p",
+  "ex",
+  "no",
+  "art",
+  "vol",
+  "pp",
+  "chap"
+]);
+function splitLines(text) {
+  return text.split("\n").map((line) => line.trim()).filter((line) => line.length > 0);
+}
+function endsWithAbbreviation(chunk) {
+  const match = chunk.match(/(\p{L}+)\.$/u);
+  if (!match) return false;
+  const word = match[1];
+  if (ABBREVIATIONS.has(word.toLowerCase())) return true;
+  return word.length === 1 && /\p{Lu}/u.test(word);
+}
+function splitSentences(text) {
+  const chunks = text.split(/(?<=[.!?])\s+|\n+/u).map((s) => s.trim()).filter((s) => s.length > 0);
+  const sentences = [];
+  for (const chunk of chunks) {
+    const prevIndex = sentences.length - 1;
+    const prev = prevIndex >= 0 ? sentences[prevIndex] : void 0;
+    if (prev !== void 0 && endsWithAbbreviation(prev)) {
+      sentences[prevIndex] = `${prev} ${chunk}`;
+    } else {
+      sentences.push(chunk);
+    }
+  }
+  return sentences;
+}
+function bulletsRatio(text) {
+  const lines = splitLines(text);
+  if (lines.length === 0) return 0;
+  const bulletLines = lines.filter((line) => /^([-*•]|\d+\.)\s+/u.test(line));
+  return bulletLines.length / lines.length;
+}
+function firstPersonRatio(text) {
+  const sentences = splitSentences(text);
+  if (sentences.length === 0) return 0;
+  const firstPersonSentences = sentences.filter((s) => FIRST_PERSON_RE.test(s));
+  return firstPersonSentences.length / sentences.length;
+}
+function questionRate(text) {
+  const sentences = splitSentences(text);
+  if (sentences.length === 0) return 0;
+  const questions = sentences.filter((s) => s.endsWith("?"));
+  return questions.length / sentences.length;
+}
+function meanSentenceLength(text) {
+  const sentences = splitSentences(text);
+  if (sentences.length === 0) return 0;
+  const total = sentences.reduce((sum, s) => {
+    const words = s.split(/\s+/u).filter((w) => w.length > 0);
+    return sum + words.length;
+  }, 0);
+  return total / sentences.length;
+}
+var DEFAULT_LENGTH_BAND = { min: 5, max: 30 };
+function computeTextStats(text, band = DEFAULT_LENGTH_BAND) {
+  const length = meanSentenceLength(text);
+  return {
+    bulletsRatio: bulletsRatio(text),
+    firstPersonRatio: firstPersonRatio(text),
+    questionRate: questionRate(text),
+    meanSentenceLength: length,
+    inBand: length >= band.min && length <= band.max
+  };
+}
+var textStatsThresholdsSchema = z.object({
+  maxBulletsRatio: z.number().min(0).max(1).optional().describe("Default 0.02."),
+  minFirstPersonRatio: z.number().min(0).max(1).optional().describe("Default 0.6."),
+  lengthBand: z.object({ min: z.number().min(0), max: z.number().min(0) }).refine((band) => band.min <= band.max, { message: "lengthBand.min must be <= lengthBand.max" }).optional().describe("Word-count band for mean sentence length. Default { min: 5, max: 30 }.")
+});
+var textStatsTool = defineTool({
+  id: "eval.text-stats",
+  description: "Deterministic scorer: computes bullet-line ratio, first-person ratio, question rate, and mean sentence length (+ inBand) over French text. `passed` is derived from caller-supplied `thresholds` \u2014 this tool owns no fixed gate.",
+  version: "0.1.0",
+  inputSchema: z.object({
+    text: z.string().describe("The French text to analyze."),
+    thresholds: textStatsThresholdsSchema.optional()
+  }),
+  outputSchema: scoreSchema,
+  mutates: [],
+  approval: "auto",
+  riskLevel: 0
+});
+var textStatsImpl = implementTool(textStatsTool, ({ input }) => {
+  const thresholds = input.thresholds ?? {};
+  const maxBulletsRatio = thresholds.maxBulletsRatio ?? 0.02;
+  const minFirstPersonRatio = thresholds.minFirstPersonRatio ?? 0.6;
+  const band = thresholds.lengthBand ?? DEFAULT_LENGTH_BAND;
+  const stats = computeTextStats(input.text, band);
+  const bulletsOk = stats.bulletsRatio <= maxBulletsRatio;
+  const firstPersonOk = stats.firstPersonRatio >= minFirstPersonRatio;
+  const checks = [bulletsOk, firstPersonOk, stats.inBand];
+  const value = checks.filter(Boolean).length / checks.length;
+  const passed = checks.every(Boolean);
+  return {
+    value,
+    passed,
+    label: "text-stats",
+    rationale: `bulletsRatio=${stats.bulletsRatio.toFixed(3)} (max ${maxBulletsRatio}), firstPersonRatio=${stats.firstPersonRatio.toFixed(3)} (min ${minFirstPersonRatio}), questionRate=${stats.questionRate.toFixed(3)}, meanSentenceLength=${stats.meanSentenceLength.toFixed(1)}, inBand=${stats.inBand} (band ${band.min}-${band.max})`
+  };
+});
+var WORD_RE = /\p{L}[\p{L}'’-]*/gu;
+var FRENCH_STOPWORDS = /* @__PURE__ */ new Set([
+  "le",
+  "la",
+  "les",
+  "un",
+  "une",
+  "des",
+  "de",
+  "du",
+  "et",
+  "\xE0",
+  "au",
+  "aux",
+  "ce",
+  "ces",
+  "cet",
+  "cette",
+  "que",
+  "qui",
+  "quoi",
+  "dont",
+  "o\xF9",
+  "je",
+  "tu",
+  "il",
+  "elle",
+  "on",
+  "nous",
+  "vous",
+  "ils",
+  "elles",
+  "est",
+  "sont",
+  "\xE9t\xE9",
+  "\xEAtre",
+  "avoir",
+  "ai",
+  "as",
+  "a",
+  "avons",
+  "avez",
+  "ont",
+  "pour",
+  "par",
+  "sur",
+  "sous",
+  "dans",
+  "avec",
+  "sans",
+  "ne",
+  "pas",
+  "plus",
+  "mais",
+  "ou",
+  "si",
+  "se",
+  "sa",
+  "son",
+  "ses",
+  "leur",
+  "leurs",
+  "en",
+  "y"
+]);
+function tokenize(text) {
+  return (text.normalize("NFC").toLowerCase().match(WORD_RE) ?? []).map((w) => w.trim()).filter((w) => w.length > 0);
+}
+function containsWord(text, term) {
+  const escaped = term.normalize("NFC").replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
+  const re = new RegExp(`(?<![\\p{L}\\p{N}_])${escaped}(?![\\p{L}\\p{N}_])`, "iu");
+  return re.test(text.normalize("NFC"));
+}
+function countTerms(texts, minLen) {
+  const counts = /* @__PURE__ */ new Map();
+  for (const text of texts) {
+    for (const word of tokenize(text)) {
+      if (word.length < minLen) continue;
+      if (FRENCH_STOPWORDS.has(word)) continue;
+      counts.set(word, (counts.get(word) ?? 0) + 1);
+    }
+  }
+  return counts;
+}
+function totalCount(counts) {
+  let total = 0;
+  for (const c of counts.values()) total += c;
+  return total;
+}
+function extractLexicon(corpusTexts, opts) {
+  const top = opts?.top ?? 20;
+  const minLen = opts?.minLen ?? 3;
+  const counts = countTerms(corpusTexts, minLen);
+  if (opts?.background && opts.background.length > 0) {
+    const bgCounts = countTerms(opts.background, minLen);
+    const fgTotal = totalCount(counts);
+    const bgTotal = totalCount(bgCounts);
+    return [...counts.entries()].map(([word, fg]) => {
+      const bg = bgCounts.get(word) ?? 0;
+      const score = Math.log((fg + 1) / (fgTotal - fg + 1)) - Math.log((bg + 1) / (bgTotal - bg + 1));
+      return [word, score];
+    }).sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, top).map(([word]) => word);
+  }
+  return [...counts.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])).slice(0, top).map(([word]) => word);
+}
+var lexiconHitRateTool = defineTool({
+  id: "eval.lexicon-hit-rate",
+  description: "Deterministic scorer: fraction of `lexicon` terms present as whole words in `text` (case-insensitive). `passed = hitRate >= threshold` (threshold default 0.5).",
+  version: "0.1.0",
+  inputSchema: z.object({
+    text: z.string().describe("The text to score."),
+    lexicon: z.array(z.string()).min(1).describe("Signature terms to look for, e.g. from extractLexicon."),
+    threshold: z.number().min(0).max(1).optional().describe("Minimum hit rate to pass. Default 0.5.")
+  }),
+  outputSchema: scoreSchema,
+  mutates: [],
+  approval: "auto",
+  riskLevel: 0
+});
+var lexiconHitRateImpl = implementTool(lexiconHitRateTool, ({ input }) => {
+  const threshold = input.threshold ?? 0.5;
+  const lexicon = input.lexicon;
+  const hits = lexicon.filter((term) => containsWord(input.text, term));
+  const hitRate = lexicon.length === 0 ? 0 : hits.length / lexicon.length;
+  const passed = hitRate >= threshold;
+  return {
+    value: hitRate,
+    passed,
+    label: "lexicon-hit-rate",
+    rationale: `${hits.length}/${lexicon.length} lexicon terms found (threshold ${threshold}): ${hits.join(", ") || "none"}`
+  };
+});
+
+// src/style/verdict.ts
+function parseVerdict(raw) {
+  const parsed = judgeVerdictSchema.safeParse(raw);
+  return parsed.success ? parsed.data : null;
+}
+
+// src/style/pairwise.ts
+var stylePairwiseTool = defineTool({
+  id: "eval.style-pairwise",
+  description: "Model-backed scorer: asks an injected judge which of `a`/`b` is closer in style to `reference` under `criteria`. value encodes preference (1 = a wins, 0 = b wins, 0.5 = tie). The judge lives in the driver (see makeStylePairwiseDriver) \u2014 this tool contract carries no model call.",
+  version: "0.1.0",
+  inputSchema: z.object({
+    reference: z.string().describe("The reference style text."),
+    a: z.string().describe("Candidate A."),
+    b: z.string().describe("Candidate B."),
+    criteria: z.string().describe("Free-form grading criteria/rubric for the judge.")
+  }),
+  outputSchema: scoreSchema,
+  mutates: [],
+  approval: "auto",
+  riskLevel: 0
+});
+function clamp012(x) {
+  return Math.min(1, Math.max(0, x));
+}
+function makeStylePairwiseDriver(judge) {
+  return defineDriver({
+    id: "eval-style-pairwise",
+    name: "Eval Style Pairwise (model-backed)",
+    description: "Model-backed scorer driver: implements eval.style-pairwise by awaiting an injected JudgeFn over {reference, a, b} and mapping its verdict.value to an a-vs-b preference score.",
+    version: "0.1.0",
+    kind: "builtin",
+    implements: [{ tool: "eval.style-pairwise", version: "0.1.0" }],
+    implementations: [
+      implementTool(stylePairwiseTool, async ({ input }) => {
+        const raw = await judge({
+          output: { reference: input.reference, a: input.a, b: input.b },
+          criteria: input.criteria,
+          expected: input.reference
+        });
+        const verdict = parseVerdict(raw);
+        if (!verdict) {
+          return {
+            value: 0,
+            passed: false,
+            label: "style-pairwise",
+            rationale: "judge returned a malformed verdict"
+          };
+        }
+        const value = clamp012(verdict.value);
+        const passed = verdict.passed ?? value >= 0.5;
+        return {
+          value,
+          passed,
+          label: "style-pairwise",
+          ...verdict.rationale ? { rationale: verdict.rationale } : {}
+        };
+      })
+    ]
+  });
+}
+function unswap(winner, order) {
+  if (order === "normal" || winner === "tie") return winner;
+  return winner === "a" ? "b" : "a";
+}
+function aWinScore(winner) {
+  if (winner === "a") return 1;
+  if (winner === "tie") return 0.5;
+  return 0;
+}
+function cohenKappa(r1, r2) {
+  const n = r1.length;
+  let agree = 0;
+  const categories = ["a", "b", "tie"];
+  const counts1 = new Map(categories.map((c) => [c, 0]));
+  const counts2 = new Map(categories.map((c) => [c, 0]));
+  for (let i = 0; i < n; i++) {
+    const x1 = r1[i];
+    const x2 = r2[i];
+    if (x1 === x2) agree++;
+    counts1.set(x1, (counts1.get(x1) ?? 0) + 1);
+    counts2.set(x2, (counts2.get(x2) ?? 0) + 1);
+  }
+  const po = agree / n;
+  const pe = categories.reduce((sum, c) => sum + (counts1.get(c) ?? 0) / n * ((counts2.get(c) ?? 0) / n), 0);
+  if (pe >= 1) return po === 1 ? 1 : 0;
+  return (po - pe) / (1 - pe);
+}
+function pairwiseWinRate(verdicts) {
+  const n = verdicts.length;
+  const normalized = verdicts.map((v) => ({ ...v, winner: unswap(v.winner, v.order) }));
+  const normalGroup = normalized.filter((v) => v.order === "normal");
+  const swappedGroup = normalized.filter((v) => v.order === "swapped");
+  const nNormal = normalGroup.length;
+  const nSwapped = swappedGroup.length;
+  const rateNormal = nNormal === 0 ? null : normalGroup.reduce((sum, v) => sum + aWinScore(v.winner), 0) / nNormal;
+  const rateSwapped = nSwapped === 0 ? null : swappedGroup.reduce((sum, v) => sum + aWinScore(v.winner), 0) / nSwapped;
+  const balanced = nNormal > 0 && nSwapped > 0;
+  const winRate = rateNormal !== null && rateSwapped !== null ? (rateNormal + rateSwapped) / 2 : rateNormal ?? rateSwapped ?? 0;
+  const byJudge = /* @__PURE__ */ new Map();
+  for (const v of normalized) {
+    let items = byJudge.get(v.judge);
+    if (!items) {
+      items = /* @__PURE__ */ new Map();
+      byJudge.set(v.judge, items);
+    }
+    if (!items.has(v.item)) items.set(v.item, v.winner);
+  }
+  const judges = [...byJudge.entries()].sort((a, b) => b[1].size - a[1].size);
+  let kappa = null;
+  if (judges.length >= 2) {
+    const [, items1] = judges[0];
+    const [, items2] = judges[1];
+    const commonItems = [...items1.keys()].filter((item) => items2.has(item)).sort();
+    if (commonItems.length >= 2) {
+      const r1 = commonItems.map((item) => items1.get(item));
+      const r2 = commonItems.map((item) => items2.get(item));
+      kappa = cohenKappa(r1, r2);
+    }
+  }
+  return { winRate, kappa, n, nNormal, nSwapped, balanced };
+}
+function dot(a, b) {
+  let sum = 0;
+  for (let i = 0; i < a.length; i++) sum += (a[i] ?? 0) * (b[i] ?? 0);
+  return sum;
+}
+function norm(v) {
+  return Math.sqrt(dot(v, v));
+}
+var EmbeddingDimensionError = class extends RangeError {
+};
+function centroid(vectors) {
+  const dims = vectors[0]?.length ?? 0;
+  for (const v of vectors) {
+    if (v.length !== dims) {
+      throw new EmbeddingDimensionError(`inconsistent embedding dimensions: expected ${dims}, got ${v.length}`);
+    }
+  }
+  const sum = new Array(dims).fill(0);
+  for (const v of vectors) {
+    for (let i = 0; i < dims; i++) sum[i] = (sum[i] ?? 0) + (v[i] ?? 0);
+  }
+  return sum.map((x) => x / vectors.length);
+}
+function cosineToCentroid(candidate, references) {
+  if (references.length === 0) return 0;
+  const c = centroid(references);
+  if (candidate.length !== c.length) {
+    throw new EmbeddingDimensionError(
+      `inconsistent embedding dimensions: candidate has ${candidate.length}, references have ${c.length}`
+    );
+  }
+  const cn = norm(c);
+  const vn = norm(candidate);
+  if (cn === 0 || vn === 0) return 0;
+  const cosine = dot(candidate, c) / (vn * cn);
+  return Math.max(0, Math.min(1, cosine));
+}
+var styleEmbeddingTool = defineTool({
+  id: "eval.style-embedding",
+  description: "Model-backed scorer: embeds `candidate` and `references[]` via an injected EmbedFn and scores value = cosine similarity of `candidate` to the references' centroid, clamped to [0, 1] via max(0, cosine) (orthogonal or opposing candidates score 0, not 0.5). The embedding call lives in the driver (see makeStyleEmbeddingDriver) \u2014 this tool contract carries no model call of its own.",
+  version: "0.1.0",
+  inputSchema: z.object({
+    candidate: z.string().describe("The text to score."),
+    references: z.array(z.string()).min(1).describe("Reference texts defining the style centroid.")
+  }),
+  outputSchema: scoreSchema,
+  mutates: [],
+  approval: "auto",
+  riskLevel: 0
+});
+function makeStyleEmbeddingDriver(embed, opts) {
+  const threshold = opts?.threshold ?? 0.5;
+  return defineDriver({
+    id: "eval-style-embedding",
+    name: "Eval Style Embedding (model-backed)",
+    description: "Model-backed scorer driver: implements eval.style-embedding by awaiting an injected EmbedFn and scoring cosine similarity of the candidate to the references' centroid. No LLM SDK or network call here.",
+    version: "0.1.0",
+    kind: "builtin",
+    implements: [{ tool: "eval.style-embedding", version: "0.1.0" }],
+    implementations: [
+      implementTool(styleEmbeddingTool, async ({ input }) => {
+        const vectors = await embed([input.candidate, ...input.references]);
+        const [candidateVec, ...referenceVecs] = vectors;
+        let value;
+        try {
+          value = cosineToCentroid(candidateVec ?? [], referenceVecs);
+        } catch (err) {
+          return {
+            value: 0,
+            passed: false,
+            label: "style-embedding",
+            rationale: err instanceof EmbeddingDimensionError ? err.message : "embedding failed"
+          };
+        }
+        const passed = value >= threshold;
+        return {
+          value,
+          passed,
+          label: "style-embedding",
+          rationale: `cosine-to-centroid over ${referenceVecs.length} reference(s), threshold ${threshold}`
+        };
+      })
+    ]
+  });
+}
+var outlineFidelityTool = defineTool({
+  id: "eval.outline-fidelity",
+  description: "Model-backed scorer: asks an injected judge whether `answer` covers every point in `outline` with zero added facts. value = covered/total as judged; passed = value >= 0.95, a fixed gate (not the judge's own passed). The judge lives in the driver (see makeOutlineFidelityDriver).",
+  version: "0.1.0",
+  inputSchema: z.object({
+    outline: z.string().describe("The source outline \u2014 bullet points the answer must cover."),
+    answer: z.string().describe("The produced prose answer to check for fidelity.")
+  }),
+  outputSchema: scoreSchema,
+  mutates: [],
+  approval: "auto",
+  riskLevel: 0
+});
+var FIDELITY_THRESHOLD = 0.95;
+var FIDELITY_CRITERIA = "Score how faithfully the answer covers the outline: value = (outline points covered) / (total outline points), where any fact in the answer not present in the outline counts as a violation and caps value at the fraction covered. 1.0 means every outline point is covered and nothing was added.";
+function makeOutlineFidelityDriver(judge) {
+  return defineDriver({
+    id: "eval-outline-fidelity",
+    name: "Eval Outline Fidelity (model-backed)",
+    description: "Model-backed scorer driver: implements eval.outline-fidelity by awaiting an injected JudgeFn over {answer, outline} and gating on a fixed 0.95 threshold, not the judge's own passed.",
+    version: "0.1.0",
+    kind: "builtin",
+    implements: [{ tool: "eval.outline-fidelity", version: "0.1.0" }],
+    implementations: [
+      implementTool(outlineFidelityTool, async ({ input }) => {
+        const raw = await judge({
+          output: input.answer,
+          criteria: FIDELITY_CRITERIA,
+          expected: input.outline
+        });
+        const verdict = parseVerdict(raw);
+        if (!verdict) {
+          return {
+            value: 0,
+            passed: false,
+            label: "outline-fidelity",
+            rationale: "judge returned a malformed verdict"
+          };
+        }
+        const value = Math.min(1, Math.max(0, verdict.value));
+        const passed = value >= FIDELITY_THRESHOLD;
+        return {
+          value,
+          passed,
+          label: "outline-fidelity",
+          ...verdict.rationale ? { rationale: verdict.rationale } : {}
+        };
+      })
+    ]
+  });
+}
+
+// src/style/index.ts
+var styleScorersProvider = defineDriver({
+  id: "eval-style-scorers",
+  name: "Eval Style Scorers (built-in)",
+  description: "In-process deterministic style scorers: text-stats and lexicon-hit-rate. Each is an AIP-14 TOOL whose output is the shared Score shape.",
+  version: "0.1.0",
+  kind: "builtin",
+  implements: [
+    { tool: "eval.text-stats", version: "0.1.0" },
+    { tool: "eval.lexicon-hit-rate", version: "0.1.0" }
+  ],
+  implementations: [textStatsImpl, lexiconHitRateImpl]
+});
 
 // src/index.ts
 var SPEC_NAME = "agenteval/v1";
@@ -439,6 +983,6 @@ var evalScorersProvider = defineDriver({
   ]
 });
 
-export { EVAL_EVENT_SCHEMA, SPEC_NAME, SPEC_VERSION, bindScorer, evalScorersProvider, exactMatchImpl, exactMatchTool, jsonSchemaValidImpl, jsonSchemaValidTool, jsonValueSchema, judgeVerdictSchema, latencyBudgetImpl, latencyBudgetTool, llmJudge, llmJudgeTool, makeLlmJudgeDriver, regexMatchImpl, regexMatchTool, runEval, scoreSchema, toVitest };
+export { EVAL_EVENT_SCHEMA, EmbeddingDimensionError, SPEC_NAME, SPEC_VERSION, bindScorer, bulletsRatio, computeTextStats, cosineToCentroid, evalScorersProvider, exactMatchImpl, exactMatchTool, extractLexicon, firstPersonRatio, jsonSchemaValidImpl, jsonSchemaValidTool, jsonValueSchema, judgeVerdictSchema, latencyBudgetImpl, latencyBudgetTool, lexiconHitRateImpl, lexiconHitRateTool, llmJudge, llmJudgeTool, makeLlmJudgeDriver, makeOutlineFidelityDriver, makeStyleEmbeddingDriver, makeStylePairwiseDriver, meanSentenceLength, outlineFidelityTool, pairwiseWinRate, parseVerdict, questionRate, regexMatchImpl, regexMatchTool, runEval, scoreSchema, splitSentences, styleEmbeddingTool, stylePairwiseTool, styleScorersProvider, textStatsImpl, textStatsTool, toVitest };
 //# sourceMappingURL=index.mjs.map
 //# sourceMappingURL=index.mjs.map