@tangle-network/agent-eval 0.99.0 → 0.100.0

package/dist/analyst/index.js

@@ -12,12 +12,12 @@ import {
   emitSkillUsageFindings,
   runSemanticConceptJudge
 } from "../chunk-SJISCGWD.js";
+import "../chunk-DJWX3GVS.js";
 import {
   behavioralAnalyst,
   buildDefaultAnalystRegistry,
   deriveEfficiencyFindings
 } from "../chunk-3NHEO6ZC.js";
-import "../chunk-DJWX3GVS.js";
 import "../chunk-BUTW4RGG.js";
 import {
   ANALYST_SEVERITIES,

package/dist/groundedness/index.d.ts

@@ -0,0 +1,112 @@
+import { S as Span } from '../schema-m0gsnbt3.js';
+
+/**
+ * Groundedness — "did the retrieval PROVIDER surface what the task needed?"
+ *
+ * A search/research provider returns text; the task needed certain facts or
+ * symbols to be solvable (the CURRENT API, a version number, a function name).
+ * This module scores how much of that required knowledge the provider's results
+ * actually surfaced — isolating PROVIDER quality (was the right thing
+ * retrievable / returned) from AGENT skill (did the agent then use it). A high
+ * groundedness score with a failed run blames the agent; a low score blames the
+ * provider. That separation is the whole point — pass/fail alone cannot make it.
+ *
+ * Structural sibling of `../authenticity`:
+ *   - authenticity scores the agent's PRODUCED files for realness.
+ *   - groundedness scores the provider's RETRIEVED text for coverage.
+ *   Both are pure deterministic scorers whose DOMAIN config is supplied by the
+ *   consumer (authenticity: `AuthenticitySignals`; groundedness:
+ *   `requiredKnowledge: string[]`) — neither bakes in a benchmark's vocabulary.
+ *
+ * Relationship to `keyword-coverage-judge`: that judge scores the agent's
+ * SERVED OUTPUT (HTML + assets) for expected concepts — a different input
+ * (produced deliverable) answering a different question (deliverable quality).
+ * Groundedness reads the RETRIEVAL side (provider results). They are
+ * complementary coverage scorers over different stages of the run, not
+ * duplicates; do not collapse one into the other.
+ *
+ * Two seams, neither forked:
+ *   - PURE SCORER `scoreGroundedness(resultText, requiredKnowledge)` — case-
+ *     insensitive substring containment over a deduped key set. Fail-open: with
+ *     no required knowledge there is nothing to ground, so `score = 1`.
+ *   - TRACE EXTRACTOR `extractRetrievedText(spans, opts?)` — pulls the provider's
+ *     returned text out of the canonical `TraceSchema` spans (`RetrievalSpan.hits`
+ *     + provider `ToolSpan.result`) instead of re-parsing bespoke run files. This
+ *     is the retrieval-side analog of `extractProducedState` (events → produced
+ *     files): structural span input, no IO, no disk walking.
+ */
+
+interface GroundednessResult {
+    /** 0..1 share of required knowledge surfaced by the provider's results.
+     *  1 when there is nothing to ground (`requiredKnowledge` empty) — fail-open. */
+    score: number;
+    /** The required-knowledge keys the result text surfaced (deduped, original casing). */
+    found: string[];
+    /** The required-knowledge keys the result text did NOT surface. */
+    missing: string[];
+    /** Distinct required-knowledge keys after dedup — the denominator of `score`. */
+    total: number;
+    /** Did the provider return any result text at all? Distinguishes "provider
+     *  surfaced nothing" (`!hadResults`) from "returned text but missed the facts"
+     *  (`hadResults && score < 1`) — the same provider-vs-agent split as the score. */
+    hadResults: boolean;
+}
+/**
+ * Score how much of `requiredKnowledge` the retrieval provider's `resultText`
+ * surfaced. Pure — same inputs, same output. No IO, no LLM.
+ *
+ * Matching is case-insensitive substring containment: each required key is
+ * checked against the lower-cased result text. This is intentionally the same
+ * cheap, deterministic containment the authenticity scorer uses for its
+ * structural signals — a key is "surfaced" if the provider's returned text
+ * mentions it. Semantic / paraphrase coverage is a separate (LLM) layer a
+ * consumer can stack on top, exactly as authenticity stacks its nuance judge.
+ *
+ * Fail-open at `total === 0`: a task with no required knowledge has nothing for
+ * the provider to ground, so it cannot be penalized (`score = 1`). The benchmark
+ * caller decides what `requiredKnowledge` is — the substrate never derives it.
+ */
+declare function scoreGroundedness(resultText: string, requiredKnowledge: readonly string[]): GroundednessResult;
+/**
+ * Predicate selecting which `ToolSpan`s are retrieval-PROVIDER calls (whose
+ * `result` carries returned text), by tool name. A parameter — never a baked-in
+ * literal — so the substrate stays free of any one benchmark's tool vocabulary,
+ * exactly as `AuthenticitySignals` keeps all domain regexes consumer-supplied.
+ */
+type ProviderToolMatcher = (toolName: string) => boolean;
+/**
+ * Default provider matcher: tool names that look like search/research but not a
+ * plain fetch/read. A sensible starting point for the common "search arm" shape;
+ * any consumer with different tool names passes its own matcher. `RetrievalSpan`s
+ * are ALWAYS included regardless of this matcher (they are retrieval by kind);
+ * the matcher only selects which generic `ToolSpan`s also count as provider calls.
+ */
+declare const defaultProviderToolMatcher: ProviderToolMatcher;
+interface ExtractRetrievedTextOptions {
+    /** Which `ToolSpan`s count as provider calls. Default: {@link defaultProviderToolMatcher}. */
+    isProviderTool?: ProviderToolMatcher;
+}
+/**
+ * Extract the retrieval PROVIDER's returned text from a span stream — the
+ * retrieval-side analog of `extractProducedState`. Reads the canonical
+ * `TraceSchema` carriers, NOT bespoke run files:
+ *   - every `RetrievalSpan`'s `hits[].content` (kind 'retrieval' — the
+ *     substrate's first-class search/research result carrier; the same `.hits`
+ *     the `bad_retrieval` failure detector already reads), and
+ *   - `ToolSpan.result` for tool spans whose `toolName` the provider matcher
+ *     accepts (kind 'tool').
+ *
+ * Pure and total: spans of other kinds, and provider tools with no result, are
+ * skipped. Returns one text blob ready for `scoreGroundedness`.
+ */
+declare function extractRetrievedText(spans: readonly Span[], opts?: ExtractRetrievedTextOptions): string;
+/**
+ * Extract the provider's retrieved text from a run's spans and score it against
+ * `requiredKnowledge` in one call — the analog of authenticity's file-in
+ * convenience. The primary contract is the standalone `scoreGroundedness`; this
+ * is the ergonomic path for a consumer holding a persisted run's `Span[]`
+ * (e.g. from `TraceStore.spans(...)`).
+ */
+declare function scoreGroundednessForRun(spans: readonly Span[], requiredKnowledge: readonly string[], opts?: ExtractRetrievedTextOptions): GroundednessResult;
+
+export { type ExtractRetrievedTextOptions, type GroundednessResult, type ProviderToolMatcher, defaultProviderToolMatcher, extractRetrievedText, scoreGroundedness, scoreGroundednessForRun };

package/dist/groundedness/index.js

@@ -0,0 +1,77 @@
+import {
+  isRetrievalSpan,
+  isToolSpan
+} from "../chunk-5BKGXME7.js";
+import "../chunk-PZ5AY32C.js";
+
+// src/groundedness/index.ts
+function dedupeKeys(keys) {
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const raw of keys) {
+    const k = raw.trim();
+    if (!k) continue;
+    const lower = k.toLowerCase();
+    if (seen.has(lower)) continue;
+    seen.add(lower);
+    out.push(k);
+  }
+  return out;
+}
+function scoreGroundedness(resultText, requiredKnowledge) {
+  const keys = dedupeKeys(requiredKnowledge);
+  const total = keys.length;
+  const text = resultText ?? "";
+  const hadResults = text.trim().length > 0;
+  const haystack = text.toLowerCase();
+  if (total === 0) {
+    return { score: 1, found: [], missing: [], total: 0, hadResults };
+  }
+  const found = [];
+  const missing = [];
+  for (const key of keys) {
+    if (haystack.includes(key.toLowerCase())) found.push(key);
+    else missing.push(key);
+  }
+  return { score: found.length / total, found, missing, total, hadResults };
+}
+var defaultProviderToolMatcher = (name) => /search|research/i.test(name) && !/fetch/i.test(name);
+function resultToText(result) {
+  if (result == null) return "";
+  if (typeof result === "string") return result;
+  try {
+    return JSON.stringify(result);
+  } catch {
+    return String(result);
+  }
+}
+function retrievalSpanText(span) {
+  return span.hits.map((h) => h.content ?? "").filter((c) => c.length > 0).join("\n");
+}
+function extractRetrievedText(spans, opts = {}) {
+  const isProviderTool = opts.isProviderTool ?? defaultProviderToolMatcher;
+  const parts = [];
+  for (const span of spans) {
+    if (isRetrievalSpan(span)) {
+      const t = retrievalSpanText(span);
+      if (t) parts.push(t);
+    } else if (isToolSpan(span)) {
+      const ts = span;
+      if (isProviderTool(ts.toolName)) {
+        const t = resultToText(ts.result);
+        if (t) parts.push(t);
+      }
+    }
+  }
+  return parts.join("\n");
+}
+function scoreGroundednessForRun(spans, requiredKnowledge, opts = {}) {
+  return scoreGroundedness(extractRetrievedText(spans, opts), requiredKnowledge);
+}
+export {
+  defaultProviderToolMatcher,
+  extractRetrievedText,
+  scoreGroundedness,
+  scoreGroundednessForRun
+};
+//# sourceMappingURL=index.js.map

package/dist/groundedness/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/groundedness/index.ts"],"sourcesContent":["/**\n * Groundedness — \"did the retrieval PROVIDER surface what the task needed?\"\n *\n * A search/research provider returns text; the task needed certain facts or\n * symbols to be solvable (the CURRENT API, a version number, a function name).\n * This module scores how much of that required knowledge the provider's results\n * actually surfaced — isolating PROVIDER quality (was the right thing\n * retrievable / returned) from AGENT skill (did the agent then use it). A high\n * groundedness score with a failed run blames the agent; a low score blames the\n * provider. That separation is the whole point — pass/fail alone cannot make it.\n *\n * Structural sibling of `../authenticity`:\n *   - authenticity scores the agent's PRODUCED files for realness.\n *   - groundedness scores the provider's RETRIEVED text for coverage.\n *   Both are pure deterministic scorers whose DOMAIN config is supplied by the\n *   consumer (authenticity: `AuthenticitySignals`; groundedness:\n *   `requiredKnowledge: string[]`) — neither bakes in a benchmark's vocabulary.\n *\n * Relationship to `keyword-coverage-judge`: that judge scores the agent's\n * SERVED OUTPUT (HTML + assets) for expected concepts — a different input\n * (produced deliverable) answering a different question (deliverable quality).\n * Groundedness reads the RETRIEVAL side (provider results). They are\n * complementary coverage scorers over different stages of the run, not\n * duplicates; do not collapse one into the other.\n *\n * Two seams, neither forked:\n *   - PURE SCORER `scoreGroundedness(resultText, requiredKnowledge)` — case-\n *     insensitive substring containment over a deduped key set. Fail-open: with\n *     no required knowledge there is nothing to ground, so `score = 1`.\n *   - TRACE EXTRACTOR `extractRetrievedText(spans, opts?)` — pulls the provider's\n *     returned text out of the canonical `TraceSchema` spans (`RetrievalSpan.hits`\n *     + provider `ToolSpan.result`) instead of re-parsing bespoke run files. This\n *     is the retrieval-side analog of `extractProducedState` (events → produced\n *     files): structural span input, no IO, no disk walking.\n */\n\nimport type { RetrievalSpan, Span, ToolSpan } from '../trace/schema'\nimport { isRetrievalSpan, isToolSpan } from '../trace/schema'\n\n// ── Pure scorer ──────────────────────────────────────────────────────────────\n\nexport interface GroundednessResult {\n  /** 0..1 share of required knowledge surfaced by the provider's results.\n   *  1 when there is nothing to ground (`requiredKnowledge` empty) — fail-open. */\n  score: number\n  /** The required-knowledge keys the result text surfaced (deduped, original casing). */\n  found: string[]\n  /** The required-knowledge keys the result text did NOT surface. */\n  missing: string[]\n  /** Distinct required-knowledge keys after dedup — the denominator of `score`. */\n  total: number\n  /** Did the provider return any result text at all? Distinguishes \"provider\n   *  surfaced nothing\" (`!hadResults`) from \"returned text but missed the facts\"\n   *  (`hadResults && score < 1`) — the same provider-vs-agent split as the score. */\n  hadResults: boolean\n}\n\n/**\n * Dedup a knowledge-key list, case-insensitively, keeping first-seen casing and\n * dropping blanks. The score denominator is distinct keys, so a config that\n * lists the same symbol twice (or with different casing) can't inflate `total`.\n */\nfunction dedupeKeys(keys: readonly string[]): string[] {\n  const seen = new Set<string>()\n  const out: string[] = []\n  for (const raw of keys) {\n    const k = raw.trim()\n    if (!k) continue\n    const lower = k.toLowerCase()\n    if (seen.has(lower)) continue\n    seen.add(lower)\n    out.push(k)\n  }\n  return out\n}\n\n/**\n * Score how much of `requiredKnowledge` the retrieval provider's `resultText`\n * surfaced. Pure — same inputs, same output. No IO, no LLM.\n *\n * Matching is case-insensitive substring containment: each required key is\n * checked against the lower-cased result text. This is intentionally the same\n * cheap, deterministic containment the authenticity scorer uses for its\n * structural signals — a key is \"surfaced\" if the provider's returned text\n * mentions it. Semantic / paraphrase coverage is a separate (LLM) layer a\n * consumer can stack on top, exactly as authenticity stacks its nuance judge.\n *\n * Fail-open at `total === 0`: a task with no required knowledge has nothing for\n * the provider to ground, so it cannot be penalized (`score = 1`). The benchmark\n * caller decides what `requiredKnowledge` is — the substrate never derives it.\n */\nexport function scoreGroundedness(\n  resultText: string,\n  requiredKnowledge: readonly string[],\n): GroundednessResult {\n  const keys = dedupeKeys(requiredKnowledge)\n  const total = keys.length\n  const text = resultText ?? ''\n  const hadResults = text.trim().length > 0\n  const haystack = text.toLowerCase()\n\n  if (total === 0) {\n    return { score: 1, found: [], missing: [], total: 0, hadResults }\n  }\n\n  const found: string[] = []\n  const missing: string[] = []\n  for (const key of keys) {\n    if (haystack.includes(key.toLowerCase())) found.push(key)\n    else missing.push(key)\n  }\n\n  return { score: found.length / total, found, missing, total, hadResults }\n}\n\n// ── Trace extractor ────────────────────────────────────────────────────────\n\n/**\n * Predicate selecting which `ToolSpan`s are retrieval-PROVIDER calls (whose\n * `result` carries returned text), by tool name. A parameter — never a baked-in\n * literal — so the substrate stays free of any one benchmark's tool vocabulary,\n * exactly as `AuthenticitySignals` keeps all domain regexes consumer-supplied.\n */\nexport type ProviderToolMatcher = (toolName: string) => boolean\n\n/**\n * Default provider matcher: tool names that look like search/research but not a\n * plain fetch/read. A sensible starting point for the common \"search arm\" shape;\n * any consumer with different tool names passes its own matcher. `RetrievalSpan`s\n * are ALWAYS included regardless of this matcher (they are retrieval by kind);\n * the matcher only selects which generic `ToolSpan`s also count as provider calls.\n */\nexport const defaultProviderToolMatcher: ProviderToolMatcher = (name) =>\n  /search|research/i.test(name) && !/fetch/i.test(name)\n\nexport interface ExtractRetrievedTextOptions {\n  /** Which `ToolSpan`s count as provider calls. Default: {@link defaultProviderToolMatcher}. */\n  isProviderTool?: ProviderToolMatcher\n}\n\n/** Stringify a `ToolSpan.result` of unknown shape into searchable text. */\nfunction resultToText(result: unknown): string {\n  if (result == null) return ''\n  if (typeof result === 'string') return result\n  try {\n    return JSON.stringify(result)\n  } catch {\n    return String(result)\n  }\n}\n\n/** Pull the retrieved text out of a `RetrievalSpan`: every hit's `content`. */\nfunction retrievalSpanText(span: RetrievalSpan): string {\n  return span.hits\n    .map((h) => h.content ?? '')\n    .filter((c) => c.length > 0)\n    .join('\\n')\n}\n\n/**\n * Extract the retrieval PROVIDER's returned text from a span stream — the\n * retrieval-side analog of `extractProducedState`. Reads the canonical\n * `TraceSchema` carriers, NOT bespoke run files:\n *   - every `RetrievalSpan`'s `hits[].content` (kind 'retrieval' — the\n *     substrate's first-class search/research result carrier; the same `.hits`\n *     the `bad_retrieval` failure detector already reads), and\n *   - `ToolSpan.result` for tool spans whose `toolName` the provider matcher\n *     accepts (kind 'tool').\n *\n * Pure and total: spans of other kinds, and provider tools with no result, are\n * skipped. Returns one text blob ready for `scoreGroundedness`.\n */\nexport function extractRetrievedText(\n  spans: readonly Span[],\n  opts: ExtractRetrievedTextOptions = {},\n): string {\n  const isProviderTool = opts.isProviderTool ?? defaultProviderToolMatcher\n  const parts: string[] = []\n  for (const span of spans) {\n    if (isRetrievalSpan(span)) {\n      const t = retrievalSpanText(span)\n      if (t) parts.push(t)\n    } else if (isToolSpan(span)) {\n      const ts = span as ToolSpan\n      if (isProviderTool(ts.toolName)) {\n        const t = resultToText(ts.result)\n        if (t) parts.push(t)\n      }\n    }\n  }\n  return parts.join('\\n')\n}\n\n// ── Convenience: extract-then-score ───────────────────────────────────────────\n\n/**\n * Extract the provider's retrieved text from a run's spans and score it against\n * `requiredKnowledge` in one call — the analog of authenticity's file-in\n * convenience. The primary contract is the standalone `scoreGroundedness`; this\n * is the ergonomic path for a consumer holding a persisted run's `Span[]`\n * (e.g. from `TraceStore.spans(...)`).\n */\nexport function scoreGroundednessForRun(\n  spans: readonly Span[],\n  requiredKnowledge: readonly string[],\n  opts: ExtractRetrievedTextOptions = {},\n): GroundednessResult {\n  return scoreGroundedness(extractRetrievedText(spans, opts), requiredKnowledge)\n}\n"],"mappings":";;;;;;;AA8DA,SAAS,WAAW,MAAmC;AACrD,QAAM,OAAO,oBAAI,IAAY;AAC7B,QAAM,MAAgB,CAAC;AACvB,aAAW,OAAO,MAAM;AACtB,UAAM,IAAI,IAAI,KAAK;AACnB,QAAI,CAAC,EAAG;AACR,UAAM,QAAQ,EAAE,YAAY;AAC5B,QAAI,KAAK,IAAI,KAAK,EAAG;AACrB,SAAK,IAAI,KAAK;AACd,QAAI,KAAK,CAAC;AAAA,EACZ;AACA,SAAO;AACT;AAiBO,SAAS,kBACd,YACA,mBACoB;AACpB,QAAM,OAAO,WAAW,iBAAiB;AACzC,QAAM,QAAQ,KAAK;AACnB,QAAM,OAAO,cAAc;AAC3B,QAAM,aAAa,KAAK,KAAK,EAAE,SAAS;AACxC,QAAM,WAAW,KAAK,YAAY;AAElC,MAAI,UAAU,GAAG;AACf,WAAO,EAAE,OAAO,GAAG,OAAO,CAAC,GAAG,SAAS,CAAC,GAAG,OAAO,GAAG,WAAW;AAAA,EAClE;AAEA,QAAM,QAAkB,CAAC;AACzB,QAAM,UAAoB,CAAC;AAC3B,aAAW,OAAO,MAAM;AACtB,QAAI,SAAS,SAAS,IAAI,YAAY,CAAC,EAAG,OAAM,KAAK,GAAG;AAAA,QACnD,SAAQ,KAAK,GAAG;AAAA,EACvB;AAEA,SAAO,EAAE,OAAO,MAAM,SAAS,OAAO,OAAO,SAAS,OAAO,WAAW;AAC1E;AAmBO,IAAM,6BAAkD,CAAC,SAC9D,mBAAmB,KAAK,IAAI,KAAK,CAAC,SAAS,KAAK,IAAI;AAQtD,SAAS,aAAa,QAAyB;AAC7C,MAAI,UAAU,KAAM,QAAO;AAC3B,MAAI,OAAO,WAAW,SAAU,QAAO;AACvC,MAAI;AACF,WAAO,KAAK,UAAU,MAAM;AAAA,EAC9B,QAAQ;AACN,WAAO,OAAO,MAAM;AAAA,EACtB;AACF;AAGA,SAAS,kBAAkB,MAA6B;AACtD,SAAO,KAAK,KACT,IAAI,CAAC,MAAM,EAAE,WAAW,EAAE,EAC1B,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,EAC1B,KAAK,IAAI;AACd;AAeO,SAAS,qBACd,OACA,OAAoC,CAAC,GAC7B;AACR,QAAM,iBAAiB,KAAK,kBAAkB;AAC9C,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,OAAO;AACxB,QAAI,gBAAgB,IAAI,GAAG;AACzB,YAAM,IAAI,kBAAkB,IAAI;AAChC,UAAI,EAAG,OAAM,KAAK,CAAC;AAAA,IACrB,WAAW,WAAW,IAAI,GAAG;AAC3B,YAAM,KAAK;AACX,UAAI,eAAe,GAAG,QAAQ,GAAG;AAC/B,cAAM,IAAI,aAAa,GAAG,MAAM;AAChC,YAAI,EAAG,OAAM,KAAK,CAAC;AAAA,MACrB;AAAA,IACF;AAAA,EACF;AACA,SAAO,MAAM,KAAK,IAAI;AACxB;AAWO,SAAS,wBACd,OACA,mBACA,OAAoC,CAAC,GACjB;AACpB,SAAO,kBAAkB,qBAAqB,OAAO,IAAI,GAAG,iBAAiB;AAC/E;","names":[]}