@tangle-network/agent-eval 0.110.1 → 0.112.0

package/dist/{chunk-7NX6ZSBG.js → chunk-DWLIGZBX.js}

@@ -44,8 +44,16 @@ var FINDING_SUBJECT_KINDS = [
   "knowledge.raw",
   "knowledge.stale",
   "system-prompt",
+  "skill",
   "tool-doc",
   "new-tool",
+  "mcp",
+  "hook",
+  "subagent",
+  "workflow",
+  "rollout-policy",
+  "agent-profile",
+  "code",
   "rag",
   "memory",
   "scaffolding",
@@ -73,6 +81,8 @@ function parseFindingSubject(raw) {
   if (stale) return { kind: "knowledge.stale", slug: stale[1] };
   const sp = trimmed.match(/^system-prompt:(.+)$/);
   if (sp && sp[1].trim().length > 0) return { kind: "system-prompt", section: sp[1].trim() };
+  const skill = trimmed.match(/^skill:([a-z0-9][a-z0-9_.-]*)$/);
+  if (skill) return { kind: "skill", name: skill[1] };
   const tdAspect = trimmed.match(/^tool-doc:([a-z0-9][a-z0-9_-]*):(.+)$/);
   if (tdAspect && tdAspect[2].trim().length > 0) {
     return { kind: "tool-doc", tool: tdAspect[1], aspect: tdAspect[2].trim() };
@@ -81,6 +91,26 @@ function parseFindingSubject(raw) {
   if (td) return { kind: "tool-doc", tool: td[1] };
   const nt = trimmed.match(/^new-tool:([a-z0-9][a-z0-9_-]*)$/);
   if (nt) return { kind: "new-tool", name: nt[1] };
+  const mcp = trimmed.match(/^mcp:([a-z0-9][a-z0-9_.-]*)(?::([a-z0-9][a-z0-9_.-]*))?$/);
+  if (mcp) {
+    return { kind: "mcp", server: mcp[1], ...mcp[2] ? { tool: mcp[2] } : {} };
+  }
+  const hook = trimmed.match(/^hook:([a-z0-9][a-z0-9_.-]*)$/);
+  if (hook) return { kind: "hook", name: hook[1] };
+  const subagent = trimmed.match(/^subagent:([a-z0-9][a-z0-9_.-]*)$/);
+  if (subagent) return { kind: "subagent", name: subagent[1] };
+  const workflow = trimmed.match(/^workflow:([a-z0-9][a-z0-9_.-]*)$/);
+  if (workflow) return { kind: "workflow", name: workflow[1] };
+  const rolloutPolicy = trimmed.match(/^rollout-policy:(.+)$/);
+  if (rolloutPolicy && rolloutPolicy[1].trim().length > 0) {
+    return { kind: "rollout-policy", field: rolloutPolicy[1].trim() };
+  }
+  const agentProfile = trimmed.match(/^agent-profile:(.+)$/);
+  if (agentProfile && agentProfile[1].trim().length > 0) {
+    return { kind: "agent-profile", field: agentProfile[1].trim() };
+  }
+  const code = trimmed.match(/^code:(.+)$/);
+  if (code && code[1].trim().length > 0) return { kind: "code", path: code[1].trim() };
   const rag = trimmed.match(/^rag:([a-z0-9][a-z0-9_-]*):(.+)$/);
   if (rag && rag[2].trim().length > 0) {
     return { kind: "rag", corpus: rag[1], docId: rag[2].trim() };
@@ -112,10 +142,26 @@ function renderFindingSubject(s) {
       return `agent-knowledge:stale:${s.slug}`;
     case "system-prompt":
       return `system-prompt:${s.section}`;
+    case "skill":
+      return `skill:${s.name}`;
     case "tool-doc":
       return s.aspect ? `tool-doc:${s.tool}:${s.aspect}` : `tool-doc:${s.tool}`;
     case "new-tool":
       return `new-tool:${s.name}`;
+    case "mcp":
+      return s.tool ? `mcp:${s.server}:${s.tool}` : `mcp:${s.server}`;
+    case "hook":
+      return `hook:${s.name}`;
+    case "subagent":
+      return `subagent:${s.name}`;
+    case "workflow":
+      return `workflow:${s.name}`;
+    case "rollout-policy":
+      return `rollout-policy:${s.field}`;
+    case "agent-profile":
+      return `agent-profile:${s.field}`;
+    case "code":
+      return `code:${s.path}`;
     case "rag":
       return `rag:${s.corpus}:${s.docId}`;
     case "memory":
@@ -132,33 +178,62 @@ function renderFindingSubject(s) {
       return s.label;
   }
 }
-var FINDING_SUBJECT_GRAMMAR_PROMPT = [
-  "Subjects MUST match this grammar \u2014 anything else is rejected at parse time and your work is wasted:",
-  "",
-  "  Knowledge loci (write to the agent-knowledge base):",
-  "    agent-knowledge:wiki:<slug>[#<heading>]   create / update a wiki page",
-  "    agent-knowledge:claim:<topic>             draft a claim / relation triple",
-  "    agent-knowledge:raw:<source-id>           lift a raw source into a curated page",
-  "    agent-knowledge:stale:<slug>              mark a page superseded",
-  "",
-  "  Runtime mutable surfaces (write to prompts / tools / scaffolding):",
-  "    system-prompt:<section>                   add / replace a system-prompt section",
-  "    tool-doc:<tool>[:<aspect>]                rewrite a tool description",
-  "    new-tool:<name>                           propose a new tool surface",
-  "    rag:<corpus>:<doc-id>                     ingest / correct a RAG document",
-  "    memory:<key>                              invalidate / set a memory entry",
-  "    scaffolding:<concern>                     change a precondition / retry / verifier",
-  "    output-schema:<field>                     constrain the agent output shape",
-  "",
-  "  Stale signals (knowledge-poisoning only):",
-  "    websearch:outdated:<topic>                stale web result",
-  "    prior-run-summary:<topic>                 stale prior-run summary",
-  "",
-  "  Cluster label (failure-mode only):",
-  '    <kebab-case-label>                        short cluster id, e.g. "tool-call-loop"',
-  "",
-  "Slugs / tool ids: [a-z0-9-]+ (lowercase kebab). Topics / keys / sections: free-form, trimmed."
-].join("\n");
+var FINDING_SUBJECT_SYNTAX = {
+  "knowledge.wiki": "agent-knowledge:wiki:<slug>[#<heading>]",
+  "knowledge.claim": "agent-knowledge:claim:<topic>",
+  "knowledge.raw": "agent-knowledge:raw:<source-id>",
+  "knowledge.stale": "agent-knowledge:stale:<slug>",
+  "system-prompt": "system-prompt:<section>",
+  skill: "skill:<name>",
+  "tool-doc": "tool-doc:<tool>[:<aspect>]",
+  "new-tool": "new-tool:<name>",
+  mcp: "mcp:<server>[:<tool>]",
+  hook: "hook:<name>",
+  subagent: "subagent:<name>",
+  workflow: "workflow:<name>",
+  "rollout-policy": "rollout-policy:<field>",
+  "agent-profile": "agent-profile:<field>",
+  code: "code:<path>",
+  rag: "rag:<corpus>:<doc-id>",
+  memory: "memory:<key>",
+  scaffolding: "scaffolding:<concern>",
+  "output-schema": "output-schema:<field>",
+  "websearch.outdated": "websearch:outdated:<topic>",
+  "prior-run-summary": "prior-run-summary:<topic>",
+  cluster: "<lowercase-cluster-label>"
+};
+var FINDING_SUBJECT_PURPOSE = {
+  "knowledge.wiki": "create or update a wiki page",
+  "knowledge.claim": "draft a claim or relation",
+  "knowledge.raw": "curate a raw source",
+  "knowledge.stale": "mark a stale page",
+  "system-prompt": "revise a system-prompt section",
+  skill: "create or revise a skill",
+  "tool-doc": "revise a tool contract",
+  "new-tool": "propose a new tool",
+  mcp: "revise an MCP server or tool",
+  hook: "revise a lifecycle hook",
+  subagent: "revise a delegated agent",
+  workflow: "revise an orchestration workflow",
+  "rollout-policy": "revise budget, sampling, or stop policy",
+  "agent-profile": "revise another AgentProfile field",
+  code: "revise an implementation path",
+  rag: "ingest or correct a RAG document",
+  memory: "invalidate or set memory",
+  scaffolding: "revise preconditions, retries, or verification",
+  "output-schema": "constrain the output shape",
+  "websearch.outdated": "identify a stale web result",
+  "prior-run-summary": "identify a stale prior-run summary",
+  cluster: "name one failure cluster"
+};
+function renderFindingSubjectGrammar(kinds) {
+  return [
+    "Subjects MUST match one of these forms \u2014 anything else is rejected at parse time:",
+    ...kinds.map((kind) => `  ${FINDING_SUBJECT_SYNTAX[kind]} \u2014 ${FINDING_SUBJECT_PURPOSE[kind]}`),
+    "Runtime ids are lowercase [a-z0-9_.-]+. Topics, keys, paths, and sections are free-form and trimmed."
+  ].join("\n");
+}
+var FINDING_SUBJECT_GRAMMAR_PROMPT = renderFindingSubjectGrammar(FINDING_SUBJECT_KINDS);
 var KIND_EXPECTED_SUBJECTS = {
   "failure-mode": ["cluster"],
   "knowledge-gap": [
@@ -168,6 +243,10 @@ var KIND_EXPECTED_SUBJECTS = {
     "knowledge.stale",
     "tool-doc",
     "system-prompt",
+    "skill",
+    "mcp",
+    "subagent",
+    "workflow",
     "memory",
     "websearch.outdated",
     "prior-run-summary"
@@ -178,14 +257,25 @@ var KIND_EXPECTED_SUBJECTS = {
     "knowledge.raw",
     "tool-doc",
     "system-prompt",
+    "skill",
+    "mcp",
+    "hook",
     "memory",
     "websearch.outdated",
     "prior-run-summary"
   ],
   improvement: [
     "system-prompt",
+    "skill",
     "tool-doc",
     "new-tool",
+    "mcp",
+    "hook",
+    "subagent",
+    "workflow",
+    "rollout-policy",
+    "agent-profile",
+    "code",
     "rag",
     "memory",
     "scaffolding",
@@ -194,6 +284,11 @@ var KIND_EXPECTED_SUBJECTS = {
     "knowledge.claim"
   ]
 };
+function findingSubjectGrammarPromptFor(kindId) {
+  const kinds = KIND_EXPECTED_SUBJECTS[kindId];
+  if (!kinds) throw new Error(`unknown analyst kind: ${kindId}`);
+  return renderFindingSubjectGrammar(kinds);
+}
 var FindingSubjectStringSchema = z.string().refine((s) => parseFindingSubject(s) !== null, {
   message: "subject does not match the finding-subject grammar"
 });
@@ -580,8 +675,11 @@ function buildTraceToolsForGroup(group, store) {
 }
 
 // src/analyst/kinds/failure-mode.ts
+var subjectGrammar = findingSubjectGrammarPromptFor("failure-mode");
 var ACTOR_PROMPT = `You are a failure-mode classifier for an OTLP trace dataset. Your job is to identify the **distinct ways agents failed** in this dataset, not to grade individual runs.
 
+${subjectGrammar}
+
 DISCOVERY \u2192 CLUSTER \u2192 CITE protocol:
 
 1. Call \`traces.getDatasetOverview({})\` first. Use \`has_errors\`, \`models\`, \`agent_names\`, \`tools\`, and \`sample_trace_ids\` to size the failure surface.
@@ -590,7 +688,7 @@ DISCOVERY \u2192 CLUSTER \u2192 CITE protocol:
 4. **Cluster, do not enumerate.** Two failures with the same root cause should be ONE finding citing both traces, not two findings. The point of this analyst is to compress N runs into K modes.
 5. For each cluster you can defend with evidence, emit ONE finding with:
    - \`area\` = "failure-mode"
-   - \`subject\` = a short label for the cluster ("tool-call-loop", "auth-revoked-mid-run", "agent-asked-clarification-too-late", ...)
+   - \`subject\` = a lowercase cluster label matching the subject grammar above ("tool-call-loop", "auth-revoked-mid-run", ...)
    - \`claim\` = one sentence stating the mode
    - \`severity\` = "critical" when it blocks the run, "high" when the run finished degraded, "medium" when it slowed convergence
    - \`evidence_uri\` = \`span://<trace_id>/<span_id>\` of the most representative span
@@ -623,10 +721,13 @@ var FAILURE_MODE_KIND_SPEC = {
 };
 
 // src/analyst/kinds/improvement.ts
+var subjectGrammar2 = findingSubjectGrammarPromptFor("improvement");
 var ACTOR_PROMPT2 = `You are a recursive-self-improvement analyst. Your job is to propose **concrete, locus-named edits** the agent's runtime should adopt to fix the failure modes, knowledge gaps, and poisonings present in this dataset.
 
 Upstream analysts have already classified the problems. Your job is to convert each problem into a *change to make* and grade its expected leverage. Each finding is one proposed edit.
 
+${subjectGrammar2}
+
 DISCOVERY \u2192 CANDIDATE-FIXES \u2192 COMPETE \u2192 CITE protocol:
 
 1. \`traces.getDatasetOverview({})\` first. Note the agents, tools, and any system-prompt fingerprints (look for the prompt text echoed in early spans).
@@ -638,13 +739,16 @@ DISCOVERY \u2192 CANDIDATE-FIXES \u2192 COMPETE \u2192 CITE protocol:
    - **Memory invalidation** \u2014 clear cached prior-run decisions that no longer apply
    - **Scaffolding** \u2014 add a precondition check, a retry policy, a turn budget, a verification step
    - **Output schema** \u2014 narrow the agent's output to forbid the failure shape
+   - **Skill / MCP / hook / subagent** \u2014 change the reusable profile component responsible for the behavior
+   - **Workflow / rollout policy** \u2014 change orchestration, budget, sampling, or stopping behavior
+   - **Code** \u2014 change an implementation path when profile edits cannot repair the behavior
 3. **Compete candidate fixes via subagents.** For each failure cluster, spawn one \`llmQuery\` per candidate-fix axis you want to evaluate. Each subagent's job: simulate the fix on the cited traces and report (i) likely effect, (ii) side effects, (iii) implementation cost as small/medium/large. Pass the cluster's failing trace_ids and the candidate axis as context.
 4. After subagents return, **pick the winning candidate per cluster** based on (effect / cost) and emit ONE finding. Discard the losing candidates \u2014 the output is the recommendation, not the candidate set.
 5. **Cross-reference upstream findings.** If a finding cites a prior failure-mode or knowledge-gap finding, use \`evidence_uri = "finding://<prior-finding-id>"\` (the registry supports this kind). This builds the dependency graph that lets the dashboard show "fix #X resolves failure modes A, B, C."
 
 For each winning recommendation, emit ONE finding with:
 - \`area\` = "improvement"
-- \`subject\` = the locus to edit: \`system-prompt:<section>\`, \`tool-doc:<tool-name>\`, \`new-tool:<proposed-name>\`, \`rag:<corpus>:<doc-id>\`, \`memory:<key>\`, \`scaffolding:<concern>\`, \`output-schema:<field>\`
+- \`subject\` = one exact locus form from the subject grammar above
 - \`claim\` = one sentence stating the edit ("Add a precondition check to refuse tool X calls without arg Y")
 - \`severity\` = leverage rating: "critical" when fix resolves a critical failure mode; "high" when it resolves a high; "medium" when it's a quality-of-life win; "info" when it's a cleanup with no behavioral effect
 - \`evidence_uri\` = the failure-mode finding id this fix targets (\`finding://<id>\`) when it exists; else the most representative span
@@ -674,10 +778,13 @@ var IMPROVEMENT_KIND_SPEC = {
 };
 
 // src/analyst/kinds/knowledge-gap.ts
+var subjectGrammar3 = findingSubjectGrammarPromptFor("knowledge-gap");
 var ACTOR_PROMPT3 = `You are a knowledge-gap analyst for an OTLP trace dataset. Your job is to identify the **specific pieces of information the agent lacked, or that were stale**, that caused poor decisions.
 
 The agent under analysis maintains a curated knowledge base via \`@tangle-network/agent-knowledge\` \u2014 a wiki of \`KnowledgePage\`s with raw source anchors, claims, and relations. The primary expected store of agent-knowable facts IS that wiki. A "knowledge gap" is anything the agent had to discover or guess at run-time that the wiki should have held \u2014 or an outdated/contradictory fact the agent picked up from a non-wiki source.
 
+${subjectGrammar3}
+
 DISCOVERY \u2192 ATTRIBUTE-TO-LAYER \u2192 CITE protocol:
 
 1. \`traces.getDatasetOverview({})\` first. Note which agents, tools, and models appear.
@@ -690,19 +797,10 @@ DISCOVERY \u2192 ATTRIBUTE-TO-LAYER \u2192 CITE protocol:
    - Agent quoting a tool's docs / system prompt incorrectly because the actual text was insufficient
    - Fabricated identifiers that don't appear in dataset \`sample_trace_ids\`
    Use \`traces.searchTrace\` with patterns like \`I (don.?t|do not) know\`, \`assumed\`, \`unclear\`, \`could you (clarify|tell me|provide)\`, \`not found\`, \`undefined\`, \`unknown\`, \`null\`, dates older than the analysis window, or the agent's specific clarification phrases.
-3. For each gap, identify the **layer of the runtime that should have prevented it**. The locus is the value of \`subject\` on the finding. Use one of:
-   - \`agent-knowledge:wiki:<page-slug>\` \u2014 the wiki page that should exist but doesn't, or exists but lacks the claim
-   - \`agent-knowledge:wiki:<page-slug>#<heading>\` \u2014 wiki page exists but a specific section is missing
-   - \`agent-knowledge:claim:<topic>\` \u2014 a specific claim/relation triple that should be in the wiki
-   - \`agent-knowledge:raw:<source-id>\` \u2014 raw source captured but never lifted into a curated page
-   - \`agent-knowledge:stale:<page-slug>\` \u2014 wiki page exists but contradicts ground-truth evidence in this trace (the wiki itself drifted)
-   - \`websearch:outdated:<topic>\` \u2014 agent relied on a web result that was stale; wiki should have superseded it
-   - \`tool-doc:<tool-name>:<aspect>\` \u2014 tool description missed a behavior aspect (return shape, failure modes, side effects)
-   - \`system-prompt:<section>\` \u2014 system prompt should have stated the rule directly
-   - \`memory:<key>\` \u2014 prior-run memory should have surfaced an earlier decision
+3. For each gap, identify the **layer of the runtime that should have prevented it** and use its exact locus from the subject grammar above.
 4. For each gap you can defend with evidence, emit ONE finding with:
    - \`area\` = "knowledge-gap"
-   - \`subject\` = the locus string from the list above
+   - \`subject\` = one exact locus form from the subject grammar above
    - \`claim\` = a sentence naming the missing or stale knowledge ("wiki has no page on invoice line-item shape, agent had to re-derive it from raw spans")
    - \`severity\` = "high" when the gap caused a failure or a clarifying question; "medium" when it caused unnecessary turns; "low" when it caused minor inefficiency
    - \`evidence_uri\` = \`span://<trace_id>/<span_id>\` of the moment the gap surfaced (the question, the self-correction, the retrieval miss, the stale web result)
@@ -730,8 +828,11 @@ var KNOWLEDGE_GAP_KIND_SPEC = {
 };
 
 // src/analyst/kinds/knowledge-poisoning.ts
+var subjectGrammar4 = findingSubjectGrammarPromptFor("knowledge-poisoning");
 var ACTOR_PROMPT4 = `You are a knowledge-poisoning analyst for an OTLP trace dataset. Your job is to identify cases where the agent **confidently used wrong information** \u2014 not where it lacked information (that's the knowledge-gap analyst).
 
+${subjectGrammar4}
+
 DISCOVERY \u2192 DUAL-VERIFY \u2192 CITE protocol:
 
 1. \`traces.getDatasetOverview({})\` first. Identify the agents, models, and tools.
@@ -752,7 +853,7 @@ DISCOVERY \u2192 DUAL-VERIFY \u2192 CITE protocol:
 
 For each confirmed poisoning, emit ONE finding with:
 - \`area\` = "knowledge-poisoning"
-- \`subject\` = the source of the false belief, one of: \`agent-knowledge:wiki:<page-slug>\` (wiki page contradicts current ground truth), \`agent-knowledge:claim:<topic>\` (a specific claim/relation went stale), \`agent-knowledge:raw:<source-id>\` (the raw source is outdated and the wiki inherited the drift), \`websearch:outdated:<url-or-topic>\`, \`tool-doc:<tool>\`, \`system-prompt:<section>\`, \`memory:<key>\`, \`prior-run-summary:<topic>\`
+- \`subject\` = the source of the false belief using one exact form from the subject grammar above
 - \`claim\` = one sentence: "agent believed X (from source S); evidence in trace shows X is false"
 - \`severity\` = "critical" when poisoning caused a wrong user-visible action; "high" when caught internally but wasted significant work; "medium" for inefficiency only
 - \`evidence_uri\` = \`span://<trace_id>/<span_id>\` of the action span (the moment the agent acted on the false belief)
@@ -1024,8 +1125,10 @@ export {
   FINDING_SUBJECT_KINDS,
   parseFindingSubject,
   renderFindingSubject,
+  FINDING_SUBJECT_SYNTAX,
   FINDING_SUBJECT_GRAMMAR_PROMPT,
   KIND_EXPECTED_SUBJECTS,
+  findingSubjectGrammarPromptFor,
   FindingSubjectStringSchema,
   stripCodeFences,
   coerceJson,
@@ -1045,4 +1148,4 @@ export {
   DEFAULT_TRACE_ANALYST_KINDS,
   AnalystRegistry
 };
-//# sourceMappingURL=chunk-7NX6ZSBG.js.map
+//# sourceMappingURL=chunk-DWLIGZBX.js.map