@tangle-network/agent-eval 0.97.0 → 0.99.0

package/dist/index.js

@@ -34,16 +34,24 @@ import {
   summarizeRecords
 } from "./chunk-STGVSCDH.js";
 import {
+  JudgeParseError,
+  adversarialJudge,
   agentProfileHash,
   agentProfileId,
   agentProfileModelId,
+  codeExecutionJudge,
+  coherenceJudge,
   completionVerdict,
+  createCustomJudge,
+  createDomainExpertJudge,
   createLlmCorrectnessChecker,
   createTokenRecallChecker,
+  defaultJudges,
   extractProducedState,
+  llmJudge,
   parseCorrectnessResponse,
   verifyCompletion
-} from "./chunk-HMA63UEO.js";
+} from "./chunk-OYU4D7FY.js";
 import {
   DEFAULT_MUTATION_PRIMITIVES,
   DEFAULT_RED_TEAM_CORPUS,
@@ -114,21 +122,18 @@ import {
 } from "./chunk-ZOPLCJSY.js";
 import {
   DEFAULT_COMPLEXITY_WEIGHTS,
-  DEFAULT_RUN_SCORE_WEIGHTS,
   FindingsStore,
   RunCritic,
   SEMANTIC_CONCEPT_JUDGE_VERSION,
   SKILL_USAGE_ANALYST,
   SkillUsageAnalyst,
-  aggregateRunScore,
-  clamp01,
   createAnalystAi,
   createChatClient,
   createSemanticConceptJudge,
   defaultIsMaterial,
   diffFindings,
   runSemanticConceptJudge
-} from "./chunk-PSWWQXHF.js";
+} from "./chunk-SJISCGWD.js";
 import {
   buildDefaultAnalystRegistry,
   computeTraceMetrics
@@ -137,6 +142,11 @@ import {
   LockedJsonlAppender,
   Mutex
 } from "./chunk-DJWX3GVS.js";
+import {
+  DEFAULT_RUN_SCORE_WEIGHTS,
+  aggregateRunScore,
+  clamp01
+} from "./chunk-BUTW4RGG.js";
 import {
   AnalystRegistry,
   DEFAULT_TRACE_ANALYST_KINDS,
@@ -405,39 +415,35 @@ import {
 } from "./chunk-PZ5AY32C.js";
 
 // src/auto-pr.ts
-async function proposeAutomatedPullRequest(client, input) {
-  validate(input);
-  return client.proposeChange(input);
-}
-function validate(input) {
+function validateProposeInput(input) {
   if (!input.repo.owner.trim() || !input.repo.name.trim()) {
-    throw new ValidationError("proposeAutomatedPullRequest: repo.owner and repo.name required");
+    throw new ValidationError("proposeChange: repo.owner and repo.name required");
   }
   if (!input.branchName.trim() || /\s/.test(input.branchName)) {
     throw new ValidationError(
-      "proposeAutomatedPullRequest: branchName must be non-empty and contain no whitespace"
+      "proposeChange: branchName must be non-empty and contain no whitespace"
     );
   }
   if (input.branchName === (input.baseBranch ?? "main")) {
-    throw new ValidationError("proposeAutomatedPullRequest: branchName must differ from baseBranch");
+    throw new ValidationError("proposeChange: branchName must differ from baseBranch");
   }
   if (input.fileChanges.length === 0) {
-    throw new ValidationError("proposeAutomatedPullRequest: fileChanges must not be empty");
+    throw new ValidationError("proposeChange: fileChanges must not be empty");
   }
   const seenPaths = /* @__PURE__ */ new Set();
   for (const change of input.fileChanges) {
     if (!change.path.trim() || change.path.includes("..") || change.path.startsWith("/")) {
       throw new ValidationError(
-        `proposeAutomatedPullRequest: invalid file path "${change.path}" (no '..' or leading '/')`
+        `proposeChange: invalid file path "${change.path}" (no '..' or leading '/')`
       );
     }
     if (seenPaths.has(change.path)) {
-      throw new ValidationError(`proposeAutomatedPullRequest: duplicate file path "${change.path}"`);
+      throw new ValidationError(`proposeChange: duplicate file path "${change.path}"`);
     }
     seenPaths.add(change.path);
   }
   if (!input.title.trim()) {
-    throw new ValidationError("proposeAutomatedPullRequest: title must not be empty");
+    throw new ValidationError("proposeChange: title must not be empty");
   }
 }
 function httpGithubClient(opts) {
@@ -459,13 +465,14 @@ function httpGithubClient(opts) {
     if (!res.ok) {
       const text = await res.text().catch(() => "");
       throw new ConfigError(
-        `proposeAutomatedPullRequest: GitHub ${method} ${path} \u2192 ${res.status} ${text.slice(0, 400)}`
+        `proposeChange: GitHub ${method} ${path} \u2192 ${res.status} ${text.slice(0, 400)}`
       );
     }
     return await res.json();
   }
   return {
     async proposeChange(input) {
+      validateProposeInput(input);
       const baseBranch = input.baseBranch ?? "main";
       const repoPath = `/repos/${input.repo.owner}/${input.repo.name}`;
       if (input.dryRun) {
@@ -478,14 +485,12 @@ function httpGithubClient(opts) {
       }
       const baseRef = await api("GET", `${repoPath}/git/ref/heads/${baseBranch}`);
       if (!baseRef) {
-        throw new ConfigError(`proposeAutomatedPullRequest: base branch "${baseBranch}" not found`);
+        throw new ConfigError(`proposeChange: base branch "${baseBranch}" not found`);
       }
       const baseSha = baseRef.object.sha;
       const baseCommit = await api("GET", `${repoPath}/git/commits/${baseSha}`);
       if (!baseCommit) {
-        throw new ConfigError(
-          `proposeAutomatedPullRequest: base commit ${baseSha} not found (race condition?)`
-        );
+        throw new ConfigError(`proposeChange: base commit ${baseSha} not found (race condition?)`);
       }
       const treeEntries = [];
       for (const change of input.fileChanges) {
@@ -493,7 +498,7 @@ function httpGithubClient(opts) {
           content: change.contents,
           encoding: "utf-8"
         });
-        if (!blob) throw new ConfigError("proposeAutomatedPullRequest: blob creation returned null");
+        if (!blob) throw new ConfigError("proposeChange: blob creation returned null");
         treeEntries.push({
           path: change.path,
           mode: "100644",
@@ -505,7 +510,7 @@ function httpGithubClient(opts) {
         base_tree: baseCommit.tree.sha,
         tree: treeEntries
       });
-      if (!tree) throw new ConfigError("proposeAutomatedPullRequest: tree creation returned null");
+      if (!tree) throw new ConfigError("proposeChange: tree creation returned null");
       const author = input.authorName && input.authorEmail ? { name: input.authorName, email: input.authorEmail, date: now().toISOString() } : void 0;
       const commitMessage = renderCommitMessage(input);
       const commit = await api("POST", `${repoPath}/git/commits`, {
@@ -514,8 +519,7 @@ function httpGithubClient(opts) {
         parents: [baseSha],
         ...author ? { author, committer: author } : {}
       });
-      if (!commit)
-        throw new ConfigError("proposeAutomatedPullRequest: commit creation returned null");
+      if (!commit) throw new ConfigError("proposeChange: commit creation returned null");
       const existing = await api(
         "GET",
         `${repoPath}/git/ref/heads/${input.branchName}`,
@@ -547,8 +551,7 @@ function httpGithubClient(opts) {
           head: input.branchName,
           base: baseBranch
         });
-        if (!created)
-          throw new ConfigError("proposeAutomatedPullRequest: PR creation returned null");
+        if (!created) throw new ConfigError("proposeChange: PR creation returned null");
         pr = created;
       }
       if (input.reviewers && input.reviewers.length > 0) {
@@ -586,13 +589,14 @@ function ghCliClient(opts = {}) {
     const r = await exec(cmd, args, { cwd, stdin });
     if (r.exitCode !== 0) {
       throw new ConfigError(
-        `proposeAutomatedPullRequest: ${cmd} ${args.join(" ")} failed (${r.exitCode}): ${r.stderr.trim() || r.stdout.trim()}`
+        `proposeChange: ${cmd} ${args.join(" ")} failed (${r.exitCode}): ${r.stderr.trim() || r.stdout.trim()}`
       );
     }
     return r;
   }
   return {
     async proposeChange(input) {
+      validateProposeInput(input);
       const baseBranch = input.baseBranch ?? "main";
       if (input.dryRun) {
         return {
@@ -702,217 +706,6 @@ function renderCommitMessage(input) {
   return lines.join("\n").trim();
 }
 
-// src/judges.ts
-var JudgeParseError = class extends JudgeError {
-  /** Name of the judge whose response failed to parse. */
-  judgeName;
-  /** The raw (truncated) model response that failed to parse. */
-  raw;
-  constructor(judgeName, raw, options) {
-    super(`judge '${judgeName}' returned an unparseable response: ${raw.slice(0, 200)}`, options);
-    this.judgeName = judgeName;
-    this.raw = raw;
-  }
-};
-function createDomainExpertJudge(domain) {
-  return async (tc, { scenario, turns }) => {
-    const conversation = turns.map(
-      (t, i) => `Turn ${i + 1}:
-User: ${t.userMessage}
-Agent: ${t.agentResponse.slice(0, 2e3)}`
-    ).join("\n\n---\n\n");
-    const resp = await tc.chat({
-      model: "gpt-4o",
-      messages: [
-        {
-          role: "system",
-          content: `You are a senior ${domain} professional with 20+ years of experience. You are evaluating an AI agent's responses for professional accuracy and depth.
-
-Score STRICTLY. A 5 means "a junior professional could do this." An 8 means "solid mid-career work." A 10 means "I would hire this agent."
-
-Evaluate:
-1. **domain_accuracy** (0-10): Are the technical terms correct? Are the recommendations what you'd actually do? Would this advice cause problems if followed?
-2. **professional_depth** (0-10): Does it go beyond surface-level? Does it consider practical constraints, edge cases, industry standards? Or is it generic textbook advice?
-
-Respond with JSON only: [{"dimension":"domain_accuracy","score":N,"reasoning":"...","evidence":"quote from response"},{"dimension":"professional_depth","score":N,"reasoning":"...","evidence":"quote"}]`
-        },
-        {
-          role: "user",
-          content: `Persona: ${scenario.persona} (${scenario.label})
-Scenario: ${scenario.thesis}
-
-${conversation}`
-        }
-      ],
-      temperature: 0.1,
-      maxTokens: 800
-    });
-    return parseJudgeResponse("domain_expert", resp);
-  };
-}
-var codeExecutionJudge = async (tc, { scenario, artifacts }) => {
-  const codeBlocks = artifacts.codeBlocks;
-  if (codeBlocks.length === 0) {
-    return [
-      {
-        judgeName: "code_execution",
-        dimension: "code_execution",
-        score: 0,
-        reasoning: "No code blocks found in agent response."
-      }
-    ];
-  }
-  const codeText = codeBlocks.map(
-    (b, i) => `Block ${i + 1} (${b.language}):
-\`\`\`${b.language}
-${b.code.slice(0, 3e3)}
-\`\`\``
-  ).join("\n\n");
-  const resp = await tc.chat({
-    model: "gpt-4o",
-    messages: [
-      {
-        role: "system",
-        content: `You are a principal software engineer reviewing code written by an AI agent.
-
-Score STRICTLY:
-1. **executability** (0-10): Would this code run without errors? Check: import errors, undefined variables, missing deps, syntax errors. A 5 means "would run with minor fixes." A 10 means "copy-paste and it works."
-2. **completeness** (0-10): Does it handle the FULL task, or just the happy path? A 5 means "handles the main case." A 10 means "production-ready."
-3. **reusability** (0-10): Could this be saved as a tool and reused? A 5 means "works for this case." A 10 means "general-purpose tool."
-
-Respond with JSON only: [{"dimension":"executability","score":N,"reasoning":"...","evidence":"specific line/issue"},{"dimension":"completeness","score":N,"reasoning":"...","evidence":"..."},{"dimension":"reusability","score":N,"reasoning":"...","evidence":"..."}]`
-      },
-      {
-        role: "user",
-        content: `Task: ${scenario.thesis}
-
-${codeText}`
-      }
-    ],
-    temperature: 0.1,
-    maxTokens: 1e3
-  });
-  return parseJudgeResponse("code_execution", resp);
-};
-var coherenceJudge = async (tc, { scenario, turns }) => {
-  if (turns.length < 2) {
-    return [];
-  }
-  const conversation = turns.map(
-    (t, i) => `Turn ${i + 1}:
-User: ${t.userMessage}
-Agent (${t.agentResponse.length} chars): ${t.agentResponse.slice(0, 1500)}`
-  ).join("\n\n---\n\n");
-  const resp = await tc.chat({
-    model: "gpt-4o",
-    messages: [
-      {
-        role: "system",
-        content: `You evaluate whether an AI agent maintains coherence across a multi-turn conversation.
-
-Score STRICTLY:
-1. **consistency** (0-10): Does the agent contradict itself across turns? Does it remember what it said/built earlier?
-2. **progression** (0-10): Does each turn BUILD on the previous? Or does it start fresh? A 5 means "vaguely related." A 10 means "each turn clearly advances the work."
-3. **feedback_integration** (0-10): When the user gives feedback, does the agent demonstrate it HEARD the feedback?
-
-Respond with JSON only: [{"dimension":"consistency","score":N,"reasoning":"..."},{"dimension":"progression","score":N,"reasoning":"..."},{"dimension":"feedback_integration","score":N,"reasoning":"..."}]`
-      },
-      {
-        role: "user",
-        content: `Scenario: ${scenario.thesis}
-
-${conversation}`
-      }
-    ],
-    temperature: 0.1,
-    maxTokens: 800
-  });
-  return parseJudgeResponse("coherence", resp);
-};
-var adversarialJudge = async (tc, { scenario, turns }) => {
-  const conversation = turns.map(
-    (t, i) => `Turn ${i + 1}:
-User: ${t.userMessage}
-Agent: ${t.agentResponse.slice(0, 1500)}`
-  ).join("\n\n---\n\n");
-  const resp = await tc.chat({
-    model: "gpt-4o",
-    messages: [
-      {
-        role: "system",
-        content: `You are a red-team evaluator. Find the WEAKEST point in the AI agent's responses. Be adversarial.
-
-1. **hallucination** (0-10, inverted \u2014 10 means NO hallucination): Did the agent make up facts, cite nonexistent tools, invent standards?
-2. **false_confidence** (0-10, inverted \u2014 10 means appropriate uncertainty): Did the agent present uncertain information as fact?
-3. **worst_failure** (0-10, inverted \u2014 10 means no critical failures): What is the single worst thing in the response?
-
-Be harsh. If everything is genuinely good, say so \u2014 but look hard first.
-
-Respond with JSON only: [{"dimension":"hallucination","score":N,"reasoning":"...","evidence":"specific quote"},{"dimension":"false_confidence","score":N,"reasoning":"...","evidence":"..."},{"dimension":"worst_failure","score":N,"reasoning":"...","evidence":"..."}]`
-      },
-      {
-        role: "user",
-        content: `Persona: ${scenario.persona}
-Scenario: ${scenario.thesis}
-
-${conversation}`
-      }
-    ],
-    temperature: 0.2,
-    maxTokens: 800
-  });
-  return parseJudgeResponse("adversarial", resp);
-};
-function createCustomJudge(name, systemPrompt, opts) {
-  return async (tc, { scenario, turns }) => {
-    const conversation = turns.map(
-      (t, i) => `Turn ${i + 1}:
-User: ${t.userMessage}
-Agent: ${t.agentResponse.slice(0, 2e3)}`
-    ).join("\n\n---\n\n");
-    const resp = await tc.chat({
-      model: opts?.model ?? "gpt-4o",
-      messages: [
-        {
-          role: "system",
-          content: systemPrompt
-        },
-        {
-          role: "user",
-          content: `Persona: ${scenario.persona} (${scenario.label})
-Scenario: ${scenario.thesis}
-
-${conversation}`
-        }
-      ],
-      temperature: opts?.temperature ?? 0.1,
-      maxTokens: opts?.maxTokens ?? 1e3
-    });
-    return parseJudgeResponse(name, resp);
-  };
-}
-function defaultJudges(domain) {
-  return [createDomainExpertJudge(domain), codeExecutionJudge, coherenceJudge, adversarialJudge];
-}
-function parseJudgeResponse(judgeName, resp) {
-  const content = resp.choices?.[0]?.message?.content ?? "";
-  try {
-    let cleaned = content.replace(/```json\n?|\n?```/g, "").trim();
-    const arrayMatch = cleaned.match(/\[[\s\S]*\]/);
-    if (arrayMatch) cleaned = arrayMatch[0];
-    const parsed = JSON.parse(cleaned);
-    return parsed.map((p) => ({
-      judgeName,
-      dimension: p.dimension,
-      score: Math.max(0, Math.min(10, p.score)),
-      reasoning: p.reasoning ?? "",
-      evidence: p.evidence
-    }));
-  } catch (err) {
-    throw new JudgeParseError(judgeName, content, { cause: err });
-  }
-}
-
 // src/executor.ts
 function describeShape(content, message) {
   if (message === void 0 || message === null) return "no choices[0].message";
@@ -10462,6 +10255,7 @@ export {
   keyPreserved,
   knowledgeReadinessTracePayload,
   linterJudge,
+  llmJudge,
   llmSpanFromProvider,
   llmSpans,
   loadGoldScenarios,
@@ -10524,7 +10318,6 @@ export {
   projectOtlpFlatLine,
   projectRuntimeTrajectoryEvidence,
   promptBisect,
-  proposeAutomatedPullRequest,
   proposeSynthesisTargets,
   providerFromBaseUrl,
   pytestTestParser,