elasticdash-test 0.1.20 → 0.1.21-alpha-2

package/src/ci/benchmark.ts

@@ -8,6 +8,7 @@
  */
 
 import { callProviderLLM } from '../matchers/index.js'
+import { prepareOutputForJudge } from '../core/judge-utils.js'
 import type { TestMeasurement } from './measurement.js'
 import type { TestBenchmarks } from './test-registry.js'
 import type { EvaluatorConfig } from './api-client.js'
@@ -34,6 +35,32 @@ const PROVIDER_NAME_MAP: Record<string, string> = {
   moonshot: 'kimi',
 }
 
+/** Default model for each provider, used when no explicit model is set or
+ *  when the evaluator config model doesn't belong to the resolved provider. */
+const DEFAULT_PROVIDER_MODELS: Record<string, string> = {
+  openai: 'gpt-4o',
+  claude: 'claude-sonnet-4-20250514',
+  gemini: 'gemini-2.0-flash',
+  grok: 'grok-3',
+  kimi: 'moonshot-v1-auto',
+}
+
+/** Known model prefixes per provider — used to check if a model belongs to a provider. */
+const PROVIDER_MODEL_PREFIXES: Record<string, string[]> = {
+  openai: ['gpt-', 'o1-', 'o3-', 'o4-', 'chatgpt-'],
+  claude: ['claude-'],
+  gemini: ['gemini-'],
+  grok: ['grok-'],
+  kimi: ['moonshot-'],
+}
+
+/** Check if a model name belongs to the given provider. */
+function isModelForProvider(model: string, provider: string): boolean {
+  const prefixes = PROVIDER_MODEL_PREFIXES[provider]
+  if (!prefixes) return false
+  return prefixes.some(p => model.toLowerCase().startsWith(p))
+}
+
 /** Normalize provider name from backend format to SDK format. */
 function normalizeSdkProvider(provider: string): string {
   return PROVIDER_NAME_MAP[provider] ?? provider
@@ -125,7 +152,17 @@ export async function compareBenchmarks(
     const resolvedProvider = normalizeSdkProvider(
       judge.judge_provider ?? evaluatorConfig?.provider ?? 'openai'
     )
-    const resolvedModel = judge.judge_model ?? evaluatorConfig?.model ?? undefined
+    // Model resolution: judge_model > evaluatorConfig.model (if compatible) > provider default
+    let resolvedModel = judge.judge_model ?? undefined
+    if (!resolvedModel && evaluatorConfig?.model) {
+      // Only use the evaluator config model if it belongs to the resolved provider
+      if (isModelForProvider(evaluatorConfig.model, resolvedProvider)) {
+        resolvedModel = evaluatorConfig.model
+      }
+    }
+    if (!resolvedModel) {
+      resolvedModel = DEFAULT_PROVIDER_MODELS[resolvedProvider]
+    }
 
     // If the backend provided an API key and we're using its provider,
     // set it in the environment so callProviderLLM can pick it up.
@@ -149,7 +186,8 @@ export async function compareBenchmarks(
 
     try {
       console.log(`  [llm_judge] provider=${resolvedProvider}, model=${resolvedModel ?? '(default)'}`)
-      const evalPrompt = `${judge.judge_prompt}\n\nOutput to evaluate:\n${outputStr}\n\nScore this output on a scale of 0-10. Respond with only the number.`
+      const preparedOutput = prepareOutputForJudge(outputStr, judge.judge_prompt)
+      const evalPrompt = `${judge.judge_prompt}\n\n<output>\n${preparedOutput}\n</output>\n\nBased on the evaluation criteria above, score this output on a scale of 0-10. Respond with only the number.`
 
       const result = await callProviderLLM(
         evalPrompt,

package/src/ci/executor.ts

@@ -1,6 +1,7 @@
 import { executePortalTask, checkToolAvailability, checkAIAvailability } from '../portal-executor.js'
 import { scanTools } from '../execution/tool-runner.js'
 import { callProviderLLM } from '../matchers/index.js'
+import { prepareOutputForJudge } from '../core/judge-utils.js'
 import type { ToolInfo } from '../execution/tool-runner.js'
 import type {
   APITestGroupTest,
@@ -591,12 +592,14 @@ async function evaluateLLMJudge(exp: APIExpectation, runs: CISingleRunResult[]):
       ? run.output
       : JSON.stringify(run.output ?? '')
 
+    const preparedOutput = prepareOutputForJudge(outputStr, exp.judge_prompt)
     const evalPrompt = `${exp.judge_prompt}
 
-Output to evaluate:
-${outputStr}
+<output>
+${preparedOutput}
+</output>
 
-Score this output on a scale of 0-10. Respond with only the number.`
+Based on the evaluation criteria above, score this output on a scale of 0-10. Respond with only the number.`
 
     try {
       const provider = (exp.judge_provider as 'openai' | 'claude' | 'gemini' | 'grok' | 'kimi') || 'openai'

package/src/core/judge-utils.ts

@@ -0,0 +1,232 @@
+/**
+ * judge-utils.ts
+ *
+ * Utilities for preprocessing outputs before sending to LLM-as-a-judge evaluators.
+ * Addresses two problems:
+ * 1. Large outputs cause slow inference
+ * 2. LLMs miss attributes in large JSON payloads due to attention degradation
+ */
+
+const DEFAULT_MAX_CHARS = 8000
+const MAX_STRING_VALUE_LENGTH = 500
+const MAX_ARRAY_EDGE_ITEMS = 5
+
+/**
+ * Prepare an output string for LLM judge evaluation.
+ *
+ * - Small outputs (< maxChars) pass through unchanged.
+ * - Large JSON outputs: extracts subtrees whose keys match keywords from the judge prompt.
+ * - Large non-JSON outputs: truncates with head + tail and a marker in between.
+ *
+ * @param output     - The raw output string to prepare.
+ * @param judgePrompt - The judge/evaluation prompt, used to identify relevant JSON keys.
+ * @param maxChars   - Maximum character budget for the prepared output. Default 8000.
+ * @returns The prepared output string, possibly trimmed.
+ */
+export function prepareOutputForJudge(
+  output: string,
+  judgePrompt: string,
+  maxChars: number = DEFAULT_MAX_CHARS,
+): string {
+  if (!output) return output
+
+  // Always attempt JSON-aware processing (value truncation + key extraction).
+  // Even outputs under maxChars can contain arrays with 100+ items that
+  // drown out the fields the judge actually needs to evaluate.
+  const jsonResult = tryJsonExtract(output, judgePrompt, maxChars)
+  if (jsonResult !== null) return jsonResult
+
+  // Non-JSON: only truncate if over budget
+  if (output.length <= maxChars) return output
+  return truncateHeadTail(output, maxChars)
+}
+
+/**
+ * Attempt to parse the output as JSON and extract only the subtrees
+ * whose keys are relevant to the judge prompt.
+ * Returns null if the output is not valid JSON.
+ */
+function tryJsonExtract(output: string, judgePrompt: string, maxChars: number): string | null {
+  let parsed: unknown
+  try {
+    parsed = JSON.parse(output)
+  } catch {
+    return null
+  }
+
+  if (typeof parsed !== 'object' || parsed === null) return null
+
+  const keywords = extractKeywords(judgePrompt)
+  if (keywords.length === 0) {
+    // No keywords to match — truncate large values then serialize
+    const trimmed = truncateJsonValues(parsed)
+    const trimmedStr = JSON.stringify(trimmed, null, 2)
+    if (trimmedStr.length <= maxChars) return trimmedStr
+    return truncateHeadTail(trimmedStr, maxChars)
+  }
+
+  const extracted = extractRelevantPaths(parsed, keywords)
+
+  // If extraction found relevant content, truncate large values then serialize
+  if (extracted !== undefined && Object.keys(extracted as Record<string, unknown>).length > 0) {
+    const trimmed = truncateJsonValues(extracted)
+    const extractedStr = JSON.stringify(trimmed, null, 2)
+
+    if (extractedStr.length <= maxChars) {
+      const omittedKeys = countOmittedKeys(parsed, extracted)
+      if (omittedKeys > 0) {
+        return `${extractedStr}\n\n[Note: ${omittedKeys} irrelevant key(s) omitted from original output for brevity. Total original size: ${output.length} chars.]`
+      }
+      return extractedStr
+    }
+
+    // Extracted content is still too large — truncate it
+    return truncateHeadTail(extractedStr, maxChars)
+  }
+
+  // Extraction found nothing relevant — truncate values in full object
+  const trimmedFull = truncateJsonValues(parsed)
+  const trimmedFullStr = JSON.stringify(trimmedFull, null, 2)
+  if (trimmedFullStr.length <= maxChars) return trimmedFullStr
+  return truncateHeadTail(trimmedFullStr, maxChars)
+}
+
+/**
+ * Extract lowercase keywords from the judge prompt.
+ * Filters out common stop words and short tokens.
+ */
+function extractKeywords(prompt: string): string[] {
+  const stopWords = new Set([
+    'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+    'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could',
+    'should', 'may', 'might', 'shall', 'can', 'need', 'must',
+    'and', 'or', 'but', 'if', 'then', 'else', 'when', 'where', 'how',
+    'what', 'which', 'who', 'whom', 'this', 'that', 'these', 'those',
+    'it', 'its', 'of', 'in', 'on', 'at', 'to', 'for', 'with', 'by',
+    'from', 'as', 'into', 'about', 'between', 'through', 'after', 'before',
+    'not', 'no', 'nor', 'only', 'also', 'just', 'more', 'most', 'very',
+    'all', 'each', 'every', 'any', 'some', 'such', 'than', 'too',
+    'output', 'evaluate', 'score', 'check', 'whether', 'contains',
+    'following', 'given', 'based', 'respond', 'number', 'scale',
+    'text', 'result', 'response', 'answer', 'return', 'value',
+  ])
+
+  const words = prompt
+    .toLowerCase()
+    .replace(/[^a-z0-9_\s-]/g, ' ')
+    .split(/\s+/)
+    .filter(w => w.length > 2 && !stopWords.has(w))
+
+  return [...new Set(words)]
+}
+
+/**
+ * Recursively extract object entries whose keys match any of the keywords.
+ * For arrays, preserves items that contain matching keys.
+ */
+function extractRelevantPaths(obj: unknown, keywords: string[]): unknown {
+  if (Array.isArray(obj)) {
+    // For arrays: keep items that have relevant keys, limit to first few items
+    const relevant = obj
+      .map(item => extractRelevantPaths(item, keywords))
+      .filter(item => item !== undefined)
+    if (relevant.length === 0) return undefined
+    return relevant
+  }
+
+  if (typeof obj === 'object' && obj !== null) {
+    const result: Record<string, unknown> = {}
+    let hasMatch = false
+
+    for (const [key, value] of Object.entries(obj)) {
+      const keyLower = key.toLowerCase()
+      const keyMatchesDirectly = keywords.some(kw =>
+        keyLower.includes(kw) || kw.includes(keyLower)
+      )
+
+      if (keyMatchesDirectly) {
+        result[key] = value
+        hasMatch = true
+      } else if (typeof value === 'object' && value !== null) {
+        // Recurse into nested objects/arrays
+        const nested = extractRelevantPaths(value, keywords)
+        if (nested !== undefined) {
+          result[key] = nested
+          hasMatch = true
+        }
+      }
+    }
+
+    return hasMatch ? result : undefined
+  }
+
+  return undefined
+}
+
+/**
+ * Recursively truncate large values inside a JSON structure.
+ * - Long strings: keep first/last portions with a marker in between.
+ * - Long arrays: keep first 5 and last 5 items, skip the rest with a marker.
+ */
+function truncateJsonValues(obj: unknown): unknown {
+  if (obj === null || obj === undefined) return obj
+
+  if (typeof obj === 'string') {
+    if (obj.length <= MAX_STRING_VALUE_LENGTH) return obj
+    const headLen = Math.floor(MAX_STRING_VALUE_LENGTH * 0.6)
+    const tailLen = MAX_STRING_VALUE_LENGTH - headLen
+    return `${obj.slice(0, headLen)}...[${obj.length - headLen - tailLen} chars truncated]...${obj.slice(obj.length - tailLen)}`
+  }
+
+  if (Array.isArray(obj)) {
+    if (obj.length <= MAX_ARRAY_EDGE_ITEMS * 2) {
+      return obj.map(item => truncateJsonValues(item))
+    }
+    const head = obj.slice(0, MAX_ARRAY_EDGE_ITEMS).map(item => truncateJsonValues(item))
+    const tail = obj.slice(obj.length - MAX_ARRAY_EDGE_ITEMS).map(item => truncateJsonValues(item))
+    const skipped = obj.length - MAX_ARRAY_EDGE_ITEMS * 2
+    return [...head, `[...${skipped} items skipped...]`, ...tail]
+  }
+
+  if (typeof obj === 'object') {
+    const result: Record<string, unknown> = {}
+    for (const [key, value] of Object.entries(obj)) {
+      result[key] = truncateJsonValues(value)
+    }
+    return result
+  }
+
+  return obj
+}
+
+/**
+ * Count how many top-level keys from the original object are not in the extracted object.
+ */
+function countOmittedKeys(original: unknown, extracted: unknown): number {
+  if (typeof original !== 'object' || original === null) return 0
+  if (typeof extracted !== 'object' || extracted === null) return 0
+  if (Array.isArray(original)) return 0
+
+  const origKeys = Object.keys(original)
+  const extKeys = new Set(Object.keys(extracted as Record<string, unknown>))
+  return origKeys.filter(k => !extKeys.has(k)).length
+}
+
+/**
+ * Truncate a string by keeping the head and tail portions,
+ * inserting a marker in the middle.
+ */
+function truncateHeadTail(text: string, maxChars: number): string {
+  if (text.length <= maxChars) return text
+
+  // Reserve space for the marker
+  const marker = `\n\n[...truncated ${text.length - maxChars} chars out of ${text.length} total...]\n\n`
+  const available = maxChars - marker.length
+  if (available <= 0) return text.slice(0, maxChars)
+
+  // 70% head, 30% tail — head is usually more important
+  const headSize = Math.floor(available * 0.7)
+  const tailSize = available - headSize
+
+  return text.slice(0, headSize) + marker + text.slice(text.length - tailSize)
+}

package/src/matchers/index.ts

@@ -1,5 +1,6 @@
 import { expect } from 'expect'
 import type { TraceHandle, LLMStep, CustomStep, CustomStepKind } from '../trace-adapter/context.js'
+import { prepareOutputForJudge } from '../core/judge-utils.js'
 
 interface LLMStepConfig {
   model?: string
@@ -186,7 +187,7 @@ export async function callProviderLLM(
         method: 'POST',
         headers: {
           'x-api-key': apiKey,
-          'anthropic-version': '2025-01-01',
+          'anthropic-version': '2023-06-01',
           'content-type': 'application/json',
         },
         body: JSON.stringify({
@@ -595,14 +596,16 @@ Use: expect(ctx.trace).toEvaluateOutputMetric(...)`,
         }
       }
 
+      const preparedText = prepareOutputForJudge(targetText, config.evaluationPrompt)
       const evalPrompt = `
 Evaluation prompt (from user):
 ${config.evaluationPrompt}
 
 Score the following text strictly between 0 and 1 (inclusive). Respond with only the number.
 
-Text:
-${targetText}
+<output>
+${preparedText}
+</output>
       `.trim()
 
       try {