@lythos/skill-arena 0.9.11 → 0.9.13

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lythos/skill-arena",
-  "version": "0.9.11",
+  "version": "0.9.13",
   "description": "Skill Arena — benchmark skill effectiveness with controlled-variable comparison",
   "keywords": [
     "ai-agent",

package/src/comparative-judge.ts

@@ -1,4 +1,3 @@
-import { zodToJsonSchema } from 'zod-to-json-schema'
 import { ComparativeReport, ScoreCell, ParetoEntry } from '@lythos/test-utils/schema'
 import type { AgentAdapter } from '@lythos/test-utils/agents'
 import type { ArenaManifest } from '@lythos/test-utils/schema'
@@ -78,13 +77,29 @@ ${criteriaDesc}
 For each participant, score them 1-5 on each criterion. Provide a brief rationale.
 Score meanings: 1=poor, 3=acceptable, 5=excellent.
 
-Use the submit_scores tool to return your structured evaluation.`
-}
+## Output Schema
+Your response must conform to this Zod schema:
+\`\`\`ts
+z.object({
+  score_matrix: z.array(z.object({
+    participant_id: z.string(),
+    criterion: z.string(),
+    weight: z.number().min(0).max(1),
+    score: z.number().int().min(1).max(5),
+    rationale: z.string(),
+  })),
+  key_findings: z.array(z.string()),
+  recommendations: z.array(z.object({
+    audience: z.string(),
+    recommendation: z.string(),
+  })),
+})
+\`\`\`
+score_matrix is a FLAT ARRAY of objects — NOT nested by participant or criterion.
+weight: 0.25 for each cell (1 / num_criteria).
+score: 1=poor, 3=acceptable, 5=excellent.
 
-const SCORE_TOOL = {
-  name: 'submit_scores',
-  description: 'Submit per-participant scores for each criterion with rationales',
-  input_schema: zodToJsonSchema(ComparativeReport.pick({ score_matrix: true, key_findings: true, recommendations: true })) as Record<string, unknown>,
+Use the submit_scores tool to return your structured evaluation.`
 }
 
 function toScoreMatrix(
@@ -94,6 +109,112 @@ function toScoreMatrix(
   return scores.map(s => ScoreCell.parse(s))
 }
 
+// ── LLM Output Normalization (handle common schema mismatches) ─────────────
+
+interface NormalizedScoreCell {
+  participant_id: string
+  criterion: string
+  weight: number
+  score: number
+  rationale: string
+}
+
+function normalizeComparativeOutput(parsed: Record<string, unknown>): Record<string, unknown> {
+  const out = { ...parsed }
+
+  // Detect pivot-table format: { participant: { criterion: { score, rationale } } }
+  // Also handles flat format: { participant: { criterion: <score>, criterion_rationale: "..." } }
+  // Convert to expected score_matrix: [{ participant_id, criterion, score, weight, rationale }]
+  if (!Array.isArray(out.score_matrix)) {
+    const participants = Object.keys(out).filter(k => {
+      const v = out[k]
+      return v && typeof v === 'object' && !Array.isArray(v) && k !== 'key_findings' && k !== 'recommendations'
+    })
+    if (participants.length >= 2) {
+      const matrix: NormalizedScoreCell[] = []
+      for (const p of participants) {
+        const criteria = out[p] as Record<string, unknown>
+        // Collect criterion keys (exclude _rationale, _reason, _note suffixed keys)
+        const criterionKeys = Object.keys(criteria).filter(k =>
+          !k.endsWith('_rationale') && !k.endsWith('_reason') && !k.endsWith('_note') && !k.endsWith('_notes')
+        )
+        for (const criterion of criterionKeys) {
+          const rawScore = criteria[criterion]
+          const rationale = criteria[`${criterion}_rationale`] ?? criteria[`${criterion}_reason`] ?? criteria[`${criterion}_note`] ?? criteria[`${criterion}_notes`] ?? ''
+          let score = 3
+          if (typeof rawScore === 'number') score = rawScore
+          else if (typeof rawScore === 'string') {
+            const n = Number(rawScore)
+            if (!isNaN(n)) score = n
+            else {
+              // If it's a descriptive string (not a score), it might be the rationale
+              if (!rationale) criteria[`${criterion}_rationale`] = rawScore
+            }
+          } else if (typeof rawScore === 'object' && rawScore !== null) {
+            const obj = rawScore as Record<string, unknown>
+            score = typeof obj.score === 'number' ? obj.score : (typeof obj.score === 'string' ? Number(obj.score) || 3 : 3)
+          }
+          matrix.push({
+            participant_id: p,
+            criterion,
+            weight: 0.25,
+            score: Math.max(1, Math.min(5, Math.round(score))),
+            rationale: String(rationale).slice(0, 300),
+          })
+        }
+      }
+      if (matrix.length > 0) {
+        out.score_matrix = matrix
+        for (const p of participants) delete out[p]
+      }
+    }
+  }
+
+  // Normalize score_matrix entries
+  if (Array.isArray(out.score_matrix)) {
+    out.score_matrix = (out.score_matrix as Record<string, unknown>[]).map((cell): NormalizedScoreCell => {
+      const c = { ...cell }
+      // Map common field name variants
+      if (!c.participant_id && c.participantId) c.participant_id = c.participantId
+      if (!c.participant_id && c.side) c.participant_id = c.side
+      // Normalize score to number
+      if (typeof c.score === 'string') c.score = Number(c.score) || 3
+      // Normalize weight: if >1, assume percentage scale
+      if (typeof c.weight === 'number' && c.weight > 1) c.weight = c.weight / 100
+      if (c.weight === undefined) c.weight = 0.25
+      // Map rationale field name variants
+      if (!c.rationale && c.reason) c.rationale = c.reason
+      if (!c.rationale && c.notes) c.rationale = c.notes
+      if (!c.rationale && c.explanation) c.rationale = c.explanation
+      if (!c.rationale) c.rationale = ''
+
+      return {
+        participant_id: String(c.participant_id ?? 'unknown'),
+        criterion: String(c.criterion ?? 'unknown'),
+        weight: Number(c.weight),
+        score: Number(c.score),
+        rationale: String(c.rationale),
+      }
+    })
+  }
+
+  // Normalize recommendations
+  if (Array.isArray(out.recommendations)) {
+    out.recommendations = (out.recommendations as Record<string, unknown>[]).map(r => ({
+      audience: String(r.audience ?? r.role ?? 'general'),
+      recommendation: String(r.recommendation ?? r.text ?? r.advice ?? ''),
+    }))
+  }
+
+  // Ensure key_findings is an array of strings
+  if (!out.key_findings) out.key_findings = []
+  if (Array.isArray(out.key_findings)) {
+    out.key_findings = out.key_findings.map(f => String(f))
+  }
+
+  return out
+}
+
 // ── Comparative Judge ─────────────────────────────────────────────────────
 
 export async function runComparativeJudge(opts: {
@@ -112,28 +233,24 @@ export async function runComparativeJudge(opts: {
 
   for (let attempt = 0; attempt <= 2; attempt++) {
     try {
-      if (judge.invokeTool) {
-        parsed = await judge.invokeTool({
-          tool: SCORE_TOOL,
-          prompt,
-          cwd: workdir,
-          timeoutMs: 120000,
-        })
-        raw = JSON.stringify(parsed)
-      } else {
-        const result = await judge.spawn({ cwd: workdir, brief: prompt, timeoutMs: 120000 })
-        raw = result.stdout
-        const fenceMatch = raw.match(/```(?:json)?\s*([\s\S]*?)\s*```/)
-        const jsonStr = fenceMatch ? fenceMatch[1].trim() : raw.trim()
-        parsed = JSON.parse(jsonStr)
-      }
+      // Use spawn directly — prompt already includes Zod schema, invokeTool's
+      // redundant JSON Schema wrapper confuses the LLM.
+      const result = await judge.spawn({ cwd: workdir, brief: prompt, timeoutMs: 120000 })
+      raw = result.stdout
+      const fenceMatch = raw.match(/```(?:json)?\s*([\s\S]*?)\s*```/)
+      const jsonStr = fenceMatch ? fenceMatch[1].trim() : raw.trim()
+      if (!jsonStr) throw new Error('Empty LLM output')
+      parsed = JSON.parse(jsonStr)
+
+      // Normalize LLM output before Zod validation
+      const normalizedParsed = normalizeComparativeOutput(parsed as Record<string, unknown>)
 
       // Validate LLM output through Zod
       const llmResult = ComparativeReport.pick({
         score_matrix: true,
         key_findings: true,
         recommendations: true,
-      }).parse(parsed)
+      }).parse(normalizedParsed)
 
       // Success — proceed to Pareto computation
       const scoreMatrix = toScoreMatrix(manifest, llmResult.score_matrix)
@@ -166,15 +283,37 @@ export async function runComparativeJudge(opts: {
     }
   }
 
-  // All retries exhausted: return fallback report
-  const empty: typeof ComparativeReport._output = {
+  // All retries exhausted: build fallback report from per-cell verdicts
+  const scoreMatrix: typeof ScoreCell._output[] = []
+  for (const v of verdicts) {
+    const jv = v.verdict as Record<string, unknown> | null
+    const criteria = (Array.isArray(jv?.criteria) ? jv!.criteria : []) as { name?: string; passed?: boolean; note?: string }[]
+    for (const c of criteria) {
+      scoreMatrix.push(ScoreCell.parse({
+        participant_id: v.participantId,
+        criterion: c.name ?? 'unknown',
+        weight: 1 / (manifest.criteria.length || 1),
+        score: c.passed ? 5 : 1,
+        rationale: c.note ?? (c.passed ? 'PASS' : 'FAIL'),
+      }))
+    }
+  }
+  const participantScores = manifest.participants.map(p => {
+    const pScores: Record<string, number> = {}
+    for (const cell of scoreMatrix) {
+      if (cell.participant_id === p.id) pScores[cell.criterion] = cell.score
+    }
+    return { participant_id: p.id, scores: pScores }
+  })
+  const pareto = computePareto(participantScores)
+
+  return ComparativeReport.parse({
     arena_id: manifest.id,
     generated_at: new Date().toISOString(),
-    score_matrix: [],
+    score_matrix: scoreMatrix,
     weighted_totals: {},
-    pareto: [],
-    key_findings: [`Comparative judge failed after 3 attempts: ${lastError}`],
+    pareto,
+    key_findings: [`Comparative judge unavailable; scores derived from per-cell verdicts. Last error: ${lastError}`],
     recommendations: [],
-  }
-  return empty
+  })
 }

package/src/runner.ts

@@ -103,12 +103,21 @@ export async function runArenaFromToml(opts: {
     try {
       const agent = useAgent(resolvePlayer(cell.player))
       const result = await runAgentScenario({
-        scenarioPath: resolve(taskPath),
+        scenarioPath: taskAbs,
         agent,
-        setupWorkdir(_scenario: AgentScenario, workdir: string) {
+        async setupWorkdir(_scenario: AgentScenario, workdir: string) {
           mkdirSync(workdir, { recursive: true })
-          const deckContent = readFileSync(resolve(cell.deck), 'utf-8')
+          const deckContent = readFileSync(cell.deck, 'utf-8')
           writeFileSync(join(workdir, 'skill-deck.toml'), deckContent)
+
+          // Link skills into .claude/skills/ so claude -p can discover them
+          const deckCli = resolve(import.meta.dir, '..', '..', 'lythoskill-deck', 'src', 'cli.ts')
+          const linkProc = Bun.spawn(['bun', 'run', deckCli, 'link'], {
+            cwd: workdir,
+            env: { ...process.env, HOME: process.env.HOME },
+          })
+          await linkProc.exited
+          log?.(`[arena] deck link for ${cell.side}: exit ${linkProc.exitCode}`)
         },
         baseDir: join(artifactsDir, 'runs', cell.side),
       })