@tangle-network/starter-foundry 0.7.3 → 0.13.2

package/registry/layers/agent-eval/trace-multi-turn/files/src/eval/trace/README.md

@@ -0,0 +1,75 @@
+# Multi-turn trajectory eval
+
+Per-turn scoring for multi-turn agent rollouts. Composes
+`@tangle-network/agent-eval`'s `Trajectory` + `TrajectoryStep` +
+`StepRubric` + `buildTrajectory` + `gradeSemanticStatus`.
+
+## Why per-turn
+
+Outcome-only eval ("did the final artifact pass?") gives a single bit of
+signal across N turns. Most agent regressions don't change the final
+verdict — they change WHERE the agent gets there:
+
+- "Pass on turn 4 instead of turn 2" — silent latency regression.
+- "Pass after a tool error on turn 3" — silent reliability regression.
+- "Pass with 8 retries instead of 1" — silent cost regression.
+
+Per-turn scoring surfaces these as `convergenceTurn` deltas and per-turn
+score deltas. CI gates on the deltas, not just the terminal pass/fail.
+
+## When to use
+
+- Multi-turn conversational agents (chat, copilot, support).
+- Tool-using agents that iterate (research, code review, planning).
+- ANY agent where the failure mode "took longer than it should have" is
+  meaningful — which is most production agents.
+
+## When NOT to use
+
+- Single-shot completions (use `agent-eval:judge-rubric` instead).
+- Agents that never emit intermediate spans (the trajectory will be empty).
+
+## API
+
+```ts
+import { runMultiTurnRollout, groupStepsByTurn } from './trajectory-runner.js'
+import { scorePerTurn } from './per-turn-scorer.js'
+import { defaultStepRubrics } from './example-rubric.js'
+
+const rollout = await runMultiTurnRollout({
+  scenario: { scenarioId: 's1', userTurns: ['hi', 'now do X'] },
+  driver,    // your TurnDriver — emits spans into `emitter`
+  store,     // TraceStore (FileSystem / InMemory / D1)
+  emitter,   // TraceEmitter wired to `store`
+})
+const stepsByTurn = groupStepsByTurn(rollout.trajectory, rollout.transcript.length / 2)
+const report = await scorePerTurn({
+  scenarioId: rollout.scenarioId,
+  runId: rollout.runId,
+  trajectory: rollout.trajectory,
+  stepsByTurn,
+  rubrics: defaultStepRubrics('book a flight to Tokyo'),
+  threshold: 0.7,
+})
+console.log(`converged at turn ${report.convergenceTurn ?? 'never'}`)
+```
+
+## Composes with
+
+| Capability                | Source                                         |
+| ------------------------- | ---------------------------------------------- |
+| `eval:scenarios`          | `agent-eval:scenarios` (Worker 1)              |
+| `eval:judge-rubric`       | `agent-eval:judge-rubric` (Worker 1)           |
+| `eval:trace-multi-turn`   | this layer                                     |
+
+The rubrics here can be combined with single-turn `JudgeFn`s — pass the
+agg result of `judgeRubric` as a custom `StepRubric` to weight per-turn
+LLM judgments alongside the deterministic ones.
+
+## Authoring custom rubrics
+
+A `StepRubric` returns `{ score: 0..1, rationale?, evidence? }` per
+matching span. See `example-rubric.ts` for two canonical patterns —
+deterministic token overlap (cheap, runs at every step) and
+goal-progress (uses scenario success criteria). Domain rubrics (e.g.
+"tool args reference real entities") follow the same contract.

package/registry/layers/agent-eval/trace-multi-turn/files/src/eval/trace/example-rubric.ts

@@ -0,0 +1,105 @@
+/**
+ * Example StepRubric set for multi-turn agent eval.
+ *
+ * Composes built-in agent-eval rubrics with two custom step-level scorers:
+ *   - `relevanceRubric`     — LLM span output should reference the user's
+ *                             current turn (cheap keyword overlap proxy).
+ *   - `progressTowardGoalRubric` — assistant outputs should monotonically
+ *                                  reduce the gap between the user's stated
+ *                                  goal and the run's intermediate state.
+ *
+ * Customize for your domain. The built-ins below are intentionally generic
+ * so the layer composes cleanly without domain assumptions.
+ */
+
+import {
+  outputLengthRubric,
+  toolSuccessRubric,
+  toolNonRedundantRubric,
+  toolIntentAlignmentRubric,
+  type StepRubric,
+} from '@tangle-network/agent-eval'
+
+const STOPWORDS = new Set([
+  'the', 'a', 'an', 'and', 'or', 'but', 'is', 'are', 'was', 'were', 'be', 'been',
+  'to', 'of', 'in', 'on', 'at', 'for', 'with', 'as', 'by', 'this', 'that',
+  'it', 'its', 'i', 'you', 'we', 'they', 'he', 'she', 'them', 'us', 'me',
+])
+
+const tokenize = (s: string): Set<string> => {
+  return new Set(
+    s
+      .toLowerCase()
+      .replace(/[^a-z0-9\s]/g, ' ')
+      .split(/\s+/)
+      .filter((t) => t.length >= 3 && !STOPWORDS.has(t)),
+  )
+}
+
+const jaccard = (a: Set<string>, b: Set<string>): number => {
+  if (a.size === 0 || b.size === 0) return 0
+  let inter = 0
+  for (const t of a) if (b.has(t)) inter += 1
+  return inter / (a.size + b.size - inter)
+}
+
+/**
+ * Reward LLM responses whose token set overlaps the most recent user turn.
+ * Cheap, deterministic, no LLM call. Pairs with stricter LLM-judged rubrics.
+ */
+export const relevanceRubric = (args: { weight?: number } = {}): StepRubric => ({
+  id: 'relevance',
+  kinds: ['llm'],
+  weight: args.weight ?? 1,
+  grade: async (ctx) => {
+    const span = ctx.step.span
+    if (span.kind !== 'llm') return null
+    const lastUser = [...span.messages].reverse().find((m) => m.role === 'user')
+    if (!lastUser?.content) return null
+    const output = span.output ?? ''
+    if (!output) return null
+    const score = jaccard(tokenize(lastUser.content), tokenize(output))
+    return {
+      score,
+      rationale: `jaccard=${score.toFixed(3)} between user turn and assistant output`,
+    }
+  },
+})
+
+/**
+ * Reward assistant outputs that reference the goal phrase (declared in the
+ * scenario's first user message). A flat or decreasing referenceability
+ * score across turns indicates topical drift.
+ */
+export const progressTowardGoalRubric = (args: { goalPhrase: string; weight?: number }): StepRubric => {
+  const goalTokens = tokenize(args.goalPhrase)
+  return {
+    id: 'progress-toward-goal',
+    kinds: ['llm'],
+    weight: args.weight ?? 1,
+    grade: async (ctx) => {
+      const span = ctx.step.span
+      if (span.kind !== 'llm') return null
+      const output = span.output ?? ''
+      if (!output) return null
+      const overlap = jaccard(goalTokens, tokenize(output))
+      return {
+        score: overlap,
+        rationale: `goal-token overlap=${overlap.toFixed(3)}`,
+      }
+    },
+  }
+}
+
+/**
+ * Default rubric set — composes built-ins with relevance + goal progress.
+ * Pass a `goalPhrase` from the scenario's success criteria.
+ */
+export const defaultStepRubrics = (goalPhrase: string): StepRubric[] => [
+  outputLengthRubric({ minChars: 16, maxChars: 4000, weight: 0.5 }),
+  toolSuccessRubric({ weight: 1 }),
+  toolNonRedundantRubric({ weight: 1 }),
+  toolIntentAlignmentRubric({ weight: 1 }),
+  relevanceRubric({ weight: 1 }),
+  progressTowardGoalRubric({ goalPhrase, weight: 1.5 }),
+]

package/registry/layers/agent-eval/trace-multi-turn/files/src/eval/trace/per-turn-scorer.ts

@@ -0,0 +1,230 @@
+/**
+ * Per-turn scorer — apply `StepRubric`s on a turn-by-turn basis and surface
+ * exactly WHERE multi-turn quality degrades.
+ *
+ * Outcome-only eval (did the final artifact pass?) hides degradation modes
+ * common to multi-turn agents:
+ *   - "smart on turn 1, drifts by turn 5"
+ *   - "regains coherence after a tool error in turn 3"
+ *   - "good response, but only after retrying the same tool 4 times"
+ *
+ * This module composes `StepRubric` (the agent-eval primitive for span-level
+ * scoring) into per-turn aggregates and emits a `convergenceTurn` (first
+ * turn whose aggregate >= threshold) for regression-friendly comparisons.
+ */
+
+import {
+  gradeSemanticStatus,
+  type LayerStatus,
+  type Severity,
+  type Trajectory,
+  type TrajectoryStep,
+} from '@tangle-network/agent-eval'
+
+// StepRubric / StepContext are no longer re-exported from the top-level
+// barrel as of agent-eval 0.30.0 (PrmGrader is the only public surface).
+// Mirror the shapes here so this layer template stays self-contained and
+// can be shipped into consumer apps without depending on internal modules.
+export interface StepContext {
+  trajectory: Trajectory
+  step: TrajectoryStep
+  prior: TrajectoryStep[]
+  next: TrajectoryStep[]
+}
+export interface StepRubric {
+  id: string
+  kinds?: Array<TrajectoryStep['span']['kind']>
+  weight?: number
+  grade: (ctx: StepContext) => Promise<{
+    score: number
+    rationale?: string
+    evidence?: string
+  } | null>
+}
+
+export type PerTurnStatus = 'measured' | 'unmeasured'
+
+export interface PerTurnScore {
+  /** 1-indexed turn number, parallel to MultiTurnScenario.userTurns. */
+  turn: number
+  /**
+   * Weighted-mean score in 0..1 across all rubrics that fired this turn.
+   * `null` when no rubric matched any span on this turn (`status: 'unmeasured'`).
+   * NEVER 0 for unmeasured turns — that conflates "rubrics ran and scored
+   * everything as failing" with "no rubrics ran" and inflates 'critical'
+   * findings via severity grading.
+   */
+  aggregateScore: number | null
+  /**
+   * `'measured'` if at least one rubric verdict landed; `'unmeasured'` if
+   * the turn had zero matching/non-null rubric verdicts. Cumulative score
+   * + convergence skip `unmeasured` turns; gates refuse to grade them.
+   */
+  status: PerTurnStatus
+  /** Number of spans rubrics actually graded (vs spans that all rubrics skipped). */
+  gradedCount: number
+  /** Per-rubric breakdown for forensic drill-down. */
+  byRubric: Array<{
+    rubricId: string
+    score: number
+    weight: number
+    rationale?: string
+  }>
+}
+
+export interface MultiTurnScoreReport {
+  scenarioId: string
+  runId: string
+  perTurn: PerTurnScore[]
+  /** Overall (weighted-mean across all turns). */
+  cumulativeScore: number
+  /**
+   * First 1-indexed turn whose `aggregateScore >= threshold`. `null` if no
+   * turn met the threshold. This is the regression-friendly headline metric:
+   * if convergence drops from turn-2 to turn-4 between runs, something
+   * upstream of turn-3 changed.
+   */
+  convergenceTurn: number | null
+  /** SemanticStatus verdict — feeds gradeSemanticStatus so the layer can
+   *  participate in MultiLayerVerifier without re-implementing thresholding. */
+  status: LayerStatus
+}
+
+export interface PerTurnScoreInput {
+  scenarioId: string
+  runId: string
+  trajectory: Trajectory
+  /** Steps grouped by 1-indexed user turn (output of groupStepsByTurn). */
+  stepsByTurn: TrajectoryStep[][]
+  rubrics: StepRubric[]
+  /** Per-turn score required to count as "converged". Default 0.7. */
+  threshold?: number
+}
+
+const DEFAULT_THRESHOLD = 0.7
+
+const severityFromScore = (score: number, threshold: number): Severity => {
+  if (score >= threshold) return 'minor'
+  if (score >= threshold - 0.2) return 'major'
+  return 'critical'
+}
+
+const rubricMatchesStep = (rubric: StepRubric, step: TrajectoryStep): boolean => {
+  if (!rubric.kinds || rubric.kinds.length === 0) return true
+  return rubric.kinds.includes(step.span.kind)
+}
+
+const buildContext = (
+  trajectory: Trajectory,
+  stepIndex: number,
+): StepContext => ({
+  trajectory,
+  step: trajectory.steps[stepIndex],
+  prior: trajectory.steps.slice(0, stepIndex),
+  next: trajectory.steps.slice(stepIndex + 1),
+})
+
+export const scorePerTurn = async (
+  input: PerTurnScoreInput,
+): Promise<MultiTurnScoreReport> => {
+  const threshold = input.threshold ?? DEFAULT_THRESHOLD
+  const perTurn: PerTurnScore[] = []
+
+  // Map span.spanId → step.index in the full trajectory for context lookup.
+  const indexBySpanId = new Map<string, number>()
+  for (const step of input.trajectory.steps) {
+    indexBySpanId.set(step.span.spanId, step.index)
+  }
+
+  for (let t = 0; t < input.stepsByTurn.length; t += 1) {
+    const turnSteps = input.stepsByTurn[t]
+    let weightedSum = 0
+    let weightTotal = 0
+    let gradedCount = 0
+    const rubricAggregates = new Map<
+      string,
+      { sum: number; weight: number; count: number; lastRationale?: string }
+    >()
+
+    for (const step of turnSteps) {
+      const fullIndex = indexBySpanId.get(step.span.spanId)
+      if (fullIndex === undefined) continue
+      const ctx = buildContext(input.trajectory, fullIndex)
+      for (const rubric of input.rubrics) {
+        if (!rubricMatchesStep(rubric, step)) continue
+        const verdict = await rubric.grade(ctx)
+        if (verdict === null) continue
+        const w = rubric.weight ?? 1
+        weightedSum += verdict.score * w
+        weightTotal += w
+        gradedCount += 1
+        const agg = rubricAggregates.get(rubric.id) ?? { sum: 0, weight: 0, count: 0 }
+        agg.sum += verdict.score * w
+        agg.weight += w
+        agg.count += 1
+        agg.lastRationale = verdict.rationale
+        rubricAggregates.set(rubric.id, agg)
+      }
+    }
+
+    const measured = weightTotal > 0
+    const aggregateScore: number | null = measured ? weightedSum / weightTotal : null
+    const byRubric = [...rubricAggregates.entries()].map(([rubricId, agg]) => ({
+      rubricId,
+      score: agg.weight > 0 ? agg.sum / agg.weight : 0, // muffle-ok: rubricAggregates only populated when a rubric verdict landed (weight>0 by construction); the unmeasured signal lives on PerTurnScore.status
+      weight: agg.weight,
+      rationale: agg.lastRationale,
+    }))
+
+    perTurn.push({
+      turn: t + 1,
+      aggregateScore,
+      status: measured ? 'measured' : 'unmeasured',
+      gradedCount,
+      byRubric,
+    })
+  }
+
+  // Cumulative aggregate weights ONLY measured turns. Unmeasured turns
+  // contribute neither score nor weight; counting them as zero (the prior
+  // behaviour) was a muffled-gate that fabricated 'critical' findings for
+  // turns that simply had no matching rubric.
+  let totalWeighted = 0
+  let totalWeight = 0
+  for (const turn of perTurn) {
+    if (turn.status !== 'measured' || turn.aggregateScore === null) continue
+    const w = Math.max(turn.gradedCount, 1)
+    totalWeighted += turn.aggregateScore * w
+    totalWeight += w
+  }
+  const cumulativeScore = totalWeight > 0 ? totalWeighted / totalWeight : 0 // muffle-ok: all-unmeasured case is disambiguated downstream — gradeSemanticStatus is given `available: false` (perTurn lacks any measured turn), so the returned `status` carries the "no measurement" signal even though cumulativeScore reads 0
+
+  const convergenceTurn =
+    perTurn.find((t) => t.status === 'measured' && t.aggregateScore !== null && t.aggregateScore >= threshold)
+      ?.turn ?? null
+
+  const status = gradeSemanticStatus({
+    score: cumulativeScore,
+    // Only feed measured turns into severity grading — unmeasured turns
+    // would otherwise be marked `severity: 'critical'` via score=0 even
+    // though no rubric ever fired on them.
+    findings: perTurn
+      .filter((t) => t.status === 'measured' && t.aggregateScore !== null)
+      .map((t) => ({
+        severity: severityFromScore(t.aggregateScore as number, threshold),
+        score: t.aggregateScore as number,
+        present: t.gradedCount > 0,
+      })),
+    available: perTurn.some((t) => t.status === 'measured'),
+    threshold,
+  })
+
+  return {
+    scenarioId: input.scenarioId,
+    runId: input.runId,
+    perTurn,
+    cumulativeScore,
+    convergenceTurn,
+    status,
+  }
+}

package/registry/layers/agent-eval/trace-multi-turn/files/src/eval/trace/trajectory-runner.ts

@@ -0,0 +1,142 @@
+/**
+ * Trajectory runner — drive a multi-turn rollout against an agent endpoint
+ * and produce a per-turn `Trajectory` consumable by `StepRubric`s.
+ *
+ * The runner is transport-agnostic: callers pass a `TurnDriver` that takes
+ * the current message + transcript and returns the agent's response plus
+ * the spans emitted during the turn. The trace store persists every span
+ * so `buildTrajectory` (from agent-eval) can produce the canonical
+ * `Trajectory` shape downstream.
+ */
+
+import {
+  buildTrajectory,
+  type TraceEmitter,
+  type TraceStore,
+  type Trajectory,
+  type TrajectoryStep,
+} from '@tangle-network/agent-eval'
+
+export interface TurnDriverInput {
+  /** Conversation history including the user message just sent. */
+  transcript: ConversationMessage[]
+  /** Run id this turn belongs to. Used for span correlation. */
+  runId: string
+  /** Trace emitter — driver MUST emit spans for every llm/tool/judge action. */
+  emitter: TraceEmitter
+}
+
+export interface TurnDriverResult {
+  /** Assistant response for this turn. */
+  assistantText: string
+  /** Optional structured tool calls the agent made (names only). */
+  toolCalls?: string[]
+}
+
+export type TurnDriver = (input: TurnDriverInput) => Promise<TurnDriverResult>
+
+export interface ConversationMessage {
+  role: 'system' | 'user' | 'assistant' | 'tool'
+  content: string
+}
+
+export interface MultiTurnScenario {
+  scenarioId: string
+  /** Initial system prompt. */
+  systemPrompt?: string
+  /** Ordered user messages. The runner sends each, capturing per-turn spans. */
+  userTurns: string[]
+  /** Hard cap on turns to drive (defaults to userTurns.length). */
+  maxTurns?: number
+  /** Per-turn timeout. */
+  timeoutMs?: number
+}
+
+export interface MultiTurnRolloutResult {
+  scenarioId: string
+  runId: string
+  trajectory: Trajectory
+  transcript: ConversationMessage[]
+  /** Tool calls captured per turn (parallel to userTurns). */
+  toolCallsPerTurn: string[][]
+  durationMs: number
+}
+
+const DEFAULT_TIMEOUT_MS = 60_000
+
+const withTimeout = async <T>(p: Promise<T>, ms: number, label: string): Promise<T> => {
+  let timer: ReturnType<typeof setTimeout> | undefined
+  const timeout = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms)
+  })
+  try {
+    return await Promise.race([p, timeout])
+  } finally {
+    if (timer) clearTimeout(timer)
+  }
+}
+
+export const runMultiTurnRollout = async (args: {
+  scenario: MultiTurnScenario
+  driver: TurnDriver
+  store: TraceStore
+  emitter: TraceEmitter
+  /** Optional run-id seed; if omitted, derived from scenarioId + Date.now(). */
+  runId?: string
+}): Promise<MultiTurnRolloutResult> => {
+  const { scenario, driver, store, emitter } = args
+  const runId = args.runId ?? `${scenario.scenarioId}-${Date.now()}`
+  const maxTurns = scenario.maxTurns ?? scenario.userTurns.length
+  const timeoutMs = scenario.timeoutMs ?? DEFAULT_TIMEOUT_MS
+
+  const transcript: ConversationMessage[] = []
+  if (scenario.systemPrompt) {
+    transcript.push({ role: 'system', content: scenario.systemPrompt })
+  }
+  const toolCallsPerTurn: string[][] = []
+
+  const start = Date.now()
+  for (let i = 0; i < maxTurns; i += 1) {
+    const userMsg = scenario.userTurns[i]
+    if (userMsg === undefined) break
+    transcript.push({ role: 'user', content: userMsg })
+    const result = await withTimeout(
+      driver({ transcript: [...transcript], runId, emitter }),
+      timeoutMs,
+      `turn ${i + 1}`,
+    )
+    transcript.push({ role: 'assistant', content: result.assistantText })
+    toolCallsPerTurn.push(result.toolCalls ?? [])
+  }
+  const durationMs = Date.now() - start
+
+  const trajectory = await buildTrajectory(store, runId)
+  return { scenarioId: scenario.scenarioId, runId, trajectory, transcript, toolCallsPerTurn, durationMs }
+}
+
+/**
+ * Group trajectory steps by user-turn index using span timestamps and the
+ * conversation transcript. Returns a parallel array `turns[i]` containing
+ * the steps that occurred between user-turn i and user-turn i+1.
+ *
+ * Used by per-turn-scorer.ts to apply rubrics on a per-turn basis rather
+ * than aggregating across the entire trajectory.
+ */
+export const groupStepsByTurn = (
+  trajectory: Trajectory,
+  userTurnCount: number,
+): TrajectoryStep[][] => {
+  // Heuristic: split steps into N buckets by trajectory index. Spans inside
+  // a single user→assistant turn share an `index` band because buildTrajectory
+  // produces topo-ordered steps; partition into equal-ish chunks falls out
+  // of the topo ordering assumption. Drivers that emit a 'user-turn-marker'
+  // event can override this by partitioning on event timestamps instead.
+  const buckets: TrajectoryStep[][] = Array.from({ length: userTurnCount }, () => [])
+  if (trajectory.steps.length === 0 || userTurnCount === 0) return buckets
+  const stepsPerTurn = Math.ceil(trajectory.steps.length / userTurnCount)
+  for (const step of trajectory.steps) {
+    const idx = Math.min(Math.floor(step.index / stepsPerTurn), userTurnCount - 1)
+    buckets[idx].push(step)
+  }
+  return buckets
+}