@qvac/harness 1.0.0-rc.2 → 1.0.0-rc.3

package/bench/skill-eval/packs/diagrams/index.ts

@@ -0,0 +1,50 @@
+import { replyFailures } from '../../laws.ts'
+import { openMermaidOracle } from '../../mermaid/client.ts'
+import type { PackWorld, SkillPack } from '../../pack.ts'
+import { CASES, LEAKED_TOOL, SKILL, TICKET_CASE, type Expect } from './cases.ts'
+import { mermaidBlocks, structureFailures } from './score.ts'
+
+// the skill grants no tools, so the only tool surface is the `skill` loader QVAC-24436 gets wrong
+function world(): PackWorld<Expect> {
+  const oracle = openMermaidOracle()
+
+  return {
+    tools: [],
+    reset() {},
+    async judge(evalCase, observed) {
+      const blocks = mermaidBlocks(observed.reply)
+      const verdicts = await Promise.all(blocks.map((block) => oracle.parse(block)))
+      const syntax = verdicts
+        .filter((verdict) => !verdict.ok)
+        .map((verdict) => `syntax: ${verdict.message}`)
+
+      const renders = blocks.length === 0 ? null : verdicts.every((verdict) => verdict.ok)
+
+      return {
+        failures: [
+          ...structureFailures(evalCase.expect, blocks),
+          ...syntax,
+          ...replyFailures(observed.reply, evalCase.expect.reply)
+        ],
+        // nothing here is answered from a fixture, so a run can never be an eval defect
+        gap: false,
+        renders,
+        // what the user saw: a drawable diagram, or none when none was wanted, with no tool framing
+        outcome:
+          (evalCase.expect.type === '' ? blocks.length === 0 : renders === true) &&
+          !LEAKED_TOOL.test(observed.reply)
+      }
+    },
+    save() {},
+    close() {
+      oracle.close()
+    }
+  }
+}
+
+export const diagrams: SkillPack<Expect> = {
+  skill: SKILL,
+  focus: TICKET_CASE,
+  cases: CASES,
+  world
+}

package/bench/skill-eval/packs/diagrams/score.ts

@@ -0,0 +1,74 @@
+import type { ToolCall } from '../../pack.ts'
+import type { Expect } from './cases.ts'
+
+// matches the renderers' case-sensitive language check, the same rule lib/diagram-repair.ts uses
+const FENCE = /^[ \t]{0,3}(`{3,}|~{3,})[ \t]*mermaid[ \t]*\r?\n([\s\S]*?)^[ \t]{0,3}\1[ \t]*$/gm
+
+// a label never spans lines: without \n here the scan runs on through the next transition
+const LABEL = /[[({]"?([^"\]})\n]{1,200})"?[\])}]/g
+const NODE_ID = /^\s*([A-Za-z]\w*)\s*[[({]/gm
+
+export const LABEL_LIMIT = 40
+
+export function mermaidBlocks(reply: string): string[] {
+  FENCE.lastIndex = 0
+  return [...reply.matchAll(FENCE)].map((match) => match[2] ?? '').filter((body) => body.trim())
+}
+
+export function diagramType(block: string): string {
+  const first = block.trim().split(/\r?\n/)[0]?.trim() ?? ''
+  // `flowchart TD` and `stateDiagram-v2` both key on the first word
+  return first.split(/\s+/)[0] ?? ''
+}
+
+// mermaid accepts a reused id (the later label wins), so the oracle never sees it
+export function duplicateIds(block: string): string[] {
+  NODE_ID.lastIndex = 0
+  const seen = new Set<string>()
+  const twice = new Set<string>()
+  for (const match of block.matchAll(NODE_ID)) {
+    const id = match[1]
+    if (seen.has(id)) {
+      twice.add(id)
+    }
+
+    seen.add(id)
+  }
+
+  return [...twice]
+}
+
+export function longLabels(block: string): string[] {
+  LABEL.lastIndex = 0
+  return [...block.matchAll(LABEL)]
+    .map((match) => match[1].trim())
+    .filter((label) => label.length > LABEL_LIMIT)
+}
+
+export function structureFailures(expect: Expect, blocks: string[]): string[] {
+  const failures: string[] = []
+  const [low, high] = Array.isArray(expect.blocks) ? expect.blocks : [expect.blocks, expect.blocks]
+  if (blocks.length < low || blocks.length > high) {
+    failures.push(`blocks: ${blocks.length} not ${low === high ? low : `${low}-${high}`}`)
+  }
+
+  for (const block of blocks) {
+    const type = diagramType(block)
+    if (expect.type && type !== expect.type) {
+      failures.push(`type: ${type || 'none'} not ${expect.type}`)
+    }
+
+    // a class member like `speak()` reads as a node id, and one per class is not a duplicate
+    if (type === 'flowchart') {
+      for (const id of duplicateIds(block)) {
+        failures.push(`duplicate id: ${id}`)
+      }
+    }
+
+    for (const label of longLabels(block)) {
+      failures.push(`label over ${LABEL_LIMIT}: ${label.slice(0, 50)}`)
+    }
+  }
+
+  return failures
+}

package/bench/skill-eval/packs/index.ts

@@ -0,0 +1,5 @@
+import type { AnyPack } from '../pack.ts'
+import { diagrams } from './diagrams/index.ts'
+import { weather } from './weather/index.ts'
+
+export const PACKS: Readonly<Record<string, AnyPack>> = { diagrams, weather }

package/bench/skill-eval/packs/weather/cases.ts

@@ -0,0 +1,279 @@
+import { skillLoadedTurn } from '../../../../lib/capabilities/tool-turns.ts'
+import type { ReplyLaws } from '../../laws.ts'
+import { assistant, user, type EvalCase as Case } from '../../pack.ts'
+import type { Suffix } from './fixtures.ts'
+
+export const CATEGORIES = [
+  'city-country',
+  'ambiguous',
+  'unambiguous',
+  'unknown',
+  'forecast',
+  'casual',
+  'no-location',
+  'follow-up'
+] as const
+
+export type Category = (typeof CATEGORIES)[number]
+
+export interface Expect {
+  // http_request calls only; a skill load is never counted
+  readonly calls: number | readonly [number, number]
+  // normalised as wttrRequest() does; any listed form passes
+  readonly location?: ReadonlyArray<string> | undefined
+  readonly suffix?: ReadonlyArray<Suffix> | undefined
+  // the reply must state a temperature the fixture served, in °C or converted to °F
+  readonly echoes?: boolean | undefined
+  readonly reply?: ReplyLaws | undefined
+}
+
+export type EvalCase = Case<Expect>
+
+export const SKILL = 'weather'
+export const TICKET_CASE = 'nassau-bahamas'
+
+export const TEMPERATURE =
+  /[+-]?\d{1,3}(?:\.\d)?\s?(?:°|º|˚|degrees?)(?:\s?(?:c|f|celsius|fahrenheit)\b)?/i
+const QUESTION = /\?/
+const WHICH =
+  /\b(?:which|do you mean|did you mean|clarify|specify|more than one|several|multiple)\b/i
+const HEDGE = /\b(?:do you mean|did you mean|which (?:one|city|location|place))\b/i
+const ASK_LOCATION = /\b(?:city|location|place|town|where)\b/i
+const NOT_FOUND =
+  /\b(?:not|no|unable|cannot|can't|couldn't|unknown|invalid|unrecogni[sz]ed|failed?|error|500)\b|n't\b/i
+
+interface Lookup {
+  readonly category: Category
+  readonly prompt: string
+  readonly location: ReadonlyArray<string>
+}
+
+const current = (id: string, lookup: Lookup): EvalCase => ({
+  id,
+  category: lookup.category,
+  messages: [user(lookup.prompt)],
+  expect: {
+    calls: 1,
+    location: lookup.location,
+    suffix: ['format=3'],
+    echoes: true,
+    reply: { none: [HEDGE] }
+  }
+})
+
+interface Ask {
+  readonly category: Category
+  readonly prompt: string
+  readonly names: ReadonlyArray<string>
+  readonly asks: RegExp
+}
+
+const asks = (id: string, ask: Ask): EvalCase => ({
+  id,
+  category: ask.category,
+  messages: [user(ask.prompt)],
+  expect: {
+    calls: 0,
+    reply: { laws: { has: ask.names }, all: [QUESTION, ask.asks], none: [TEMPERATURE] }
+  }
+})
+
+interface Ambiguous {
+  readonly prompt: string
+  readonly city: string
+}
+
+const ambiguous = (id: string, ask: Ambiguous): EvalCase =>
+  asks(id, { category: 'ambiguous', prompt: ask.prompt, names: [ask.city], asks: WHICH })
+
+interface UnknownPlace {
+  readonly prompt: string
+}
+
+// declining to look up an obviously invented name is as good as asking wttr.in and relaying its miss
+const unknown = (id: string, ask: UnknownPlace): EvalCase => ({
+  id,
+  category: 'unknown',
+  messages: [user(ask.prompt)],
+  expect: { calls: [0, 2], suffix: ['format=3'], reply: { all: [NOT_FOUND], none: [TEMPERATURE] } }
+})
+
+interface Forecast {
+  readonly prompt: string
+  readonly location: ReadonlyArray<string>
+  readonly suffix: ReadonlyArray<Suffix>
+}
+
+const forecast = (id: string, ask: Forecast): EvalCase => ({
+  id,
+  category: 'forecast',
+  messages: [user(ask.prompt)],
+  expect: { calls: 1, location: ask.location, suffix: ask.suffix, reply: { none: [HEDGE] } }
+})
+
+export const CASES: ReadonlyArray<EvalCase> = [
+  current(TICKET_CASE, {
+    category: 'city-country',
+    prompt: 'Whats the weather like in Nassau, Bahamas?',
+    location: ['nassau bahamas', 'nassau the bahamas']
+  }),
+  current('paris-texas', {
+    category: 'city-country',
+    prompt: 'weather in Paris, Texas',
+    location: ['paris texas', 'paris tx']
+  }),
+  current('cambridge-uk', {
+    category: 'city-country',
+    prompt: 'is it raining in Cambridge, UK right now',
+    location: ['cambridge uk', 'cambridge united kingdom', 'cambridge england']
+  }),
+  current('san-jose-cr', {
+    category: 'city-country',
+    prompt: 'temperature in San Jose, Costa Rica',
+    location: ['san jose costa rica']
+  }),
+  current('portland-oregon', {
+    category: 'city-country',
+    prompt: 'Portland, Oregon weather please',
+    location: ['portland oregon', 'portland or']
+  }),
+  current('georgia-country', {
+    category: 'city-country',
+    prompt: 'How hot is it in Georgia, the country?',
+    location: ['tbilisi', 'tbilisi georgia']
+  }),
+
+  ambiguous('nassau-bare', { prompt: "what's the weather in Nassau", city: 'nassau' }),
+  ambiguous('springfield', { prompt: 'Springfield weather', city: 'springfield' }),
+  ambiguous('georgia', { prompt: "how's the weather in Georgia today", city: 'georgia' }),
+  ambiguous('portland', { prompt: 'Portland weather forecast for tomorrow', city: 'portland' }),
+  ambiguous('cambridge', { prompt: 'weather in Cambridge', city: 'cambridge' }),
+
+  current('london', {
+    category: 'unambiguous',
+    prompt: 'weather in London',
+    location: ['london', 'london uk', 'london united kingdom', 'london england']
+  }),
+  current('tokyo', {
+    category: 'unambiguous',
+    prompt: 'tokyo weather rn',
+    location: ['tokyo', 'tokyo japan']
+  }),
+  current('berlin', {
+    category: 'unambiguous',
+    prompt: 'is it cold in Berlin?',
+    location: ['berlin', 'berlin germany']
+  }),
+  current('reykjavik', {
+    category: 'unambiguous',
+    prompt: "how's the weather in Reykjavik",
+    location: ['reykjavik', 'reykjavik iceland']
+  }),
+  current('cairo', {
+    category: 'unambiguous',
+    prompt: 'hows the weather in Cairo',
+    location: ['cairo', 'cairo egypt']
+  }),
+  current('paris', {
+    category: 'unambiguous',
+    prompt: 'weather in Paris',
+    location: ['paris', 'paris france']
+  }),
+
+  unknown('xyzzyqwerty', { prompt: 'weather in Xyzzyqwerty' }),
+  unknown('zzyzxville', { prompt: "what's the weather like in Zzyzxville" }),
+  unknown('qxzvtown', { prompt: 'how is the weather in Qxzvtown' }),
+  unknown('vrblxstan', { prompt: 'Vrblxstan weather today' }),
+
+  forecast('lisbon-weekend', {
+    prompt: 'will it rain in Lisbon, Portugal this weekend',
+    location: ['lisbon portugal'],
+    suffix: ['2T', 'T']
+  }),
+  forecast('rome-tomorrow', {
+    prompt: "what's the weather in Rome tomorrow?",
+    location: ['rome', 'rome italy'],
+    suffix: ['2T', 'T']
+  }),
+  forecast('oslo-today', {
+    prompt: "what's today's weather forecast for Oslo?",
+    location: ['oslo', 'oslo norway'],
+    suffix: ['1T', 'format=3']
+  }),
+  forecast('madrid-3-days', {
+    prompt: 'Madrid weather for the next 3 days',
+    location: ['madrid', 'madrid spain'],
+    suffix: ['2T', 'T']
+  }),
+  {
+    id: 'madrid-7-days',
+    category: 'forecast',
+    messages: [user('Madrid weather for the next 7 days')],
+    expect: {
+      calls: [0, 1],
+      location: ['madrid', 'madrid spain'],
+      suffix: ['2T', 'T'],
+      reply: { all: [/\b3\b|three/i] }
+    }
+  },
+
+  current('nyc', {
+    category: 'casual',
+    prompt: 'yo whats the weather like in nyc',
+    location: [
+      'nyc',
+      'new york',
+      'new york city',
+      'new york ny',
+      'new york usa',
+      'nyc usa',
+      'new york city usa'
+    ]
+  }),
+  current('sao-paulo', {
+    category: 'casual',
+    prompt: 'temp in São Paulo?',
+    location: ['sao paulo', 'sao paulo brazil']
+  }),
+  current('rio', {
+    category: 'casual',
+    prompt: 'weather Rio de Janeiro, Brazil',
+    location: ['rio de janeiro brazil']
+  }),
+  current('zurich', {
+    category: 'casual',
+    prompt: 'Wetter in Zürich, Schweiz?',
+    location: ['zurich schweiz', 'zurich switzerland']
+  }),
+
+  asks('no-location', {
+    category: 'no-location',
+    prompt: "what's the weather like?",
+    names: [],
+    asks: ASK_LOCATION
+  }),
+  asks('rain-later', {
+    category: 'no-location',
+    prompt: 'is it going to rain later?',
+    names: [],
+    asks: ASK_LOCATION
+  }),
+
+  // the host replays an earlier load as a name-only stub, so the model must reload before calling
+  {
+    id: 'nassau-follow-up',
+    category: 'follow-up',
+    messages: [
+      user('weather in Nassau?'),
+      { role: 'tool', content: skillLoadedTurn(SKILL) },
+      assistant('Which Nassau do you mean: the one in the Bahamas, or Nassau County in New York?'),
+      user('the one in the Bahamas')
+    ],
+    expect: {
+      calls: 1,
+      location: ['nassau bahamas', 'nassau the bahamas'],
+      suffix: ['format=3'],
+      echoes: true
+    }
+  }
+]