@qvac/harness 1.0.0-rc.2 → 1.0.0-rc.3

package/bench/skill-eval/packs/weather/fixtures.ts

@@ -0,0 +1,84 @@
+import type { FetchFn } from '../../../../lib/tools/http-request.ts'
+
+export const SUFFIXES = ['format=3', '1T', '2T', 'T', 'bare', 'other'] as const
+
+export type Suffix = (typeof SUFFIXES)[number]
+
+export interface WttrRequest {
+  readonly url: string
+  readonly host: string
+  readonly location: string
+  readonly suffix: Suffix
+}
+
+export interface Fixture {
+  readonly status: number
+  readonly body: string
+}
+
+export type Fixtures = Record<string, Fixture>
+
+export const FIXTURE_MISS: Fixture = { status: 599, body: 'eval fixture miss' }
+
+// one key for fixtures and judging; wttr.in treats `Paris,+Texas`, `Paris+Texas` and `Zürich` alike
+export function wttrRequest(url: URL): WttrRequest {
+  const location = decodeURIComponent(url.pathname.slice(1))
+    .normalize('NFD')
+    .replace(/[\u0300-\u036f]/g, '')
+    .replace(/[+,]/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim()
+    .toLowerCase()
+  return { url: url.href, host: url.hostname, location, suffix: suffixOf(url.search.slice(1)) }
+}
+
+export function wttrKey(request: WttrRequest): string {
+  return `${request.location}|${request.suffix}`
+}
+
+function suffixOf(query: string): Suffix {
+  if (query === '') return 'bare'
+  return SUFFIXES.find((suffix) => suffix === query) ?? 'other'
+}
+
+export interface FakeFetchOptions {
+  readonly onServe: (key: string, fixture: Fixture) => void
+  readonly onMiss: (key: string) => void
+  // present only under --record: a miss is fetched live and kept
+  readonly live?: FetchFn | undefined
+}
+
+// offline, an off-list suffix gets the one-liner: the scorer fails the suffix, the model sees weather
+export function fakeFetch(fixtures: Fixtures, options: FakeFetchOptions): FetchFn {
+  return async (url, init) => {
+    const request = wttrRequest(url)
+    const key = wttrKey(request)
+    const offList = request.suffix === 'other' && !options.live
+    const known = fixtures[key] ?? (offList ? fixtures[`${request.location}|format=3`] : undefined)
+    if (known) {
+      options.onServe(key, known)
+      return response(known)
+    }
+
+    if (!options.live) {
+      options.onMiss(key)
+      return response(FIXTURE_MISS)
+    }
+
+    const recorded = await record(options.live, url, init)
+    fixtures[key] = recorded
+    options.onServe(key, recorded)
+
+    return response(recorded)
+  }
+}
+
+// wttr.in serves HTML to a browser agent; the product sends an empty one, so recording does too
+async function record(live: FetchFn, url: URL, init: Parameters<FetchFn>[1]): Promise<Fixture> {
+  const res = await live(url, { ...init, headers: { ...init.headers, 'user-agent': '' } })
+  return { status: res.status, body: await res.text() }
+}
+
+function response(fixture: Fixture): Awaited<ReturnType<FetchFn>> {
+  return { status: fixture.status, headers: { get: () => null }, text: () => fixture.body }
+}

package/bench/skill-eval/packs/weather/index.ts

@@ -0,0 +1,55 @@
+import fs from 'fs'
+import { fileURLToPath } from 'url'
+import { createHttpRequestTool } from '../../../../lib/tools/http-request.ts'
+import type { PackWorld, SkillPack, WorldOptions } from '../../pack.ts'
+import { CASES, SKILL, TICKET_CASE, type Expect } from './cases.ts'
+import { fakeFetch, type Fixture, type Fixtures } from './fixtures.ts'
+import { requested, score } from './score.ts'
+
+// any public address: the url guard resolves the host on every call, and the eval is offline
+const WTTR_ADDRESSES = ['5.9.243.187']
+const FIXTURES_PATH = fileURLToPath(new URL('./fixtures/wttr.json', import.meta.url))
+
+function world(options: WorldOptions): PackWorld<Expect> {
+  const fixtures: Fixtures = JSON.parse(fs.readFileSync(FIXTURES_PATH, 'utf8'))
+  const served: Fixture[] = []
+  const misses: string[] = []
+  const fetch = fakeFetch(fixtures, {
+    onServe: (_key, fixture) => served.push(fixture),
+    onMiss: (key) => misses.push(key),
+    live: options.live
+  })
+  const http = createHttpRequestTool({
+    fetch,
+    resolveHost: () => Promise.resolve(WTTR_ADDRESSES)
+  })
+
+  return {
+    tools: [http],
+    reset() {
+      served.length = 0
+      misses.length = 0
+    },
+    judge(evalCase, observed) {
+      const calls = observed.calls
+        .filter((call) => call.name === http.name)
+        .map((call) => requested(call.args))
+      const verdict = score(evalCase, { calls, served, reply: observed.reply })
+      // a wrong call is judged on its own; only a right call the fixtures could not answer is a gap
+      const callWrong = verdict.some((failure) => /^(?:calls|host|location|suffix):/.test(failure))
+
+      return {
+        failures: [...misses.map((key) => `fixture miss: ${key}`), ...verdict],
+        gap: misses.length > 0 && !callWrong
+      }
+    },
+    save() {
+      const sorted = Object.fromEntries(
+        Object.entries(fixtures).sort(([a], [b]) => a.localeCompare(b))
+      )
+      fs.writeFileSync(FIXTURES_PATH, `${JSON.stringify(sorted, null, 2)}\n`)
+    }
+  }
+}
+
+export const weather: SkillPack<Expect> = { skill: SKILL, focus: TICKET_CASE, cases: CASES, world }

package/bench/skill-eval/packs/weather/score.ts

@@ -0,0 +1,92 @@
+import { applyQuery, parseObjectArg } from '../../../../lib/tools/http-request/request.ts'
+import { replyFailures } from '../../laws.ts'
+import { TEMPERATURE, type EvalCase, type Expect } from './cases.ts'
+import { wttrRequest, type Fixture, type WttrRequest } from './fixtures.ts'
+
+export interface Observed {
+  readonly calls: ReadonlyArray<WttrRequest>
+  readonly served: ReadonlyArray<Fixture>
+  readonly reply: string
+}
+
+const WTTR_HOST = 'wttr.in'
+
+export function score(evalCase: EvalCase, observed: Observed): string[] {
+  return [
+    ...callFailures(evalCase.expect, observed.calls),
+    ...replyChecks(evalCase.expect, observed)
+  ]
+}
+
+function callFailures(expect: Expect, calls: ReadonlyArray<WttrRequest>): string[] {
+  const failures: string[] = []
+  const [min, max] = typeof expect.calls === 'number' ? [expect.calls, expect.calls] : expect.calls
+  if (calls.length < min || calls.length > max) {
+    failures.push(`calls: ${calls.length}, expected ${min === max ? min : `${min}-${max}`}`)
+  }
+
+  for (const call of calls) {
+    if (call.host !== WTTR_HOST) {
+      failures.push(`host: ${call.host}`)
+    }
+
+    if (expect.location && !expect.location.includes(call.location)) {
+      failures.push(`location: ${call.location}`)
+    }
+
+    if (expect.suffix && !expect.suffix.includes(call.suffix)) {
+      failures.push(`suffix: ${call.suffix}`)
+    }
+  }
+
+  return failures
+}
+
+function replyChecks(expect: Expect, observed: Observed): string[] {
+  const failures = replyFailures(observed.reply, expect.reply)
+  if (failures.length === 1 && failures[0] === 'reply: empty') {
+    return failures
+  }
+
+  return expect.echoes ? [...echoFailures(observed), ...failures] : failures
+}
+
+// judged as the tool sends it: `query` merged into the url, an unparseable url failing on the host
+export function requested(args: Record<string, unknown>): WttrRequest {
+  try {
+    const url = new URL(String(args.url))
+    const query = parseObjectArg(args.query, 'query')
+    if (query.ok && query.value) {
+      applyQuery(url, query.value)
+    }
+
+    return wttrRequest(url)
+  } catch {
+    return { url: String(args.url), host: 'invalid', location: '', suffix: 'other' }
+  }
+}
+
+// the fixture is the truth, so re-recording on another day never breaks a case
+function echoFailures(observed: Observed): string[] {
+  const celsius = observed.served.flatMap((fixture) => temperatures(fixture.body))
+  if (celsius.length === 0) {
+    return ['echo: no temperature was served']
+  }
+
+  const accepted = [...celsius, ...celsius.map(fahrenheit)]
+  const stated = temperatures(observed.reply)
+
+  return stated.some((value) => accepted.some((known) => Math.abs(known - value) <= 1))
+    ? []
+    : [`echo: reply says ${stated.join(', ') || 'nothing'}, served ${celsius.join(', ')}°C`]
+}
+
+function fahrenheit(celsius: number): number {
+  return Math.round((celsius * 9) / 5 + 32)
+}
+
+function temperatures(text: string): number[] {
+  return Array.from(text.matchAll(new RegExp(TEMPERATURE.source, 'gi')), (match) =>
+    Number.parseFloat(match[0])
+  )
+}

package/bench/skill-eval/report.ts

@@ -0,0 +1,194 @@
+// prose is a turn the harness preloads the skill into; self-load makes the model fetch it first
+export const ROUTES = ['mention', 'prose', 'self-load'] as const
+
+export type Route = (typeof ROUTES)[number]
+
+export interface RunRow {
+  readonly variant: string
+  readonly route: Route
+  readonly caseId: string
+  readonly category: string
+  readonly repeat: number
+  readonly passed: boolean
+  readonly gap: boolean
+  // null when the reply carried no diagram to parse, so it leaves the renders column
+  readonly renders: boolean | null
+  // the pack's user-facing verdict; null for a pack that has only its laws
+  readonly outcome: boolean | null
+  readonly failures: ReadonlyArray<string>
+  readonly reply: string
+  readonly calls: ReadonlyArray<string>
+  readonly skillLoaded: boolean
+  readonly thinkingChars: number
+  // the whole prompt the last round sent, and the share of it the skills index took
+  readonly contextTokens: number | null
+  readonly skillTokens: number | null
+  readonly ttftMs: number | null
+  readonly generatedTokens: number | null
+  readonly wallMs: number
+  readonly error: string | null
+}
+
+export interface SummaryOptions {
+  readonly focus: string
+  readonly categories: ReadonlyArray<string>
+}
+
+const CSV_COLUMNS = [
+  'variant',
+  'route',
+  'caseId',
+  'category',
+  'repeat',
+  'passed',
+  'gap',
+  'renders',
+  'outcome',
+  'failures',
+  'calls',
+  'skillLoaded',
+  'thinkingChars',
+  'contextTokens',
+  'skillTokens',
+  'ttftMs',
+  'generatedTokens',
+  'wallMs',
+  'error'
+] as const
+
+export function toJsonl(rows: ReadonlyArray<RunRow>): string {
+  return rows.map((row) => JSON.stringify(row)).join('\n') + '\n'
+}
+
+export function toCsv(rows: ReadonlyArray<RunRow>): string {
+  const lines = rows.map((row) => CSV_COLUMNS.map((column) => csvCell(row[column])).join(','))
+  return [CSV_COLUMNS.join(','), ...lines].join('\n') + '\n'
+}
+
+function csvCell(value: unknown): string {
+  const text = Array.isArray(value) ? value.join('; ') : String(value ?? '')
+  return /[",\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text
+}
+
+export function summaryMarkdown(rows: ReadonlyArray<RunRow>, options: SummaryOptions): string {
+  const variants = unique(rows.map((row) => row.variant))
+  const routes = ROUTES.filter((route) => rows.some((row) => row.route === route))
+  const cases = unique(rows.map((row) => row.caseId))
+  const used = options.categories.filter((category) =>
+    rows.some((row) => row.category === category)
+  )
+
+  const header = [
+    'variant',
+    'route',
+    'overall',
+    'outcome',
+    'renders',
+    ...used,
+    options.focus,
+    'ctx tok',
+    'skills idx tok',
+    'ttft ms',
+    'wall ms',
+    'gaps',
+    'think',
+    'errors'
+  ]
+  const summary = variants.flatMap((variant) =>
+    routes.map((route) => {
+      const mine = rows.filter((row) => row.variant === variant && row.route === route)
+      return [
+        variant,
+        route,
+        rate(mine),
+        outcomeRate(mine),
+        renderRate(mine),
+        ...used.map((category) => rate(mine.filter((row) => row.category === category))),
+        rate(mine.filter((row) => row.caseId === options.focus)),
+        mean(mine.map((row) => row.contextTokens)),
+        mean(mine.map((row) => row.skillTokens)),
+        mean(mine.map((row) => row.ttftMs)),
+        mean(mine.map((row) => row.wallMs)),
+        String(mine.filter((row) => row.gap).length),
+        String(mine.filter((row) => row.thinkingChars > 0).length),
+        String(mine.filter((row) => row.error !== null).length)
+      ]
+    })
+  )
+
+  const columns = variants.flatMap((variant) => routes.map((route) => ({ variant, route })))
+  const matrixHeader = [
+    'case',
+    'category',
+    ...columns.map((column) => `${column.variant}/${column.route}`)
+  ]
+  const matrix = cases.map((caseId) => {
+    const mine = rows.filter((row) => row.caseId === caseId)
+    return [
+      caseId,
+      mine[0]?.category ?? '',
+      ...columns.map((column) =>
+        count(mine.filter((row) => row.variant === column.variant && row.route === column.route))
+      )
+    ]
+  })
+
+  return `## Variants\n\n${table(header, summary)}\n\n## Cases\n\n${table(matrixHeader, matrix)}\n`
+}
+
+// a gap row is an eval defect, not a model result, so it leaves the denominator
+export function rate(rows: ReadonlyArray<RunRow>): string {
+  const judged = rows.filter((row) => !row.gap)
+  if (judged.length === 0) {
+    return '-'
+  }
+
+  const passed = judged.filter((row) => row.passed).length
+
+  return `${Math.round((passed / judged.length) * 100)}% (${passed}/${judged.length})`
+}
+
+// what the user got, by the pack's own definition; '-' for a pack that defines none
+export function outcomeRate(rows: ReadonlyArray<RunRow>): string {
+  const judged = rows.filter((row) => !row.gap && row.outcome !== null)
+  if (judged.length === 0) {
+    return '-'
+  }
+
+  const good = judged.filter((row) => row.outcome).length
+
+  return `${Math.round((good / judged.length) * 100)}% (${good}/${judged.length})`
+}
+
+// a diagram that does not parse shows as unavailable; a reply with no diagram leaves the denominator
+export function renderRate(rows: ReadonlyArray<RunRow>): string {
+  const drawn = rows.filter((row) => !row.gap && row.renders !== null)
+  if (drawn.length === 0) {
+    return '-'
+  }
+
+  const parsed = drawn.filter((row) => row.renders).length
+
+  return `${Math.round((parsed / drawn.length) * 100)}% (${parsed}/${drawn.length})`
+}
+
+function count(rows: ReadonlyArray<RunRow>): string {
+  const judged = rows.filter((row) => !row.gap)
+  return judged.length === 0 ? '-' : `${judged.filter((row) => row.passed).length}/${judged.length}`
+}
+
+function mean(values: ReadonlyArray<number | null>): string {
+  const known = values.filter((value): value is number => value !== null)
+  return known.length === 0
+    ? '-'
+    : String(Math.round(known.reduce((a, b) => a + b, 0) / known.length))
+}
+
+function unique<T>(values: ReadonlyArray<T>): T[] {
+  return [...new Set(values)]
+}
+
+function table(header: ReadonlyArray<string>, body: ReadonlyArray<ReadonlyArray<string>>): string {
+  const line = (cells: ReadonlyArray<string>) => `| ${cells.join(' | ')} |`
+  return [line(header), line(header.map(() => '---')), ...body.map(line)].join('\n')
+}