@qvac/harness 1.0.0-rc.2 → 1.0.0-rc.3

package/bench/skill-eval/harness.ts

@@ -0,0 +1,30 @@
+import process from 'process'
+import { Harness } from '../../index.ts'
+import type { ToolDefinition } from '../../lib/contract/index.ts'
+import { createSkillTool } from '../../lib/tools/skill.ts'
+import { configureSdkStorage } from '../../lib/sdk-storage.ts'
+import { MODEL_CACHE_PATH_ENV, resolveStorage } from '../../lib/storage.ts'
+
+export interface EvalHarnessOptions {
+  readonly root: string
+  readonly skillsDir: string
+  readonly tools: ReadonlyArray<ToolDefinition>
+  readonly ctxSize: number
+}
+
+// storage first: the skill tool captures the overlay dir from QVAC_CONFIG_PATH when it is built
+export async function openEvalHarness(options: EvalHarnessOptions): Promise<Harness> {
+  const storage = resolveStorage('dev', {
+    root: options.root,
+    models: process.env[MODEL_CACHE_PATH_ENV]
+  })
+  configureSdkStorage(storage.sdk)
+
+  const { skillsDir } = options
+  const tools = [...options.tools, createSkillTool({ skillsDir })]
+  const harness = new Harness({ tools, skillsDir, contextSize: options.ctxSize })
+  await harness.ready()
+  await harness.clearKvCache()
+
+  return harness
+}

package/bench/skill-eval/laws.ts

@@ -0,0 +1,37 @@
+import { check, type Laws } from 'asenion'
+
+// asenion laws compare on whole words; the regex lists cover what a word law cannot say
+export interface ReplyLaws {
+  readonly laws?: Laws | undefined
+  readonly all?: ReadonlyArray<RegExp> | undefined
+  readonly none?: ReadonlyArray<RegExp> | undefined
+}
+
+export function replyFailures(reply: string, laws: ReplyLaws | undefined): string[] {
+  if (reply.trim() === '') {
+    return ['reply: empty']
+  }
+
+  const failures: string[] = []
+  if (laws?.laws) {
+    for (const failure of check(reply, laws.laws).failures) {
+      failures.push(
+        `${failure.rule}${failure.term === undefined ? '' : ` "${failure.term}"`}: ${failure.detail}`
+      )
+    }
+  }
+
+  for (const pattern of laws?.all ?? []) {
+    if (!pattern.test(reply)) {
+      failures.push(`missing: ${pattern.source}`)
+    }
+  }
+
+  for (const pattern of laws?.none ?? []) {
+    if (pattern.test(reply)) {
+      failures.push(`forbidden: ${pattern.source}`)
+    }
+  }
+
+  return failures
+}

package/bench/skill-eval/mermaid/client.ts

@@ -0,0 +1,132 @@
+import { spawn } from 'child_process'
+import { fileURLToPath } from 'url'
+
+const SIDECAR = fileURLToPath(new URL('./validate.mjs', import.meta.url))
+
+// mermaid needs a DOM, so the parse runs under node even when the sweep runs under Bare
+const NODE = 'node'
+
+// a parse takes milliseconds: a request this old means the sidecar is gone or wedged
+const PARSE_TIMEOUT_MS = 30_000
+
+export interface Verdict {
+  readonly ok: boolean
+  readonly message?: string | undefined
+}
+
+export interface MermaidOracle {
+  // the version the sidecar loaded, so a sweep's summary records what judged it
+  readonly version: Promise<string>
+  parse(code: string): Promise<Verdict>
+  close(): void
+}
+
+interface Waiting {
+  resolve(verdict: Verdict): void
+  reject(reason: Error): void
+}
+
+interface SidecarMessage {
+  ready?: string
+  id?: number
+  ok?: boolean
+  message?: string
+}
+
+// one sidecar per sweep: mermaid takes ~1s to load; a dead one fails every request loudly
+export function openMermaidOracle(): MermaidOracle {
+  const child = spawn(NODE, [SIDECAR], { stdio: ['pipe', 'pipe', 'inherit'] })
+  const waiting = new Map<number, Waiting>()
+  let ready: (version: string) => void
+  let notReady: (reason: Error) => void
+  const version = new Promise<string>((resolve, reject) => {
+    ready = resolve
+    notReady = reject
+  })
+  version.catch(noop)
+  let failure: Error | null = null
+  let closed = false
+  let next = 0
+  let pending = ''
+
+  child.stdout.setEncoding('utf8')
+  child.stdout.on('data', (chunk: string) => {
+    pending += chunk
+    for (;;) {
+      const newline = pending.indexOf('\n')
+      if (newline === -1) {
+        return
+      }
+
+      const line = pending.slice(0, newline)
+      pending = pending.slice(newline + 1)
+      if (line.trim()) {
+        receive(JSON.parse(line))
+      }
+    }
+  })
+  child.on('error', (err: Error) => fail(`mermaid oracle could not start node: ${err.message}`))
+  child.on('exit', (code: number | null, signal: string | null) => {
+    if (!closed) {
+      fail(`mermaid oracle exited (${code ?? signal})`)
+    }
+  })
+
+  function fail(why: string): void {
+    failure = new Error(why)
+    notReady(failure)
+    for (const entry of waiting.values()) {
+      entry.reject(failure)
+    }
+
+    waiting.clear()
+  }
+
+  function receive(message: SidecarMessage): void {
+    if (message.ready !== undefined) {
+      return ready(message.ready)
+    }
+
+    if (message.id === undefined) return
+    const entry = waiting.get(message.id)
+    waiting.delete(message.id)
+    entry?.resolve({ ok: message.ok === true, message: message.message })
+  }
+
+  return {
+    version,
+    parse(code) {
+      if (failure) {
+        return Promise.reject(failure)
+      }
+
+      const id = next++
+
+      return new Promise<Verdict>((resolve, reject) => {
+        const timer = setTimeout(() => {
+          waiting.delete(id)
+          reject(new Error(`mermaid oracle did not answer request ${id} in ${PARSE_TIMEOUT_MS}ms`))
+        }, PARSE_TIMEOUT_MS)
+        waiting.set(id, {
+          resolve: (verdict) => {
+            clearTimeout(timer)
+            resolve(verdict)
+          },
+          reject: (reason) => {
+            clearTimeout(timer)
+            reject(reason)
+          }
+        })
+        child.stdin.write(`${JSON.stringify({ id, code })}\n`)
+      })
+    },
+    close() {
+      closed = true
+      failure = new Error('mermaid oracle closed')
+      child.stdin.end()
+      child.kill()
+    }
+  }
+}
+
+function noop(): void {}

package/bench/skill-eval/mermaid/validate.mjs

@@ -0,0 +1,115 @@
+// the app's diagram acceptance test, out of process: mermaid needs a DOM and the eval runs on Bare
+import { createRequire } from 'module'
+import { JSDOM } from 'jsdom'
+
+const MERMAID_CONFIG = {
+  startOnLoad: false,
+  securityLevel: 'strict',
+  suppressErrorRendering: true,
+  htmlLabels: false,
+  maxTextSize: 50000,
+  maxEdges: 500,
+  theme: 'base',
+  flowchart: { htmlLabels: false }
+}
+
+const DIRECTIVE_OPEN = '%%{'
+const DIRECTIVE_CLOSE = '}%%'
+
+const dom = new JSDOM('<!DOCTYPE html><html><body></body></html>', { pretendToBeVisual: true })
+for (const key of ['window', 'document', 'Element', 'SVGElement', 'Node', 'HTMLElement']) {
+  const value = key === 'window' ? dom.window : dom.window[key]
+  Object.defineProperty(globalThis, key, { value, configurable: true, writable: true })
+}
+
+Object.defineProperty(globalThis, 'navigator', { value: dom.window.navigator, configurable: true })
+
+const mermaid = (await import('mermaid')).default
+mermaid.initialize(MERMAID_CONFIG)
+
+const require = createRequire(import.meta.url)
+send({ ready: require('mermaid/package.json').version })
+
+let pending = ''
+process.stdin.setEncoding('utf8')
+process.stdin.on('data', (chunk) => {
+  pending += chunk
+  for (;;) {
+    const newline = pending.indexOf('\n')
+    if (newline === -1) {
+      return
+    }
+
+    const line = pending.slice(0, newline)
+    pending = pending.slice(newline + 1)
+    if (line.trim()) {
+      void answer(JSON.parse(line))
+    }
+  }
+})
+
+async function answer(request) {
+  send({ id: request.id, ...(await verdictFor(request.code)) })
+}
+
+async function verdictFor(code) {
+  try {
+    await mermaid.parse(sanitize(code))
+    return { ok: true }
+  } catch (error) {
+    // the first line is the diagnosis; mermaid echoes the whole block back after it
+    return { ok: false, message: firstLine(error) }
+  }
+}
+
+function firstLine(error) {
+  return String(error?.message ?? error)
+    .split('\n')[0]
+    .slice(0, 200)
+}
+
+function send(message) {
+  process.stdout.write(`${JSON.stringify(message)}\n`)
+}
+
+function sanitize(code) {
+  return stripFrontmatterConfig(stripDirectives(code))
+}
+
+function stripDirectives(code) {
+  let out = ''
+  let cursor = 0
+  for (;;) {
+    const start = code.indexOf(DIRECTIVE_OPEN, cursor)
+    if (start === -1) {
+      return out + code.slice(cursor)
+    }
+
+    const end = code.indexOf(DIRECTIVE_CLOSE, start + DIRECTIVE_OPEN.length)
+    if (end === -1) {
+      return out + code.slice(cursor)
+    }
+
+    out += code.slice(cursor, start)
+    cursor = end + DIRECTIVE_CLOSE.length
+  }
+}
+
+function stripFrontmatterConfig(code) {
+  const lines = code.split(/\r?\n/)
+  const open = lines.findIndex((line) => line.trim() !== '')
+  if (open === -1 || lines[open] !== '---') {
+    return code
+  }
+
+  const close = lines.indexOf('---', open + 1)
+  if (close === -1) {
+    return code
+  }
+
+  const hasConfig = lines
+    .slice(open + 1, close)
+    .some((line) => /^config\s*:/.test(line.trimStart()))
+
+  return hasConfig ? lines.slice(close + 1).join('\n') : code
+}

package/bench/skill-eval/pack.ts

@@ -0,0 +1,61 @@
+import type { Message, ToolDefinition } from '../../lib/contract/index.ts'
+import type { FetchFn } from '../../lib/tools/http-request.ts'
+
+export interface EvalCase<Expect> {
+  readonly id: string
+  readonly category: string
+  readonly messages: ReadonlyArray<Message>
+  readonly expect: Expect
+}
+
+export interface ToolCall {
+  readonly name: string
+  readonly args: Record<string, unknown>
+}
+
+// what one run produced, before any pack-specific reading of it
+export interface Observed {
+  readonly calls: ReadonlyArray<ToolCall>
+  readonly reply: string
+}
+
+export interface Judgement {
+  readonly failures: ReadonlyArray<string>
+  // the fake world could not answer a correct call: the row says nothing about the model
+  readonly gap: boolean
+  // every diagram in the reply parsed; null when the reply carried none to judge
+  readonly renders?: boolean | null | undefined
+  // the user-facing result when a pack defines one apart from its laws: the reply did the job
+  readonly outcome?: boolean | undefined
+}
+
+// the fake tools a pack stands up around the model, and the verdict they let it reach
+export interface PackWorld<Expect> {
+  readonly tools: ReadonlyArray<ToolDefinition>
+  // called before every run so per-run traces start empty
+  reset(): void
+  judge(evalCase: EvalCase<Expect>, observed: Observed): Judgement | Promise<Judgement>
+  // persists anything --record captured
+  save(): void
+  // releases whatever world() opened, once the sweep is done
+  close?(): void
+}
+
+export interface WorldOptions {
+  // present only under --record: a pack may fetch what its fixtures lack
+  readonly live?: FetchFn | undefined
+}
+
+export interface SkillPack<Expect> {
+  // the skill directory name; also the @mention and the overlay slot variants are written to
+  readonly skill: string
+  // one case reported on its own column, usually the ticket's prompt
+  readonly focus: string
+  readonly cases: ReadonlyArray<EvalCase<Expect>>
+  world(options: WorldOptions): PackWorld<Expect>
+}
+
+export type AnyPack = SkillPack<unknown>
+
+export const user = (content: string): Message => ({ role: 'user', content })
+export const assistant = (content: string): Message => ({ role: 'assistant', content })

package/bench/skill-eval/packs/diagrams/cases.ts

@@ -0,0 +1,281 @@
+import { skillLoadedTurn } from '../../../../lib/capabilities/tool-turns.ts'
+import type { ReplyLaws } from '../../laws.ts'
+import { assistant, user, type EvalCase as Case } from '../../pack.ts'
+
+export const CATEGORIES = [
+  'flowchart',
+  'sequence',
+  'state',
+  'er',
+  'class',
+  'gantt',
+  'pie',
+  'mindmap',
+  'timeline',
+  'hard-label',
+  'iterate',
+  'route-away',
+  'long-context'
+] as const
+
+export type Category = (typeof CATEGORIES)[number]
+
+export interface Expect {
+  // the Mermaid keyword the first line inside the fence must be; '' when no diagram is wanted
+  readonly type: string
+  readonly blocks: number | readonly [number, number]
+  readonly reply?: ReplyLaws | undefined
+}
+
+export type EvalCase = Case<Expect>
+
+export const SKILL = 'diagrams'
+// QVAC-24436's own prompt, reported verbatim
+export const TICKET_CASE = 'vegetable-inventory-state'
+
+const FENCE_FIRST = /^```mermaid/
+const ASCII_ART = /^[ \t]*[+|][-+|_ ]{4,}/m
+const CANNOT = /\b(?:cannot|can't|unable to|I'm sorry|sorry,|as a text)\b/i
+// QVAC-24436's second symptom: the model's own framing printed as the answer
+export const LEAKED_TOOL = /"arguments"\s*:|rag_search|"error"\s*:|<tool_call>/
+const STYLING = /%%\{|classDef|^\s*style\s/m
+
+// every diagram case wants the same reply shape, so the laws live in one place
+const DRAWN: ReplyLaws = { all: [FENCE_FIRST], none: [ASCII_ART, CANNOT, LEAKED_TOOL, STYLING] }
+const NOT_DRAWN: ReplyLaws = { none: [/```mermaid/, LEAKED_TOOL] }
+
+interface Drawn {
+  readonly category: Category
+  readonly prompt: string
+  readonly type: string
+}
+
+const draws = (id: string, spec: Drawn): EvalCase => ({
+  id,
+  category: spec.category,
+  messages: [user(spec.prompt)],
+  expect: { type: spec.type, blocks: 1, reply: DRAWN }
+})
+
+const FLOW = { type: 'flowchart' } as const
+const STATE = { type: 'stateDiagram-v2' } as const
+
+// QVAC-24502: the doc is already in the window, so the skill body and answer share what is left
+const ONBOARDING_DOC = [
+  'ACME onboarding runbook, revision 7.',
+  '',
+  'A new hire is created in the HR system by the people team. That record syncs to the',
+  'directory overnight. The following morning IT receives a ticket, provisions a laptop from',
+  'stock, and images it with the standard build. The manager is emailed a checklist and must',
+  'confirm the desk assignment before the laptop is released. On day one the hire collects',
+  'the laptop from the front desk, signs the equipment register, and sets their password at',
+  'first login. Access to production systems is not granted at this point: the hire completes',
+  'security training first, which takes two days, and their manager then requests elevation.',
+  'The security team reviews the request against the role matrix and either grants access,',
+  'asks for more detail, or refuses it. A refusal returns to the manager with a reason.',
+  'Once access is granted the hire is added to the on-call rota after their first month.'
+].join('\n')
+
+export const CASES: ReadonlyArray<EvalCase> = [
+  // the ticket prompt, and the type it names
+  draws(TICKET_CASE, {
+    category: 'state',
+    prompt: 'Create a state diagram for vegetable inventory',
+    ...STATE
+  }),
+  draws('order-states', {
+    category: 'state',
+    prompt: 'show me the states an order goes through from checkout to delivery',
+    ...STATE
+  }),
+  draws('traffic-light', {
+    category: 'state',
+    prompt: 'diagram of a traffic light and how it changes',
+    ...STATE
+  }),
+
+  draws('coffee-flow', {
+    category: 'flowchart',
+    prompt: 'draw a flowchart of making coffee',
+    ...FLOW
+  }),
+  draws('login-flow', {
+    category: 'flowchart',
+    prompt: 'map out how a user logs in, including the failure case',
+    ...FLOW
+  }),
+  draws('deploy-pipeline', {
+    category: 'flowchart',
+    prompt: 'can you sketch our deploy pipeline left to right',
+    ...FLOW
+  }),
+  draws('support-triage', {
+    category: 'flowchart',
+    prompt: 'visualize how a support ticket gets triaged',
+    ...FLOW
+  }),
+  draws('org-chart', {
+    category: 'flowchart',
+    prompt: 'draw the reporting lines for a team of eight',
+    ...FLOW
+  }),
+
+  draws('rest-call', {
+    category: 'sequence',
+    prompt: 'show who calls whom when the app fetches a profile from the API',
+    type: 'sequenceDiagram'
+  }),
+  draws('oauth-handshake', {
+    category: 'sequence',
+    prompt: 'diagram the oauth handshake between browser, our server and the provider',
+    type: 'sequenceDiagram'
+  }),
+
+  draws('shop-schema', {
+    category: 'er',
+    prompt: 'draw the tables for a shop: customers, orders, line items',
+    type: 'erDiagram'
+  }),
+  draws('blog-schema', {
+    category: 'er',
+    prompt: 'diagram the relationships between posts, authors and comments',
+    type: 'erDiagram'
+  }),
+
+  draws('animal-classes', {
+    category: 'class',
+    prompt: 'show the class hierarchy for animals, dogs and cats',
+    type: 'classDiagram'
+  }),
+  draws('payment-classes', {
+    category: 'class',
+    prompt: 'diagram the payment classes and what they inherit from',
+    type: 'classDiagram'
+  }),
+
+  draws('sprint-plan', {
+    category: 'gantt',
+    prompt: 'give me a schedule for a three week sprint: design, build, test, release',
+    type: 'gantt'
+  }),
+  draws('house-move', {
+    category: 'gantt',
+    prompt: 'plan my house move over six weeks with durations',
+    type: 'gantt'
+  }),
+
+  draws('browser-share', {
+    category: 'pie',
+    prompt: 'chart the browser split: chrome 65, firefox 20, safari 15',
+    type: 'pie'
+  }),
+  draws('budget-split', {
+    category: 'pie',
+    prompt: 'show what share of my budget goes to rent, food and savings',
+    type: 'pie'
+  }),
+
+  draws('pm-mindmap', {
+    category: 'mindmap',
+    prompt: 'brainstorm project management as a mindmap',
+    type: 'mindmap'
+  }),
+  draws('launch-ideas', {
+    category: 'mindmap',
+    prompt: 'mindmap the ideas around our product launch',
+    type: 'mindmap'
+  }),
+
+  draws('apollo-timeline', {
+    category: 'timeline',
+    prompt: 'timeline of the apollo program from 1961 to 1972',
+    type: 'timeline'
+  }),
+  draws('company-history', {
+    category: 'timeline',
+    prompt: 'show the dated milestones of the company in order',
+    type: 'timeline'
+  }),
+
+  // the three failures the repair prompt names, in prompts whose natural labels contain them
+  draws('http-retry', {
+    category: 'hard-label',
+    prompt: 'draw how we send a request (HTTP) and retry it on failure',
+    ...FLOW
+  }),
+  draws('finish-step', {
+    category: 'hard-label',
+    prompt: 'flowchart: start, do the work, end',
+    ...FLOW
+  }),
+  draws('ratio-step', {
+    category: 'hard-label',
+    prompt: 'diagram a step that checks the ratio 2:1 before continuing',
+    ...FLOW
+  }),
+  draws('quoted-stage', {
+    category: 'hard-label',
+    prompt: 'draw a pipeline where one stage is called "review & sign-off"',
+    ...FLOW
+  }),
+
+  // a diagram already drawn, then an edit: the model must redraw the whole block
+  {
+    id: 'add-2fa',
+    category: 'iterate',
+    messages: [
+      user('draw a flowchart of the login flow'),
+      { role: 'tool', content: skillLoadedTurn(SKILL) },
+      assistant(
+        '```mermaid\nflowchart TD\n  A["User submits credentials"] --> B{"Valid?"}\n  B -->|yes| C["Start session"]\n  B -->|no| D["Show error"]\n```\n\nThe login flow.'
+      ),
+      user('add a two-factor step between validating and starting the session')
+    ],
+    expect: { type: 'flowchart', blocks: 1, reply: DRAWN }
+  },
+  {
+    id: 'turn-horizontal',
+    category: 'iterate',
+    messages: [
+      user('draw a flowchart of making coffee'),
+      { role: 'tool', content: skillLoadedTurn(SKILL) },
+      assistant(
+        '```mermaid\nflowchart TD\n  A["Boil water"] --> B["Add grounds"]\n  B --> C["Brew"]\n  C --> D["Pour"]\n```\n\nMaking coffee.'
+      ),
+      user('same thing but horizontal please')
+    ],
+    expect: { type: 'flowchart', blocks: 1, reply: DRAWN }
+  },
+
+  // the control every category needs: asks a diagram must NOT answer with a fence
+  {
+    id: 'onboarding-from-doc',
+    category: 'long-context',
+    messages: [user(`${ONBOARDING_DOC}\n\nGenerate a state diagram based on the doc above.`)],
+    expect: { type: 'stateDiagram-v2', blocks: 1, reply: DRAWN }
+  },
+  {
+    id: 'onboarding-flow-from-doc',
+    category: 'long-context',
+    messages: [user(`${ONBOARDING_DOC}\n\nDraw the onboarding process above as a flowchart.`)],
+    expect: { type: 'flowchart', blocks: 1, reply: DRAWN }
+  },
+  {
+    id: 'logo-request',
+    category: 'route-away',
+    messages: [user('design a logo for my coffee shop')],
+    expect: { type: '', blocks: 0, reply: NOT_DRAWN }
+  },
+  {
+    id: 'photo-request',
+    category: 'route-away',
+    messages: [user('make me a picture of a mountain at sunset')],
+    expect: { type: '', blocks: 0, reply: NOT_DRAWN }
+  },
+  {
+    id: 'revenue-plot',
+    category: 'route-away',
+    messages: [user('plot our monthly revenue for the last year')],
+    expect: { type: '', blocks: 0, reply: NOT_DRAWN }
+  }
+]