@qvac/harness 1.0.0-rc.2 → 1.0.0-rc.3

package/bench/redact-adversarial.ts

@@ -0,0 +1,256 @@
+import process from 'process'
+import { CompletionEngine } from '../lib/capabilities/completion-engine/index.ts'
+import type { CompletionSdk, ToolDefinition } from '../lib/contract/index.ts'
+import { isRecord } from '../lib/guards.ts'
+import { RunLog, type EngineBoot, type EngineStep, type RunRecord } from '../lib/run-log/index.ts'
+
+const SIZES = [100_000, 500_000, 1_000_000]
+const PATTERNS = {
+  prose: 'A pagination cursor remains ordinary free text. ',
+  jwtPrefix: 'eyJ-',
+  markup: '<parameter=',
+  punctuation: 'the hyphen-token\\path stays byte-for-byte plain text - '
+} as const
+const SECRET = 'credential-value-must-not-be-persisted'
+const SAMPLES = 3
+const WARMUPS = 1
+const encoder = new TextEncoder()
+
+function runtimeInfo() {
+  const bun = Reflect.get(globalThis, 'Bun')
+  if (bun && typeof bun === 'object' && 'version' in bun && typeof bun.version === 'string') {
+    return { runtime: 'bun', version: bun.version }
+  }
+
+  return { runtime: 'node', version: process.version }
+}
+
+function stressText(pattern: string, bytes: number): string {
+  return pattern.repeat(Math.ceil(bytes / pattern.length)).slice(0, bytes)
+}
+
+function hasPair(value: unknown, key: string, expected: unknown): boolean {
+  if (Array.isArray(value)) {
+    return value.some((entry) => hasPair(entry, key, expected))
+  }
+
+  if (!isRecord(value)) return false
+
+  return Object.entries(value).some(
+    ([childKey, child]) =>
+      (childKey === key && (child === expected || (isRecord(child) && child.$ === expected))) ||
+      hasPair(child, key, expected)
+  )
+}
+
+function sdkFor(payload: string): CompletionSdk {
+  let round = 0
+  return {
+    completion: ({ tools }) => {
+      const toolCalls =
+        round === 0 && tools
+          ? [
+              {
+                id: 'sensitive-call',
+                name: 'credential_tool',
+                arguments: { credential: SECRET, password: 739184, token: 1024, note: payload }
+              }
+            ]
+          : []
+      const contentText = round++ === 0 ? '' : 'done'
+
+      return {
+        tokenStream: (async function* () {
+          if (contentText) {
+            yield contentText
+          }
+        })(),
+        final: Promise.resolve({ contentText, toolCalls, raw: { fullText: contentText } })
+      }
+    }
+  }
+}
+
+function createTool(execute: (args: Record<string, unknown>) => void): ToolDefinition {
+  return {
+    name: 'credential_tool',
+    description: 'Use the credential without displaying it',
+    parameters: {
+      type: 'object',
+      properties: {
+        credential: { type: 'string', sensitive: true },
+        password: { type: 'number' },
+        token: { type: 'number' },
+        note: { type: 'string' }
+      },
+      required: ['credential', 'note']
+    },
+    toolset: null,
+    autoApprove: true,
+    effect: () => 'read',
+    execute: (args) => {
+      execute(args)
+      return Promise.resolve({ content: payloadForResult(args) })
+    }
+  }
+}
+
+function payloadForResult(args: Record<string, unknown>): string {
+  return typeof args.note === 'string' ? args.note : ''
+}
+
+interface Observation {
+  records: RunRecord[]
+  boot: EngineBoot
+  executedSecret: boolean
+  executedPassword: boolean
+  executedToken: boolean
+  sawRedactedCredential: boolean
+  sawRedactedPassword: boolean
+  sawTypedToken: boolean
+}
+
+function createObservation() {
+  const state: Observation = {
+    records: [],
+    boot: RunLog.createEngineBoot(),
+    executedSecret: false,
+    executedPassword: false,
+    executedToken: false,
+    sawRedactedCredential: false,
+    sawRedactedPassword: false,
+    sawTypedToken: false
+  }
+  const tool = createTool((args) => {
+    state.executedSecret = args.credential === SECRET
+    state.executedPassword = args.password === 739184
+    state.executedToken = args.token === 1024
+  })
+  const onStep = (step: EngineStep) => observeStep(state, step)
+
+  return { state, tool, onStep }
+}
+
+function observeStep(state: Observation, step: EngineStep): Promise<void> {
+  const next = RunLog.applyEngineStep(state.boot, step)
+  if (step.expectedLength !== state.records.length) {
+    throw new Error('noncontiguous host step')
+  }
+
+  for (const record of step.records) {
+    if (hasPair(record, 'credential', SECRET)) {
+      throw new Error('A schema-marked credential reached typed durable fields')
+    }
+
+    if (hasPair(record, 'password', 739184)) {
+      throw new Error('A numeric password reached typed durable fields')
+    }
+
+    state.sawRedactedCredential ||= hasPair(record, 'credential', '[redacted]')
+    state.sawRedactedPassword ||= hasPair(record, 'password', '[redacted]')
+    state.sawTypedToken ||= hasPair(record, 'token', 1024)
+  }
+
+  state.records.push(...step.records)
+  state.boot = next
+
+  return Promise.resolve()
+}
+
+function assertObservation(state: Observation, label: string, payload: string): void {
+  if (!state.executedSecret) {
+    throw new Error('The tool did not receive its unredacted credential')
+  }
+
+  if (!state.executedPassword || !state.executedToken) {
+    throw new Error('Numeric tool arguments changed at runtime')
+  }
+
+  const metadata = state.records.find((record) => record.type === 'metadata')
+  if (!metadata || !isRecord(metadata.request)) {
+    throw new Error('Missing request metadata')
+  }
+
+  const messages = metadata.request.messages
+  if (!Array.isArray(messages) || !isRecord(messages[0]) || messages[0].content !== payload) {
+    throw new Error(`Free text changed for ${label}/${payload.length}`)
+  }
+
+  if (!state.sawRedactedCredential) {
+    throw new Error('The schema-marked tool argument was not redacted')
+  }
+
+  if (!state.sawRedactedPassword) {
+    throw new Error('A numeric password was not redacted')
+  }
+
+  if (!state.sawTypedToken) {
+    throw new Error('An ambiguous token count was over-redacted')
+  }
+}
+
+async function runOne(label: string, payload: string, sample: number): Promise<number> {
+  const { state, tool, onStep } = createObservation()
+  const started = process.cpuUsage()
+  for await (const _event of CompletionEngine.run(
+    sdkFor(payload),
+    {
+      runId: `redact-${label}-${payload.length}-${sample}`,
+      modelId: 'benchmark-model',
+      messages: [{ role: 'user', content: payload }],
+      tools: [tool]
+    },
+    { onStep, requestDigestKey: new Uint8Array(32).fill(7) }
+  )) {
+    // Consume the real completion iterator; the durable callback is the production redaction path.
+  }
+
+  const usage = process.cpuUsage(started)
+  assertObservation(state, label, payload)
+
+  return (usage.user + usage.system) / 1000
+}
+
+function median(values: number[]): number {
+  const ordered = [...values].sort((left, right) => left - right)
+  return ordered[Math.floor(ordered.length / 2)]
+}
+
+async function main(): Promise<void> {
+  const results: Array<Record<string, number | string>> = []
+  for (const [label, pattern] of Object.entries(PATTERNS)) {
+    for (const size of SIZES) {
+      const payload = stressText(pattern, size)
+      for (let sample = 0; sample < WARMUPS; sample++) {
+        await runOne(label, payload, -sample - 1)
+      }
+
+      const times: number[] = []
+      for (let sample = 0; sample < SAMPLES; sample++) {
+        times.push(await runOne(label, payload, sample))
+      }
+
+      results.push({
+        pattern: label,
+        bytes: encoder.encode(payload).byteLength,
+        samples: SAMPLES,
+        medianCpuMs: median(times),
+        maxCpuMs: Math.max(...times)
+      })
+    }
+  }
+
+  console.log(
+    JSON.stringify({
+      ...runtimeInfo(),
+      adapter: 'completion() production redactor + schema registration + awaited host onStep',
+      warmups: WARMUPS,
+      results
+    })
+  )
+}
+
+main().catch((error) => {
+  console.error(error)
+  process.exitCode = 1
+})

package/bench/repeat-cpu.mjs

@@ -0,0 +1,281 @@
+import { spawnSync } from 'node:child_process'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..')
+const engineBench = path.join(root, 'bench/cpu.ts')
+const mainBench = path.join(root, 'bench/cpu-main.mjs')
+const bunConfig = path.join(root, 'bench/bun-tsconfig.json')
+const options = parseOptions(process.argv.slice(2))
+const bench = options.driver === 'main' ? mainBench : engineBench
+
+const command =
+  options.runtime === 'bun'
+    ? { executable: 'bun', prefix: [`--tsconfig-override=${bunConfig}`] }
+    : { executable: process.execPath, prefix: ['--experimental-strip-types'] }
+
+const samples = []
+for (let i = 0; i < options.processes; i++) {
+  const args = [
+    ...command.prefix,
+    bench,
+    String(options.runs),
+    `--spacing=${options.spacing}`,
+    `--workload=${options.workload}`,
+    '--json'
+  ]
+  if (options.driver === 'engine') {
+    args.splice(args.length - 1, 0, `--mode=${options.mode}`)
+  }
+
+  const child = spawnSync(command.executable, args, {
+    cwd: root,
+    encoding: 'utf8'
+  })
+  if (child.error) {
+    throw child.error
+  }
+
+  if (child.status !== 0) {
+    throw new Error(
+      `${command.executable} exited with ${child.status}:\n${child.stderr || child.stdout}`
+    )
+  }
+
+  const lines = child.stdout.trim().split('\n')
+  const result = JSON.parse(lines.at(-1))
+  validateChildReport(result, i + 1)
+  samples.push({ process: i + 1, ...result })
+}
+
+validateProcessReports(samples)
+
+const keys = new Map()
+for (const sample of samples) {
+  for (const result of sample.results) {
+    const key = [result.workload, result.mode, result.spacing ?? ''].join(':')
+    if (!keys.has(key)) {
+      keys.set(key, [])
+    }
+
+    keys.get(key).push(result)
+  }
+}
+
+const medians = [...keys].map(([key, results]) => {
+  const first = results[0]
+  const aggregate = { workload: first.workload, mode: first.mode, spacing: first.spacing }
+
+  const cpuSamples = results.flatMap((result) => result.cpuSamplesMs)
+  aggregate.rawSampleCount = cpuSamples.length
+  aggregate.cpuMs = median(cpuSamples)
+  aggregate.processMedianMs = median(results.map((result) => result.cpuMs))
+  if (typeof first.usPerToken === 'number' && first.cpuMs > 0) {
+    aggregate.usPerToken = aggregate.cpuMs * (first.usPerToken / first.cpuMs)
+  }
+
+  for (const metric of ['logBytes', 'records', 'contentEvents']) {
+    const values = results.map((result) => result[metric]).filter(Number.isFinite)
+    if (values.length) {
+      aggregate[metric] = median(values)
+    }
+  }
+
+  return { key, ...aggregate }
+})
+
+console.log(
+  JSON.stringify({
+    runtime: samples[0].runtime,
+    version: samples[0].version,
+    driver: options.driver,
+    revision: options.revision,
+    processes: options.processes,
+    runsPerProcess: options.runs,
+    reduction: 'pooled median of all raw cpuSamplesMs; process medians are reported separately',
+    workload: options.workload,
+    samples,
+    medians
+  })
+)
+
+function validateOptions(options) {
+  if (!['node', 'bun'].includes(options.runtime)) {
+    throw new Error('runtime must be node or bun')
+  }
+
+  if (!['engine', 'main'].includes(options.driver)) {
+    throw new Error('driver must be engine or main')
+  }
+
+  if (!['tools', 'stream', 'all'].includes(options.workload)) {
+    throw new Error('workload must be tools, stream or all')
+  }
+
+  if (!['immediate', 'timeout', 'all'].includes(options.spacing)) {
+    throw new Error('spacing must be immediate, timeout or all')
+  }
+
+  if (!['plain', 'bench', 'production', 'all'].includes(options.mode)) {
+    throw new Error('mode must be plain, bench, production or all')
+  }
+
+  for (const key of ['processes', 'runs']) {
+    if (!Number.isInteger(options[key]) || options[key] < 1) {
+      throw new Error(`${key} must be a positive integer`)
+    }
+  }
+
+  return options
+}
+
+function expectedResultKeys() {
+  const workloads = options.workload === 'all' ? ['tools', 'stream'] : [options.workload]
+  let modes
+  if (options.driver === 'main') {
+    modes = ['main']
+  } else if (options.mode === 'all') {
+    modes = ['plain', 'bench', 'production']
+  } else {
+    modes = [options.mode]
+  }
+
+  const spacings = options.spacing === 'all' ? ['immediate', 'timeout'] : [options.spacing]
+  const expected = []
+
+  for (const mode of modes) {
+    for (const workload of workloads) {
+      const workloadSpacings = workload === 'tools' ? [''] : spacings
+      for (const spacing of workloadSpacings) {
+        expected.push([workload, mode, spacing].join(':'))
+      }
+    }
+  }
+
+  return expected.sort()
+}
+
+function resultKey(result) {
+  return [result.workload, result.mode, result.spacing ?? ''].join(':')
+}
+
+function validateChildReport(report, processNumber) {
+  if (!report || report.runtime !== options.runtime || typeof report.version !== 'string') {
+    throw new Error(`process ${processNumber} reported an unexpected runtime identity`)
+  }
+
+  if (report.runs !== options.runs || !Array.isArray(report.results)) {
+    throw new Error(`process ${processNumber} reported an unexpected run count or results list`)
+  }
+
+  const expectedKeys = expectedResultKeys()
+  const actualKeys = report.results.map(resultKey)
+  if (
+    actualKeys.length !== expectedKeys.length ||
+    new Set(actualKeys).size !== actualKeys.length ||
+    actualKeys.sort().join('\n') !== expectedKeys.join('\n')
+  ) {
+    throw new Error(
+      `process ${processNumber} returned an incomplete or inconsistent workload matrix`
+    )
+  }
+
+  for (const result of report.results) {
+    if (
+      result.runs !== options.runs ||
+      !Array.isArray(result.cpuSamplesMs) ||
+      result.cpuSamplesMs.length !== options.runs ||
+      !result.cpuSamplesMs.every((sample) => Number.isFinite(sample) && sample >= 0)
+    ) {
+      throw new Error(
+        `process ${processNumber} result ${resultKey(result)} did not preserve ${options.runs} finite raw CPU samples`
+      )
+    }
+
+    if (result.cpuMs !== median(result.cpuSamplesMs)) {
+      throw new Error(
+        `process ${processNumber} result ${resultKey(result)} has an invalid process median`
+      )
+    }
+  }
+}
+
+function validateProcessReports(samples) {
+  const expectedKeys = expectedResultKeys()
+  const referenceVersion = samples[0].version
+
+  for (const sample of samples) {
+    if (sample.version !== referenceVersion) {
+      throw new Error('benchmark processes reported different runtime versions')
+    }
+
+    const keys = sample.results.map(resultKey).sort()
+    if (keys.join('\n') !== expectedKeys.join('\n')) {
+      throw new Error('benchmark processes returned different workload matrices')
+    }
+  }
+}
+
+function applyOption(options, arg) {
+  const [key, value] = arg.split('=', 2)
+  const values = {
+    '--runtime': () => {
+      options.runtime = value
+    },
+    '--driver': () => {
+      options.driver = value
+    },
+    '--processes': () => {
+      options.processes = Number(value)
+    },
+    '--runs': () => {
+      options.runs = Number(value)
+    },
+    '--mode': () => {
+      options.mode = value
+    },
+    '--spacing': () => {
+      options.spacing = value
+    },
+    '--workload': () => {
+      options.workload = value
+    },
+    '--revision': () => {
+      options.revision = value
+    }
+  }
+  const setValue = values[key]
+  if (!setValue) {
+    throw new Error(`Unknown option: ${arg}`)
+  }
+
+  setValue()
+}
+
+function parseOptions(args) {
+  const options = {
+    runtime: 'node',
+    driver: 'engine',
+    processes: 3,
+    runs: 3,
+    mode: 'all',
+    spacing: 'all',
+    workload: 'all',
+    revision: ''
+  }
+  for (const arg of args) {
+    applyOption(options, arg)
+  }
+
+  return validateOptions(options)
+}
+
+function median(values) {
+  if (values.length === 0) {
+    throw new Error('Cannot calculate a median without samples')
+  }
+
+  const sorted = [...values].sort((a, b) => a - b)
+
+  return sorted[Math.floor(sorted.length / 2)]
+}