@qvac/harness 1.0.0-rc.2 → 1.0.0-rc.3

package/bench/evidence/redact-bun1.3.14.json

@@ -0,0 +1,92 @@
+{
+  "runtime": "bun",
+  "version": "1.3.14",
+  "adapter": "completion() production redactor + schema registration + awaited host onStep",
+  "warmups": 1,
+  "results": [
+    {
+      "pattern": "prose",
+      "bytes": 100000,
+      "samples": 3,
+      "medianCpuMs": 40.991,
+      "maxCpuMs": 50.851
+    },
+    {
+      "pattern": "prose",
+      "bytes": 500000,
+      "samples": 3,
+      "medianCpuMs": 92.336,
+      "maxCpuMs": 95.606
+    },
+    {
+      "pattern": "prose",
+      "bytes": 1000000,
+      "samples": 3,
+      "medianCpuMs": 178.887,
+      "maxCpuMs": 179.778
+    },
+    {
+      "pattern": "jwtPrefix",
+      "bytes": 100000,
+      "samples": 3,
+      "medianCpuMs": 26.832,
+      "maxCpuMs": 29.781
+    },
+    {
+      "pattern": "jwtPrefix",
+      "bytes": 500000,
+      "samples": 3,
+      "medianCpuMs": 100.205,
+      "maxCpuMs": 103.24
+    },
+    {
+      "pattern": "jwtPrefix",
+      "bytes": 1000000,
+      "samples": 3,
+      "medianCpuMs": 171.772,
+      "maxCpuMs": 194.774
+    },
+    {
+      "pattern": "markup",
+      "bytes": 100000,
+      "samples": 3,
+      "medianCpuMs": 28.644,
+      "maxCpuMs": 28.959
+    },
+    {
+      "pattern": "markup",
+      "bytes": 500000,
+      "samples": 3,
+      "medianCpuMs": 107.861,
+      "maxCpuMs": 110.075
+    },
+    {
+      "pattern": "markup",
+      "bytes": 1000000,
+      "samples": 3,
+      "medianCpuMs": 196.109,
+      "maxCpuMs": 204.343
+    },
+    {
+      "pattern": "punctuation",
+      "bytes": 100000,
+      "samples": 3,
+      "medianCpuMs": 54.76,
+      "maxCpuMs": 56.285
+    },
+    {
+      "pattern": "punctuation",
+      "bytes": 500000,
+      "samples": 3,
+      "medianCpuMs": 223.607,
+      "maxCpuMs": 224.801
+    },
+    {
+      "pattern": "punctuation",
+      "bytes": 1000000,
+      "samples": 3,
+      "medianCpuMs": 438.701,
+      "maxCpuMs": 447.244
+    }
+  ]
+}

package/bench/evidence/redact-node24.11.json

@@ -0,0 +1,92 @@
+{
+  "runtime": "node",
+  "version": "v24.11.0",
+  "adapter": "completion() production redactor + schema registration + awaited host onStep",
+  "warmups": 1,
+  "results": [
+    {
+      "pattern": "prose",
+      "bytes": 100000,
+      "samples": 3,
+      "medianCpuMs": 36.119,
+      "maxCpuMs": 37.424
+    },
+    {
+      "pattern": "prose",
+      "bytes": 500000,
+      "samples": 3,
+      "medianCpuMs": 113.27,
+      "maxCpuMs": 124.599
+    },
+    {
+      "pattern": "prose",
+      "bytes": 1000000,
+      "samples": 3,
+      "medianCpuMs": 223.539,
+      "maxCpuMs": 248.6
+    },
+    {
+      "pattern": "jwtPrefix",
+      "bytes": 100000,
+      "samples": 3,
+      "medianCpuMs": 32.039,
+      "maxCpuMs": 36.487
+    },
+    {
+      "pattern": "jwtPrefix",
+      "bytes": 500000,
+      "samples": 3,
+      "medianCpuMs": 116.359,
+      "maxCpuMs": 128.674
+    },
+    {
+      "pattern": "jwtPrefix",
+      "bytes": 1000000,
+      "samples": 3,
+      "medianCpuMs": 218.697,
+      "maxCpuMs": 220.754
+    },
+    {
+      "pattern": "markup",
+      "bytes": 100000,
+      "samples": 3,
+      "medianCpuMs": 31.789,
+      "maxCpuMs": 38.346
+    },
+    {
+      "pattern": "markup",
+      "bytes": 500000,
+      "samples": 3,
+      "medianCpuMs": 125.848,
+      "maxCpuMs": 148.413
+    },
+    {
+      "pattern": "markup",
+      "bytes": 1000000,
+      "samples": 3,
+      "medianCpuMs": 250.558,
+      "maxCpuMs": 256.972
+    },
+    {
+      "pattern": "punctuation",
+      "bytes": 100000,
+      "samples": 3,
+      "medianCpuMs": 33.315,
+      "maxCpuMs": 37.608
+    },
+    {
+      "pattern": "punctuation",
+      "bytes": 500000,
+      "samples": 3,
+      "medianCpuMs": 141.881,
+      "maxCpuMs": 154.584
+    },
+    {
+      "pattern": "punctuation",
+      "bytes": 1000000,
+      "samples": 3,
+      "medianCpuMs": 282.219,
+      "maxCpuMs": 282.905
+    }
+  ]
+}

package/bench/graph-metrics.ts

@@ -0,0 +1,43 @@
+import { CompletionMachine } from '../lib/machine/index.ts'
+
+interface Transition {
+  target?: string[]
+}
+interface Node {
+  type: string
+  states?: Record<string, Node>
+  on?: Record<string, Transition[]>
+  onDone?: Transition[]
+  onError?: Transition[]
+  always?: Transition[]
+  after?: Record<string, Transition[]> | Transition[]
+}
+
+interface Metrics {
+  states: number
+  transitions: number
+}
+interface CountOptions {
+  root?: boolean
+}
+
+function count(node: Node, { root = true }: CountOptions = {}): Metrics {
+  const transitions = [
+    ...Object.values(node.on ?? {}).flat(),
+    ...(node.onDone ?? []),
+    ...(node.onError ?? []),
+    ...(node.always ?? []),
+    ...(Array.isArray(node.after) ? node.after : Object.values(node.after ?? {}).flat())
+  ]
+  const nested = Object.values(node.states ?? {}).map((child) => count(child, { root: false }))
+
+  return {
+    states: (root || node.type === 'history' ? 0 : 1) + nested.reduce((n, x) => n + x.states, 0),
+    transitions: transitions.length + nested.reduce((n, x) => n + x.transitions, 0)
+  }
+}
+
+// lunte-disable-next-line qvac/no-as-cast -- xstate root nodes are untyped.
+const metrics = count(CompletionMachine.create().root as Node)
+console.log(`states: ${metrics.states}`)
+console.log(`transitions: ${metrics.transitions}`)

package/bench/harness.ts

@@ -0,0 +1,73 @@
+import process from 'process'
+import { Harness, loadSkillCatalog } from '../index.ts'
+import { ObsidianRoute } from '../plugins/obsidian/index.ts'
+import { Python } from '../plugins/python/index.ts'
+import { TranslationPlugin } from '../plugins/translation/index.ts'
+import { configureSdkStorage } from '../lib/sdk-storage.ts'
+import { MODEL_CACHE_PATH_ENV, resolveStorage } from '../lib/storage.ts'
+import { Tools } from '../lib/tools/index.ts'
+import { Weather } from '../plugins/weather/index.ts'
+import { WebSearchPlugin } from '../plugins/web-search/index.ts'
+import { McpPlugin } from '../plugins/mcp/index.ts'
+import { CodeMode } from '../plugins/code-mode/index.ts'
+import { QuickJs } from '../plugins/quickjs/index.ts'
+import type { HarnessPlugin } from '../lib/plugins/index.ts'
+import type { CredentialStoreFn } from '../lib/tools/skills/index.ts'
+
+export interface BenchHarnessOptions {
+  readonly root: string
+  readonly skillsDir: string
+  readonly contextSize?: number | undefined
+}
+
+export async function openBenchHarness(options: BenchHarnessOptions): Promise<Harness> {
+  const storage = resolveStorage('dev', {
+    root: options.root,
+    models: process.env[MODEL_CACHE_PATH_ENV]
+  })
+  configureSdkStorage(storage.sdk)
+  process.env.QVAC_PYODIDE_CACHE_DIR ??= `${storage.sdk.home}/pyodide-cache`
+
+  const { skillsDir } = options
+  let harness: Harness | null = null
+  const skillCatalog = await loadSkillCatalog(skillsDir)
+  const credentialStore: CredentialStoreFn = (key, ctx) =>
+    harness?.resolveCredential(key, ctx) ?? null
+  const tools = Tools.product({
+    sdkHome: storage.sdk.home,
+    skillsDir,
+    skillCatalog,
+    credentialStore
+  })
+  harness = new Harness({
+    tools,
+    exec: true,
+    skillsDir,
+    sdkHome: storage.sdk.home,
+    modelsDir: storage.sdk.models,
+    contextSize: options.contextSize,
+    plugins: [
+      new Python(),
+      new ObsidianRoute({ credentialStore }),
+      new Weather({ read: ['weather_lookup'] }),
+      new TranslationPlugin(),
+      new WebSearchPlugin({ read: ['web_search'] }),
+      new McpPlugin({ generic: true, skillCatalog, credentialStore, read: ['mcp_call'] }),
+      ...codeModePlugins()
+    ]
+  })
+  await harness.ready()
+
+  return harness
+}
+
+// QVAC_BENCH_CODE_MODE=1 adds run_code, which the sidecar does not compose
+function codeModePlugins(): HarnessPlugin[] {
+  if (process.env.QVAC_BENCH_CODE_MODE !== '1') {
+    return []
+  }
+
+  const quickjs = new QuickJs()
+
+  return [quickjs, new CodeMode({ quickjs, read: ['run_code'] })]
+}