@attest-ai/core 0.6.0 → 1.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/index.cjs +890 -127
- package/dist/cjs/index.cjs.map +1 -1
- package/dist/cjs/index.d.cts +158 -16
- package/dist/cjs/proto/index.cjs +117 -0
- package/dist/cjs/proto/index.cjs.map +1 -1
- package/dist/cjs/proto/index.d.cts +135 -1
- package/dist/esm/cli.d.ts +19 -4
- package/dist/esm/cli.js +478 -116
- package/dist/esm/cli.js.map +1 -1
- package/dist/esm/index.d.ts +158 -16
- package/dist/esm/index.js +870 -128
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/proto/index.d.ts +135 -1
- package/dist/esm/proto/index.js +103 -0
- package/dist/esm/proto/index.js.map +1 -1
- package/package.json +4 -3
package/dist/esm/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/cli.ts","../../src/version.ts"],"sourcesContent":["#!/usr/bin/env node\n/**\n * CLI entry point for Attest: npx attest or attest (after global install).\n *\n * Ported from Python sdks/python/src/attest/__main__.py\n */\n\nimport * as fs from \"node:fs\";\nimport * as path from \"node:path\";\nimport * as os from \"node:os\";\nimport { VERSION } from \"./version.js\";\n\nfunction cacheDir(): string {\n const envOverride = process.env[\"ATTEST_CACHE_DIR\"];\n if (envOverride) return envOverride;\n return path.join(os.homedir(), \".attest\", \"cache\");\n}\n\nfunction cacheDbPath(): string {\n return path.join(cacheDir(), \"attest.db\");\n}\n\nfunction cmdCacheStats(): void {\n const dbPath = cacheDbPath();\n const exists = fs.existsSync(dbPath);\n let fileSize = 0;\n if (exists) {\n fileSize = fs.statSync(dbPath).size;\n }\n const stats = { exists, file_size: fileSize, path: dbPath };\n process.stdout.write(JSON.stringify(stats) + \"\\n\");\n}\n\nfunction cmdCacheClear(): void {\n const dbPath = cacheDbPath();\n if (fs.existsSync(dbPath)) {\n fs.unlinkSync(dbPath);\n process.stdout.write(`Cleared cache: ${dbPath}\\n`);\n } else {\n process.stdout.write(`No cache to clear: ${dbPath}\\n`);\n }\n}\n\nconst CONFTEST_TEMPLATE = `import { describe, it, expect } from \"vitest\";\nimport { attestExpect, TraceBuilder } from \"@attest-ai/core\";\n\ndescribe(\"my agent\", () => {\n it(\"returns expected output\", async () => {\n const trace = new TraceBuilder(\"my-agent\")\n .setInput({ user_message: \"hello\" })\n .setOutput({ message: \"world\" })\n .build();\n\n attestExpect(trace).outputContains(\"world\");\n });\n});\n`;\n\nfunction cmdInit(targetDir: string): void {\n const testsDir = path.join(targetDir, \"tests\");\n fs.mkdirSync(testsDir, { recursive: true });\n process.stdout.write(`Directory: ${testsDir}\\n`);\n\n const sampleTestPath = path.join(testsDir, \"test_my_agent.test.ts\");\n if (fs.existsSync(sampleTestPath)) {\n process.stderr.write(`Skipped (exists): ${sampleTestPath}\\n`);\n } else {\n fs.writeFileSync(sampleTestPath, CONFTEST_TEMPLATE);\n process.stdout.write(`Created: ${sampleTestPath}\\n`);\n }\n}\n\nfunction cmdValidate(targetDir: string): void {\n const testsDir = path.join(targetDir, \"tests\");\n\n let testFiles: string[] = [];\n if (fs.existsSync(testsDir)) {\n testFiles = fs\n .readdirSync(testsDir)\n .filter((f) => f.endsWith(\".test.ts\") || f.endsWith(\".test.js\"));\n }\n\n if (testFiles.length === 0) {\n process.stderr.write(\n `Warning: no test files (*.test.ts / *.test.js) found in ${testsDir}.\\n`,\n );\n } else {\n process.stdout.write(`Found ${testFiles.length} test file(s) in ${testsDir}\\n`);\n }\n\n // Check for stale golden traces\n const STALE_DAYS = 30;\n const now = Date.now();\n const goldenFiles = findGoldenFiles(targetDir);\n for (const golden of goldenFiles) {\n const mtime = fs.statSync(golden).mtimeMs;\n const ageDays = Math.floor((now - mtime) / (1000 * 60 * 60 * 24));\n if (ageDays > STALE_DAYS) {\n process.stderr.write(\n `Warning: golden trace ${golden} is ${ageDays} days old (>${STALE_DAYS} days). Consider regenerating.\\n`,\n );\n }\n }\n}\n\nfunction findGoldenFiles(dir: string): string[] {\n const results: string[] = [];\n if (!fs.existsSync(dir)) return results;\n\n const entries = fs.readdirSync(dir, { withFileTypes: true });\n for (const entry of entries) {\n const fullPath = path.join(dir, entry.name);\n if (entry.isDirectory() && entry.name !== \"node_modules\") {\n results.push(...findGoldenFiles(fullPath));\n } else if (entry.isFile() && entry.name.endsWith(\".golden\")) {\n results.push(fullPath);\n }\n }\n return results;\n}\n\nexport function main(argv: string[] = process.argv.slice(2)): void {\n const args = argv;\n\n if (args[0] === \"--version\") {\n process.stdout.write(`attest ${VERSION}\\n`);\n process.exit(0);\n }\n\n if (args[0] === \"cache\" && args.length >= 2) {\n if (args[1] === \"stats\") {\n cmdCacheStats();\n process.exit(0);\n } else if (args[1] === \"clear\") {\n cmdCacheClear();\n process.exit(0);\n } else {\n process.stderr.write(`Unknown cache subcommand: ${args[1]}\\n`);\n process.stderr.write(\"Available: cache stats, cache clear\\n\");\n process.exit(1);\n }\n }\n\n if (args[0] === \"init\") {\n cmdInit(process.cwd());\n process.exit(0);\n }\n\n if (args[0] === \"validate\") {\n cmdValidate(process.cwd());\n process.exit(0);\n }\n\n process.stderr.write(\n \"Usage: attest <command>\\n\\n\" +\n \"Commands:\\n\" +\n \" --version Print version\\n\" +\n \" init Scaffold a test project\\n\" +\n \" validate Validate test suite\\n\" +\n \" cache stats Show cache statistics\\n\" +\n \" cache clear Clear cache database\\n\",\n );\n process.exit(args.length === 0 ? 0 : 1);\n}\n\n// Run when executed directly\nconst isDirectExecution =\n typeof import.meta.url === \"string\" &&\n import.meta.url.startsWith(\"file:\") &&\n process.argv[1] !== undefined &&\n import.meta.url.endsWith(path.basename(process.argv[1]));\n\nif (isDirectExecution) {\n main();\n}\n","export const VERSION = \"0.6.0\";\nexport const ENGINE_VERSION = \"0.6.0\";\n"],"mappings":";;;AAOA,YAAY,QAAQ;AACpB,YAAY,UAAU;AACtB,YAAY,QAAQ;;;ACTb,IAAM,UAAU;;;ADYvB,SAAS,WAAmB;AAC1B,QAAM,cAAc,QAAQ,IAAI,kBAAkB;AAClD,MAAI,YAAa,QAAO;AACxB,SAAY,UAAQ,WAAQ,GAAG,WAAW,OAAO;AACnD;AAEA,SAAS,cAAsB;AAC7B,SAAY,UAAK,SAAS,GAAG,WAAW;AAC1C;AAEA,SAAS,gBAAsB;AAC7B,QAAM,SAAS,YAAY;AAC3B,QAAM,SAAY,cAAW,MAAM;AACnC,MAAI,WAAW;AACf,MAAI,QAAQ;AACV,eAAc,YAAS,MAAM,EAAE;AAAA,EACjC;AACA,QAAM,QAAQ,EAAE,QAAQ,WAAW,UAAU,MAAM,OAAO;AAC1D,UAAQ,OAAO,MAAM,KAAK,UAAU,KAAK,IAAI,IAAI;AACnD;AAEA,SAAS,gBAAsB;AAC7B,QAAM,SAAS,YAAY;AAC3B,MAAO,cAAW,MAAM,GAAG;AACzB,IAAG,cAAW,MAAM;AACpB,YAAQ,OAAO,MAAM,kBAAkB,MAAM;AAAA,CAAI;AAAA,EACnD,OAAO;AACL,YAAQ,OAAO,MAAM,sBAAsB,MAAM;AAAA,CAAI;AAAA,EACvD;AACF;AAEA,IAAM,oBAAoB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAe1B,SAAS,QAAQ,WAAyB;AACxC,QAAM,WAAgB,UAAK,WAAW,OAAO;AAC7C,EAAG,aAAU,UAAU,EAAE,WAAW,KAAK,CAAC;AAC1C,UAAQ,OAAO,MAAM,cAAc,QAAQ;AAAA,CAAI;AAE/C,QAAM,iBAAsB,UAAK,UAAU,uBAAuB;AAClE,MAAO,cAAW,cAAc,GAAG;AACjC,YAAQ,OAAO,MAAM,qBAAqB,cAAc;AAAA,CAAI;AAAA,EAC9D,OAAO;AACL,IAAG,iBAAc,gBAAgB,iBAAiB;AAClD,YAAQ,OAAO,MAAM,YAAY,cAAc;AAAA,CAAI;AAAA,EACrD;AACF;AAEA,SAAS,YAAY,WAAyB;AAC5C,QAAM,WAAgB,UAAK,WAAW,OAAO;AAE7C,MAAI,YAAsB,CAAC;AAC3B,MAAO,cAAW,QAAQ,GAAG;AAC3B,gBACG,eAAY,QAAQ,EACpB,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,KAAK,EAAE,SAAS,UAAU,CAAC;AAAA,EACnE;AAEA,MAAI,UAAU,WAAW,GAAG;AAC1B,YAAQ,OAAO;AAAA,MACb,2DAA2D,QAAQ;AAAA;AAAA,IACrE;AAAA,EACF,OAAO;AACL,YAAQ,OAAO,MAAM,SAAS,UAAU,MAAM,oBAAoB,QAAQ;AAAA,CAAI;AAAA,EAChF;AAGA,QAAM,aAAa;AACnB,QAAM,MAAM,KAAK,IAAI;AACrB,QAAM,cAAc,gBAAgB,SAAS;AAC7C,aAAW,UAAU,aAAa;AAChC,UAAM,QAAW,YAAS,MAAM,EAAE;AAClC,UAAM,UAAU,KAAK,OAAO,MAAM,UAAU,MAAO,KAAK,KAAK,GAAG;AAChE,QAAI,UAAU,YAAY;AACxB,cAAQ,OAAO;AAAA,QACb,yBAAyB,MAAM,OAAO,OAAO,eAAe,UAAU;AAAA;AAAA,MACxE;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,gBAAgB,KAAuB;AAC9C,QAAM,UAAoB,CAAC;AAC3B,MAAI,CAAI,cAAW,GAAG,EAAG,QAAO;AAEhC,QAAM,UAAa,eAAY,KAAK,EAAE,eAAe,KAAK,CAAC;AAC3D,aAAW,SAAS,SAAS;AAC3B,UAAM,WAAgB,UAAK,KAAK,MAAM,IAAI;AAC1C,QAAI,MAAM,YAAY,KAAK,MAAM,SAAS,gBAAgB;AACxD,cAAQ,KAAK,GAAG,gBAAgB,QAAQ,CAAC;AAAA,IAC3C,WAAW,MAAM,OAAO,KAAK,MAAM,KAAK,SAAS,SAAS,GAAG;AAC3D,cAAQ,KAAK,QAAQ;AAAA,IACvB;AAAA,EACF;AACA,SAAO;AACT;AAEO,SAAS,KAAK,OAAiB,QAAQ,KAAK,MAAM,CAAC,GAAS;AACjE,QAAM,OAAO;AAEb,MAAI,KAAK,CAAC,MAAM,aAAa;AAC3B,YAAQ,OAAO,MAAM,UAAU,OAAO;AAAA,CAAI;AAC1C,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,MAAI,KAAK,CAAC,MAAM,WAAW,KAAK,UAAU,GAAG;AAC3C,QAAI,KAAK,CAAC,MAAM,SAAS;AACvB,oBAAc;AACd,cAAQ,KAAK,CAAC;AAAA,IAChB,WAAW,KAAK,CAAC,MAAM,SAAS;AAC9B,oBAAc;AACd,cAAQ,KAAK,CAAC;AAAA,IAChB,OAAO;AACL,cAAQ,OAAO,MAAM,6BAA6B,KAAK,CAAC,CAAC;AAAA,CAAI;AAC7D,cAAQ,OAAO,MAAM,uCAAuC;AAC5D,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF;AAEA,MAAI,KAAK,CAAC,MAAM,QAAQ;AACtB,YAAQ,QAAQ,IAAI,CAAC;AACrB,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,MAAI,KAAK,CAAC,MAAM,YAAY;AAC1B,gBAAY,QAAQ,IAAI,CAAC;AACzB,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,UAAQ,OAAO;AAAA,IACb;AAAA,EAOF;AACA,UAAQ,KAAK,KAAK,WAAW,IAAI,IAAI,CAAC;AACxC;AAGA,IAAM,oBACJ,OAAO,YAAY,QAAQ,YAC3B,YAAY,IAAI,WAAW,OAAO,KAClC,QAAQ,KAAK,CAAC,MAAM,UACpB,YAAY,IAAI,SAAc,cAAS,QAAQ,KAAK,CAAC,CAAC,CAAC;AAEzD,IAAI,mBAAmB;AACrB,OAAK;AACP;","names":[]}
|
|
1
|
+
{"version":3,"sources":["../../src/cli.ts","../../src/version.ts","../../src/engine-manager.ts","../../src/engine-downloader.ts","../../src/scaffold.ts"],"sourcesContent":["#!/usr/bin/env node\n/**\n * The `attest` command line (`npx attest`).\n *\n * `attest init` runs in the SDK. Every other command (`doctor`, `baseline`, `policy`,\n * `cache`, `calibrate`, `version` and any command a later engine adds) runs in\n * `attest-engine`: the CLI resolves a version-checked engine with `resolveEngine`, checks\n * that the engine's `help` lists the command, and forwards the arguments, stdio and exit\n * code unchanged. A missing, mismatched or too-old engine exits 1 before anything runs, so\n * a gate such as `attest policy evaluate` never exits 0 without evaluating.\n */\n\nimport { spawnSync } from \"node:child_process\";\nimport * as fs from \"node:fs\";\nimport * as os from \"node:os\";\nimport { fileURLToPath } from \"node:url\";\nimport { ENGINE_VERSION, VERSION } from \"./version.js\";\nimport { resolveEngine } from \"./engine-manager.js\";\nimport { scaffoldProject } from \"./scaffold.js\";\n\nconst HELP_FLAGS = new Set([\"-h\", \"--help\"]);\n\nconst SDK_USAGE =\n \"usage: attest <command> [args]\\n\" +\n \"\\n\" +\n \"SDK commands:\\n\" +\n \" init [--ci github] scaffold a sample test and merge policy here\\n\" +\n \" (--ci github also writes .github/workflows/attest.yml)\\n\" +\n \"\\n\" +\n \"options:\\n\" +\n \" --version print the SDK version and the engine version it requires\\n\" +\n \" --help print this help\\n\";\n\nconst INIT_USAGE = \"usage: attest init [--ci github] (scaffolds into the current directory)\";\n\n/** Run `attest init [--ci github]` in the current directory. */\nfunction cmdInit(args: string[]): number {\n if (args.some((a) => HELP_FLAGS.has(a))) {\n process.stdout.write(`${INIT_USAGE}\\n`);\n return 0;\n }\n const joined = args.join(\" \");\n if (args.length > 0 && joined !== \"--ci github\" && joined !== \"--ci=github\") {\n process.stderr.write(`${INIT_USAGE}\\n`);\n return 2;\n }\n scaffoldProject(process.cwd(), args.length > 0 ? { ci: \"github\" } : {});\n return 0;\n}\n\nconst HELP_PROBE_TIMEOUT_MS = 10_000;\n\nfunction errorMessage(err: unknown): string {\n return err instanceof Error ? err.message : String(err);\n}\n\n/**\n * `{command: summary}` from `<engine> help`, or null if it lists none.\n *\n * stdin is closed and the probe times out after 10 s, so an engine that predates the\n * command registry (and starts the JSON-RPC server for `help`) cannot hang the CLI.\n */\nexport function engineCommands(\n engine: string,\n timeoutMs: number = HELP_PROBE_TIMEOUT_MS,\n): Map<string, string> | null {\n const proc = spawnSync(engine, [\"help\"], {\n stdio: [\"ignore\", \"pipe\", \"pipe\"],\n encoding: \"utf-8\",\n timeout: timeoutMs,\n });\n if (proc.error || proc.status !== 0) return null;\n const lines = proc.stdout.split(\"\\n\");\n const start = lines.indexOf(\"commands:\");\n if (start === -1) return null;\n const commands = new Map<string, string>();\n for (const line of lines.slice(start + 1)) {\n const match = /^\\s*(\\S+)\\s*(.*)$/.exec(line);\n if (match?.[1]) commands.set(match[1], (match[2] ?? \"\").trim());\n }\n return commands;\n}\n\n/** Print SDK commands, plus engine commands when an engine is available without download. */\nasync function printHelp(): Promise<number> {\n process.stdout.write(`${SDK_USAGE}\\n`);\n let engine: string;\n try {\n engine = await resolveEngine(undefined, { allowDownload: false });\n } catch (err) {\n const firstLine = (errorMessage(err).split(\"\\n\")[0] ?? \"\").replace(/\\.$/, \"\");\n process.stdout.write(\n `engine commands: not available (${firstLine}). Run \\`attest doctor\\` to download ` +\n `attest-engine ${ENGINE_VERSION}, or set ATTEST_ENGINE_PATH.\\n`,\n );\n return 0;\n }\n const commands = engineCommands(engine);\n if (commands === null) {\n process.stdout.write(\n `engine commands: not available (attest-engine at ${engine} lists no commands; ` +\n \"it predates the command registry).\\n\",\n );\n return 0;\n }\n const width = Math.max(0, ...[...commands.keys()].map((name) => name.length));\n process.stdout.write(`engine commands (attest-engine ${ENGINE_VERSION}, ${engine}):\\n`);\n for (const [name, summary] of commands) {\n process.stdout.write(` ${name.padEnd(width)} ${summary}\\n`);\n }\n return 0;\n}\n\n/** Map a child's exit to a shell exit status (signal N → 128 + N). */\nexport function exitStatus(status: number | null, signal: NodeJS.Signals | null): number {\n if (status !== null) return status;\n if (signal !== null) return 128 + (os.constants.signals[signal] ?? 0);\n return 1;\n}\n\nasync function forward(args: string[]): Promise<number> {\n let engine: string;\n try {\n engine = await resolveEngine();\n } catch (err) {\n process.stderr.write(`attest: ${errorMessage(err)}\\n`);\n return 1;\n }\n const command = args[0] ?? \"\";\n const commands = engineCommands(engine);\n if (commands === null || (command !== \"help\" && !commands.has(command))) {\n process.stderr.write(\n `attest: attest-engine at ${engine} (version ${ENGINE_VERSION}) predates ` +\n `\\`attest ${command}\\`: its help does not list that command. Set ATTEST_ENGINE_PATH ` +\n \"to an engine built from this release (make engine), or install a release whose \" +\n \"engine provides it.\\n\",\n );\n return 1;\n }\n const proc = spawnSync(engine, args, { stdio: \"inherit\" });\n if (proc.error) throw proc.error;\n return exitStatus(proc.status, proc.signal);\n}\n\n/** Dispatch `attest` arguments and resolve to the process exit code. */\nexport async function main(argv: string[] = process.argv.slice(2)): Promise<number> {\n const [command] = argv;\n if (command === undefined || HELP_FLAGS.has(command) || (command === \"help\" && argv.length === 1)) {\n return printHelp();\n }\n if (command === \"--version\") {\n process.stdout.write(`attest ${VERSION} (requires attest-engine ${ENGINE_VERSION})\\n`);\n return 0;\n }\n if (command === \"init\") {\n return cmdInit(argv.slice(1));\n }\n if (command.startsWith(\"-\")) {\n process.stderr.write(`attest: unknown option ${command}\\n\\n${SDK_USAGE}`);\n return 2;\n }\n return forward(argv);\n}\n\n// Run when executed directly, including through the node_modules/.bin/attest symlink.\nfunction isDirectExecution(): boolean {\n const script = process.argv[1];\n if (script === undefined || !fs.existsSync(script)) return false;\n return fs.realpathSync(script) === fs.realpathSync(fileURLToPath(import.meta.url));\n}\n\nif (isDirectExecution()) {\n main().then(\n (code) => process.exit(code),\n (err: unknown) => {\n process.stderr.write(`attest: ${errorMessage(err)}\\n`);\n process.exit(1);\n },\n );\n}\n","export const VERSION = \"1.0.0-rc.1\";\nexport const ENGINE_VERSION = \"1.0.0-rc.1\";\n","import { spawn, spawnSync, type ChildProcess } from \"node:child_process\";\nimport { createInterface, type Interface as ReadlineInterface } from \"node:readline\";\nimport { existsSync, accessSync, statSync, constants as fsConstants } from \"node:fs\";\nimport { delimiter, join } from \"node:path\";\nimport type { InitializeResult } from \"./proto/types.js\";\nimport { decodeResponse, encodeRequest, extractResult } from \"./proto/codec.js\";\nimport { ENGINE_VERSION, VERSION } from \"./version.js\";\nimport { binaryFilename, cachedEnginePath, downloadEngine } from \"./engine-downloader.js\";\n\nconst ENGINE_BINARY_NAME = \"attest-engine\";\nconst VERSION_PROBE_TIMEOUT_MS = 10_000;\n\n/** A discovered engine does not report the version this SDK requires. */\nexport class EngineVersionMismatchError extends Error {\n readonly enginePath: string;\n readonly source: string;\n readonly found: string | null;\n readonly required: string;\n\n constructor(enginePath: string, source: string, found: string | null, required: string) {\n const reported = found === null ? \"reports no version\" : `reports version ${found}`;\n super(\n `attest-engine at ${enginePath} (from ${source}) ${reported}; ` +\n `this SDK requires attest-engine ${required}. ` +\n \"Point ATTEST_ENGINE_PATH at a matching engine, or remove the mismatched one \" +\n `from PATH so the SDK can download ${required} into ~/.attest/bin/${required}/.`,\n );\n this.name = \"EngineVersionMismatchError\";\n this.enginePath = enginePath;\n this.source = source;\n this.found = found;\n this.required = required;\n }\n}\n\n/**\n * The version `<path> version` reports, or null if it reports none.\n *\n * Every engine since 0.4 answers `version` with `attest-engine <version>`. stdin is\n * closed so a binary that starts the JSON-RPC server instead reads EOF and exits.\n */\nexport function engineVersion(enginePath: string): string | null {\n const proc = spawnSync(enginePath, [\"version\"], {\n stdio: [\"ignore\", \"pipe\", \"pipe\"],\n encoding: \"utf-8\",\n timeout: VERSION_PROBE_TIMEOUT_MS,\n });\n if (proc.error) throw proc.error;\n const out = proc.stdout.trim();\n const prefix = `${ENGINE_BINARY_NAME} `;\n if (proc.status !== 0 || !out.startsWith(prefix)) return null;\n return out.slice(prefix.length).trim();\n}\n\nfunction onPath(): string | null {\n for (const dir of (process.env[\"PATH\"] ?? \"\").split(delimiter)) {\n if (!dir) continue;\n const candidate = join(dir, binaryFilename());\n try {\n accessSync(candidate, fsConstants.X_OK);\n } catch {\n continue;\n }\n if (statSync(candidate).isFile()) return candidate;\n }\n return null;\n}\n\nfunction requireFile(enginePath: string, label: string): void {\n if (!existsSync(enginePath) || !statSync(enginePath).isFile()) {\n throw new Error(`${label}=${enginePath} does not exist or is not a file.`);\n }\n}\n\nasync function locateEngine(\n explicitPath: string | undefined,\n allowDownload: boolean,\n): Promise<[string, string]> {\n if (explicitPath) {\n requireFile(explicitPath, \"Engine path\");\n return [explicitPath, \"explicit engine path\"];\n }\n const envPath = process.env[\"ATTEST_ENGINE_PATH\"];\n if (envPath) {\n requireFile(envPath, \"ATTEST_ENGINE_PATH\");\n return [envPath, \"ATTEST_ENGINE_PATH\"];\n }\n const found = onPath();\n if (found) return [found, \"PATH\"];\n const cached = cachedEnginePath();\n if (cached) return [cached, \"engine cache\"];\n\n if (!allowDownload) {\n throw new Error(\n `No '${ENGINE_BINARY_NAME}' ${ENGINE_VERSION} in ATTEST_ENGINE_PATH, PATH or ` +\n `~/.attest/bin/${ENGINE_VERSION}/.`,\n );\n }\n\n const noDownload = process.env[\"ATTEST_ENGINE_NO_DOWNLOAD\"]?.toLowerCase();\n if (noDownload === \"1\" || noDownload === \"true\" || noDownload === \"yes\") {\n throw new Error(\n `Cannot find '${ENGINE_BINARY_NAME}' ${ENGINE_VERSION} and ATTEST_ENGINE_NO_DOWNLOAD is set.\\n` +\n \"Install options:\\n\" +\n \" 1. Build from source: make engine, then set ATTEST_ENGINE_PATH\\n\" +\n ` 2. Download release v${ENGINE_VERSION}: https://github.com/attest-framework/attest/releases\\n` +\n \" 3. Allow auto-download by unsetting ATTEST_ENGINE_NO_DOWNLOAD\",\n );\n }\n return [await downloadEngine(), \"download\"];\n}\n\n/**\n * Find an engine whose version matches this SDK and return its path.\n *\n * The single discovery function behind `EngineManager` and the `attest` CLI. Order:\n * explicit path → `ATTEST_ENGINE_PATH` → `attest-engine` on `PATH` →\n * `~/.attest/bin/<ENGINE_VERSION>/` → download (unless `ATTEST_ENGINE_NO_DOWNLOAD` is set or\n * `allowDownload` is false).\n * The first engine found must report exactly `ENGINE_VERSION`; a mismatched engine is\n * never skipped in favor of the next location.\n */\nexport async function resolveEngine(\n explicitPath?: string,\n options: { allowDownload?: boolean } = {},\n): Promise<string> {\n const [enginePath, source] = await locateEngine(explicitPath, options.allowDownload ?? true);\n const found = engineVersion(enginePath);\n if (found !== ENGINE_VERSION) {\n throw new EngineVersionMismatchError(enginePath, source, found, ENGINE_VERSION);\n }\n return enginePath;\n}\n\nexport class EngineManager {\n private enginePath: string;\n private readonly logLevel: string;\n private process: ChildProcess | null = null;\n private reader: ReadlineInterface | null = null;\n private initialized = false;\n private requestId = 0;\n private initResult: InitializeResult | null = null;\n\n constructor(options?: { enginePath?: string; logLevel?: string }) {\n this.enginePath = options?.enginePath ?? \"\";\n this.logLevel = options?.logLevel ?? \"warn\";\n }\n\n async start(): Promise<InitializeResult> {\n this.enginePath = await resolveEngine(this.enginePath || undefined);\n\n this.process = spawn(this.enginePath, [`--log-level=${this.logLevel}`], {\n stdio: [\"pipe\", \"pipe\", \"pipe\"],\n });\n\n this.reader = createInterface({ input: this.process.stdout! });\n\n try {\n this.initResult = await this.initialize();\n } catch (err) {\n await this.stop();\n throw err;\n }\n this.initialized = true;\n return this.initResult;\n }\n\n /** Send initialize and check the engine's version and capabilities. */\n private async initialize(): Promise<InitializeResult> {\n const init = (await this.sendRequestInternal(\"initialize\", {\n sdk_name: \"attest-typescript\",\n sdk_version: VERSION,\n protocol_version: 1,\n required_capabilities: [\"layers_1_4\"],\n preferred_encoding: \"json\",\n })) as InitializeResult;\n if (init.engine_version !== ENGINE_VERSION) {\n throw new EngineVersionMismatchError(\n this.enginePath,\n \"initialize\",\n init.engine_version,\n ENGINE_VERSION,\n );\n }\n if (!init.compatible) {\n throw new Error(`Engine incompatible. Missing capabilities: ${JSON.stringify(init.missing)}`);\n }\n return init;\n }\n\n async stop(): Promise<void> {\n if (this.process === null) return;\n\n if (this.initialized) {\n try {\n await this.sendRequestInternal(\"shutdown\", {});\n } catch {\n // shutdown failed, will kill\n }\n }\n\n if (this.reader) {\n this.reader.close();\n this.reader = null;\n }\n\n if (this.process.exitCode === null) {\n this.process.kill(\"SIGTERM\");\n await new Promise<void>((res) => {\n const timer = setTimeout(() => {\n this.process?.kill(\"SIGKILL\");\n }, 5000);\n this.process!.on(\"exit\", () => {\n clearTimeout(timer);\n res();\n });\n });\n }\n\n this.initialized = false;\n this.process = null;\n }\n\n async sendRequest(method: string, params: Record<string, unknown>): Promise<unknown> {\n if (!this.initialized && method !== \"initialize\") {\n throw new Error(\"Engine not initialized. Call start() first.\");\n }\n return this.sendRequestInternal(method, params);\n }\n\n private async sendRequestInternal(method: string, params: Record<string, unknown>): Promise<unknown> {\n if (this.process === null || this.process.stdin === null || this.reader === null) {\n throw new Error(\"Engine process not started.\");\n }\n\n this.requestId += 1;\n const requestStr = encodeRequest(this.requestId, method, params);\n\n return new Promise<unknown>((resolve, reject) => {\n const onLine = (line: string): void => {\n try {\n const response = decodeResponse(line);\n resolve(extractResult(response));\n } catch (err) {\n reject(err);\n }\n };\n\n this.reader!.once(\"line\", onLine);\n\n const ok = this.process!.stdin!.write(requestStr, (err: Error | null | undefined) => {\n if (err) {\n this.reader!.removeListener(\"line\", onLine);\n reject(err);\n }\n });\n\n if (!ok) {\n this.process!.stdin!.once(\"drain\", () => {\n // write will proceed after drain\n });\n }\n });\n }\n\n get isRunning(): boolean {\n return this.process !== null && this.process.exitCode === null;\n }\n\n /** Expose process for AttestClient reader loop access */\n get childProcess(): ChildProcess | null {\n return this.process;\n }\n\n /** Expose readline interface for AttestClient reader loop */\n get readlineInterface(): ReadlineInterface | null {\n return this.reader;\n }\n}\n","import * as https from \"node:https\";\nimport * as http from \"node:http\";\nimport * as fs from \"node:fs\";\nimport * as path from \"node:path\";\nimport * as os from \"node:os\";\nimport * as crypto from \"node:crypto\";\nimport { ENGINE_VERSION } from \"./version.js\";\n\nconst RELEASE_BASE = `https://github.com/attest-framework/attest/releases/download/v${ENGINE_VERSION}`;\nconst CHECKSUMS_URL = `${RELEASE_BASE}/checksums-sha256.txt`;\nconst MAX_REDIRECTS = 5;\n\n/**\n * Cache directory for this SDK's engine: `~/.attest/bin/<ENGINE_VERSION>/`.\n *\n * The directory name is the canonical engine version (`1.0.0-rc.N`). The legacy\n * unversioned `~/.attest/bin/attest-engine` is neither read nor modified.\n */\nexport function versionedBinDir(): string {\n return path.join(os.homedir(), \".attest\", \"bin\", ENGINE_VERSION);\n}\n\nexport function platformKey(): string {\n const platform = os.platform();\n const arch = os.arch();\n\n const mapping: Record<string, Record<string, string>> = {\n darwin: { arm64: \"darwin-arm64\", x64: \"darwin-amd64\" },\n linux: { arm64: \"linux-arm64\", x64: \"linux-amd64\" },\n win32: { x64: \"windows-amd64\", arm64: \"windows-arm64\" }\n };\n\n const osMap = mapping[platform];\n if (!osMap) {\n throw new Error(\n `Unsupported platform: ${platform}. Supported: darwin, linux, win32.`\n );\n }\n\n const key = osMap[arch];\n if (!key) {\n throw new Error(\n `Unsupported architecture '${arch}' on platform '${platform}'. ` +\n `Supported architectures: ${Object.keys(osMap).join(\", \")}.`\n );\n }\n\n return key;\n}\n\nexport function binaryFilename(): string {\n return os.platform() === \"win32\" ? \"attest-engine.exe\" : \"attest-engine\";\n}\n\n/** The engine binary in the versioned cache, or null if it is absent. */\nexport function cachedEnginePath(): string | null {\n const binPath = path.join(versionedBinDir(), binaryFilename());\n return fs.statSync(binPath, { throwIfNoEntry: false })?.isFile() ? binPath : null;\n}\n\nexport function parseChecksums(text: string): Map<string, string> {\n const checksums = new Map<string, string>();\n for (const line of text.split(\"\\n\")) {\n const trimmed = line.trim();\n if (!trimmed) continue;\n // Format: \"{hash} {filename}\" (two spaces between hash and filename)\n const match = trimmed.match(/^([a-f0-9]{64})\\s+(.+)$/);\n if (match) {\n checksums.set(match[2], match[1]);\n }\n }\n return checksums;\n}\n\nfunction httpsGet(url: string, redirectCount = 0): Promise<Buffer> {\n if (redirectCount > MAX_REDIRECTS) {\n return Promise.reject(\n new Error(`Too many redirects (>${MAX_REDIRECTS}) following ${url}`)\n );\n }\n\n return new Promise<Buffer>((resolve, reject) => {\n const client = url.startsWith(\"https:\") ? https : http;\n client.get(url, (res) => {\n const status = res.statusCode ?? 0;\n\n if (status === 301 || status === 302) {\n const location = res.headers.location;\n if (!location) {\n reject(new Error(`Redirect ${status} without Location header from ${url}`));\n return;\n }\n res.resume();\n resolve(httpsGet(location, redirectCount + 1));\n return;\n }\n\n if (status < 200 || status >= 300) {\n res.resume();\n reject(new Error(`HTTP ${status} fetching ${url}`));\n return;\n }\n\n const chunks: Buffer[] = [];\n res.on(\"data\", (chunk: Buffer) => chunks.push(chunk));\n res.on(\"end\", () => resolve(Buffer.concat(chunks)));\n res.on(\"error\", reject);\n }).on(\"error\", reject);\n });\n}\n\nexport async function downloadEngine(): Promise<string> {\n const key = platformKey();\n const filename = `attest-engine-${key}${os.platform() === \"win32\" ? \".exe\" : \"\"}`;\n const binaryUrl = `${RELEASE_BASE}/${filename}`;\n\n process.stderr.write(`[attest] Downloading engine v${ENGINE_VERSION} for ${key}...\\n`);\n\n // Fetch checksums\n process.stderr.write(`[attest] Fetching checksums...\\n`);\n const checksumsText = (await httpsGet(CHECKSUMS_URL)).toString(\"utf-8\");\n const checksums = parseChecksums(checksumsText);\n const expectedHash = checksums.get(filename);\n if (!expectedHash) {\n throw new Error(\n `No checksum found for '${filename}' in checksums-sha256.txt. ` +\n `Available files: ${[...checksums.keys()].join(\", \")}`\n );\n }\n\n // Download binary\n process.stderr.write(`[attest] Downloading ${binaryUrl}...\\n`);\n const data = await httpsGet(binaryUrl);\n\n // Verify SHA256\n const actualHash = crypto.createHash(\"sha256\").update(data).digest(\"hex\");\n if (actualHash !== expectedHash) {\n throw new Error(\n `SHA256 mismatch for ${filename}. ` +\n `Expected: ${expectedHash}, got: ${actualHash}. ` +\n `The download may be corrupted. Retry or download manually.`\n );\n }\n process.stderr.write(`[attest] Checksum verified.\\n`);\n\n // Atomic write: temp file + rename\n const dir = versionedBinDir();\n fs.mkdirSync(dir, { recursive: true });\n const destPath = path.join(dir, binaryFilename());\n const suffix = `${String(process.pid)}.${crypto.randomBytes(8).toString(\"hex\")}`;\n const tmpPath = path.join(dir, `.${binaryFilename()}.tmp.${suffix}`);\n try {\n fs.writeFileSync(tmpPath, data, { flag: \"wx\", mode: 0o755 });\n fs.chmodSync(tmpPath, 0o755);\n fs.renameSync(tmpPath, destPath);\n } catch (err) {\n // EEXIST means the temp name belongs to someone else; leave it alone.\n if ((err as NodeJS.ErrnoException).code !== \"EEXIST\") fs.rmSync(tmpPath, { force: true });\n throw err;\n }\n\n process.stderr.write(`[attest] Engine installed at ${destPath}\\n`);\n return destPath;\n}\n","/**\n * Scaffolding for `attest init`.\n *\n * `attest init` writes a sample Vitest test that runs with no model, API key or\n * network: the agent run is a trace built by hand with `TraceBuilder` and every\n * assertion is deterministic (layers 1-4). `attest init --ci github` also writes a\n * GitHub Actions workflow that gates merges on the test run, the report and the\n * policy.\n */\nimport * as fs from \"node:fs\";\nimport * as path from \"node:path\";\nimport { VERSION } from \"./version.js\";\n\n/** Marks the line a new user edits to see a failure. */\nexport const DEMO_MARKER = \"ATTEST-DEMO\";\n\nexport const CI_PROVIDERS = [\"github\"] as const;\nexport type CiProvider = (typeof CI_PROVIDERS)[number];\n\nexport const SAMPLE_TEST_TEMPLATE = `// Sample Attest test written by \\`attest init\\`. Run it with \\`npx vitest run\\`.\nimport { it } from \"vitest\";\nimport { attestExpect, TraceBuilder, type Trace } from \"@attest-ai/core\";\nimport { evaluate } from \"@attest-ai/vitest\";\n\n// The trace of one agent run, built by hand so the sample needs no model, API key\n// or network. Replace it with a trace from your agent: wrap the agent with\n// \\`new Agent(...)\\` or record it with an adapter such as OpenAIAdapter.\nfunction myAgentResult(): Trace {\n return new TraceBuilder(\"refund-agent\")\n .setInput({ user_message: \"Please refund order ORD-1001.\" })\n .addToolCall(\"lookup_order\", {\n args: { order_id: \"ORD-1001\" },\n result: { status: \"delivered\", amount: 42.0 },\n })\n .addToolCall(\"issue_refund\", {\n args: { order_id: \"ORD-1001\", amount: 42.0 },\n result: { refund_id: \"RF-1001\" },\n })\n .setOutput({\n message: \"Your refund of $42.00 has been issued.\",\n structured: { refund_id: \"RF-1001\", amount: 42.0 },\n })\n .setMetadata({ total_tokens: 180, cost_usd: 0.002, latency_ms: 900 })\n .build();\n}\n\nit(\"my agent issues the refund\", async (context) => {\n const chain = attestExpect(myAgentResult())\n .outputMatchesSchema({ type: \"object\", required: [\"refund_id\"] }) // layer 1: schema\n .costUnder(0.01) // layer 2: constraint\n .toolsCalledInOrder([\"lookup_order\", \"issue_refund\"]) // layer 3: trace\n .outputContains(\"refund\"); // ${DEMO_MARKER}: change \"refund\" to \"store credit\"\n await evaluate(chain, { context });\n});\n`;\n\n/** Vitest arguments that add AttestReporter and write the report-v2 file. */\nexport const REPORT_ARGS =\n \"--reporter=default --reporter=@attest-ai/vitest/reporter --outputFile.attest=attest-report.json\";\n\nexport const POLICY_TEMPLATE = `# Merge policy for \\`attest policy evaluate --report <report> --policy attest.policy.yaml\\`.\n# Exit 0: pass. Exit 1: a blocking rule failed. Exit 2: warnings only.\nlayers:\n 0: # layer 0 caps every layer together\n max_hard_fails: 0\n# Assertions that could not run (no judge configured, simulation) warn instead of block.\nnot_evaluated: warn\n`;\n\nexport const GITHUB_WORKFLOW_TEMPLATE = `# Written by \\`attest init --ci github\\`.\n# The test step records its outcome and continues so the report is always uploaded,\n# the policy evaluated and the pull request annotated; the last step fails the job\n# when the tests failed. A policy warning (exit 2) does not fail the job.\nname: attest\n\non:\n pull_request:\n push:\n branches: [main]\n\npermissions:\n contents: read\n pull-requests: write\n\njobs:\n attest:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v6\n with:\n persist-credentials: false\n - uses: actions/setup-node@v6\n with:\n node-version: 22\n - name: Install dependencies\n run: npm ci\n - name: Run tests\n id: tests\n continue-on-error: true\n run: npx vitest run ${REPORT_ARGS}\n - name: Upload the Attest report\n if: always()\n uses: actions/upload-artifact@v7\n with:\n name: attest-report\n path: attest-report.json\n if-no-files-found: error\n - name: Evaluate the merge policy\n if: always()\n run: |\n set +e\n npx attest policy evaluate --report attest-report.json --policy attest.policy.yaml\n code=$?\n if [ \"$code\" -eq 2 ]; then\n echo \"::warning::attest policy: warnings only (exit 2)\"\n exit 0\n fi\n exit \"$code\"\n - name: Annotate the pull request\n # Fork pull requests get a read-only token that cannot post reviews; the last step\n # still fails the job when tests failed.\n if: >-\n always() && github.event_name == 'pull_request' &&\n github.event.pull_request.head.repo.full_name == github.repository\n uses: attest-framework/attest/.github/actions/pr-annotate@v${VERSION}\n with:\n report: attest-report.json\n - name: Fail the job when tests failed\n if: always() && steps.tests.outcome == 'failure'\n run: |\n echo \"::error::vitest failed; see the Run tests step\"\n exit 1\n`;\n\nfunction write(file: string, content: string): void {\n if (fs.existsSync(file)) {\n process.stderr.write(`Skipped (exists): ${file}\\n`);\n return;\n }\n fs.mkdirSync(path.dirname(file), { recursive: true });\n fs.writeFileSync(file, content);\n process.stdout.write(`Created: ${file}\\n`);\n}\n\n/**\n * Write the sample test and the merge policy into `targetDir`; `ci: \"github\"` also\n * writes `.github/workflows/attest.yml`. Existing files are never overwritten. No\n * Vitest config is written: the workflow adds AttestReporter on the command line\n * (`REPORT_ARGS`), so an existing config keeps working unchanged.\n */\nexport function scaffoldProject(targetDir: string, options: { ci?: CiProvider } = {}): void {\n write(path.join(targetDir, \"tests\", \"my-agent.test.ts\"), SAMPLE_TEST_TEMPLATE);\n write(path.join(targetDir, \"attest.policy.yaml\"), POLICY_TEMPLATE);\n if (options.ci === \"github\") {\n write(path.join(targetDir, \".github\", \"workflows\", \"attest.yml\"), GITHUB_WORKFLOW_TEMPLATE);\n }\n process.stdout.write(\n `\\nNext: run \\`npx vitest run\\`. Then change the line marked ${DEMO_MARKER} in ` +\n \"tests/my-agent.test.ts and run it again to see a failure.\\n\",\n );\n}\n"],"mappings":";;;AAYA,SAAS,aAAAA,kBAAiB;AAC1B,YAAYC,SAAQ;AACpB,YAAYC,SAAQ;AACpB,SAAS,qBAAqB;;;ACfvB,IAAM,UAAU;AAChB,IAAM,iBAAiB;;;ACD9B,SAAS,OAAO,iBAAoC;AACpD,SAAS,uBAA4D;AACrE,SAAS,YAAY,YAAY,YAAAC,WAAU,aAAa,mBAAmB;AAC3E,SAAS,WAAW,QAAAC,aAAY;;;ACHhC,YAAY,WAAW;AACvB,YAAY,UAAU;AACtB,YAAY,QAAQ;AACpB,YAAY,UAAU;AACtB,YAAY,QAAQ;AACpB,YAAY,YAAY;AAGxB,IAAM,eAAe,iEAAiE,cAAc;AACpG,IAAM,gBAAgB,GAAG,YAAY;AACrC,IAAM,gBAAgB;AAQf,SAAS,kBAA0B;AACxC,SAAY,UAAQ,WAAQ,GAAG,WAAW,OAAO,cAAc;AACjE;AAEO,SAAS,cAAsB;AACpC,QAAMC,YAAc,YAAS;AAC7B,QAAMC,QAAU,QAAK;AAErB,QAAM,UAAkD;AAAA,IACtD,QAAQ,EAAE,OAAO,gBAAgB,KAAK,eAAe;AAAA,IACrD,OAAO,EAAE,OAAO,eAAe,KAAK,cAAc;AAAA,IAClD,OAAO,EAAE,KAAK,iBAAiB,OAAO,gBAAgB;AAAA,EACxD;AAEA,QAAM,QAAQ,QAAQD,SAAQ;AAC9B,MAAI,CAAC,OAAO;AACV,UAAM,IAAI;AAAA,MACR,yBAAyBA,SAAQ;AAAA,IACnC;AAAA,EACF;AAEA,QAAM,MAAM,MAAMC,KAAI;AACtB,MAAI,CAAC,KAAK;AACR,UAAM,IAAI;AAAA,MACR,6BAA6BA,KAAI,kBAAkBD,SAAQ,+BAC/B,OAAO,KAAK,KAAK,EAAE,KAAK,IAAI,CAAC;AAAA,IAC3D;AAAA,EACF;AAEA,SAAO;AACT;AAEO,SAAS,iBAAyB;AACvC,SAAU,YAAS,MAAM,UAAU,sBAAsB;AAC3D;AAGO,SAAS,mBAAkC;AAChD,QAAM,UAAe,UAAK,gBAAgB,GAAG,eAAe,CAAC;AAC7D,SAAU,YAAS,SAAS,EAAE,gBAAgB,MAAM,CAAC,GAAG,OAAO,IAAI,UAAU;AAC/E;AAEO,SAAS,eAAe,MAAmC;AAChE,QAAM,YAAY,oBAAI,IAAoB;AAC1C,aAAW,QAAQ,KAAK,MAAM,IAAI,GAAG;AACnC,UAAM,UAAU,KAAK,KAAK;AAC1B,QAAI,CAAC,QAAS;AAEd,UAAM,QAAQ,QAAQ,MAAM,yBAAyB;AACrD,QAAI,OAAO;AACT,gBAAU,IAAI,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC;AAAA,IAClC;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,SAAS,KAAa,gBAAgB,GAAoB;AACjE,MAAI,gBAAgB,eAAe;AACjC,WAAO,QAAQ;AAAA,MACb,IAAI,MAAM,wBAAwB,aAAa,eAAe,GAAG,EAAE;AAAA,IACrE;AAAA,EACF;AAEA,SAAO,IAAI,QAAgB,CAAC,SAAS,WAAW;AAC9C,UAAM,SAAS,IAAI,WAAW,QAAQ,IAAI,QAAQ;AAClD,WAAO,IAAI,KAAK,CAAC,QAAQ;AACvB,YAAM,SAAS,IAAI,cAAc;AAEjC,UAAI,WAAW,OAAO,WAAW,KAAK;AACpC,cAAM,WAAW,IAAI,QAAQ;AAC7B,YAAI,CAAC,UAAU;AACb,iBAAO,IAAI,MAAM,YAAY,MAAM,iCAAiC,GAAG,EAAE,CAAC;AAC1E;AAAA,QACF;AACA,YAAI,OAAO;AACX,gBAAQ,SAAS,UAAU,gBAAgB,CAAC,CAAC;AAC7C;AAAA,MACF;AAEA,UAAI,SAAS,OAAO,UAAU,KAAK;AACjC,YAAI,OAAO;AACX,eAAO,IAAI,MAAM,QAAQ,MAAM,aAAa,GAAG,EAAE,CAAC;AAClD;AAAA,MACF;AAEA,YAAM,SAAmB,CAAC;AAC1B,UAAI,GAAG,QAAQ,CAAC,UAAkB,OAAO,KAAK,KAAK,CAAC;AACpD,UAAI,GAAG,OAAO,MAAM,QAAQ,OAAO,OAAO,MAAM,CAAC,CAAC;AAClD,UAAI,GAAG,SAAS,MAAM;AAAA,IACxB,CAAC,EAAE,GAAG,SAAS,MAAM;AAAA,EACvB,CAAC;AACH;AAEA,eAAsB,iBAAkC;AACtD,QAAM,MAAM,YAAY;AACxB,QAAM,WAAW,iBAAiB,GAAG,GAAM,YAAS,MAAM,UAAU,SAAS,EAAE;AAC/E,QAAM,YAAY,GAAG,YAAY,IAAI,QAAQ;AAE7C,UAAQ,OAAO,MAAM,gCAAgC,cAAc,QAAQ,GAAG;AAAA,CAAO;AAGrF,UAAQ,OAAO,MAAM;AAAA,CAAkC;AACvD,QAAM,iBAAiB,MAAM,SAAS,aAAa,GAAG,SAAS,OAAO;AACtE,QAAM,YAAY,eAAe,aAAa;AAC9C,QAAM,eAAe,UAAU,IAAI,QAAQ;AAC3C,MAAI,CAAC,cAAc;AACjB,UAAM,IAAI;AAAA,MACR,0BAA0B,QAAQ,+CACd,CAAC,GAAG,UAAU,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC;AAAA,IACtD;AAAA,EACF;AAGA,UAAQ,OAAO,MAAM,wBAAwB,SAAS;AAAA,CAAO;AAC7D,QAAM,OAAO,MAAM,SAAS,SAAS;AAGrC,QAAM,aAAoB,kBAAW,QAAQ,EAAE,OAAO,IAAI,EAAE,OAAO,KAAK;AACxE,MAAI,eAAe,cAAc;AAC/B,UAAM,IAAI;AAAA,MACR,uBAAuB,QAAQ,eAClB,YAAY,UAAU,UAAU;AAAA,IAE/C;AAAA,EACF;AACA,UAAQ,OAAO,MAAM;AAAA,CAA+B;AAGpD,QAAM,MAAM,gBAAgB;AAC5B,EAAG,aAAU,KAAK,EAAE,WAAW,KAAK,CAAC;AACrC,QAAM,WAAgB,UAAK,KAAK,eAAe,CAAC;AAChD,QAAM,SAAS,GAAG,OAAO,QAAQ,GAAG,CAAC,IAAW,mBAAY,CAAC,EAAE,SAAS,KAAK,CAAC;AAC9E,QAAM,UAAe,UAAK,KAAK,IAAI,eAAe,CAAC,QAAQ,MAAM,EAAE;AACnE,MAAI;AACF,IAAG,iBAAc,SAAS,MAAM,EAAE,MAAM,MAAM,MAAM,IAAM,CAAC;AAC3D,IAAG,aAAU,SAAS,GAAK;AAC3B,IAAG,cAAW,SAAS,QAAQ;AAAA,EACjC,SAAS,KAAK;AAEZ,QAAK,IAA8B,SAAS,SAAU,CAAG,UAAO,SAAS,EAAE,OAAO,KAAK,CAAC;AACxF,UAAM;AAAA,EACR;AAEA,UAAQ,OAAO,MAAM,gCAAgC,QAAQ;AAAA,CAAI;AACjE,SAAO;AACT;;;AD1JA,IAAM,qBAAqB;AAC3B,IAAM,2BAA2B;AAG1B,IAAM,6BAAN,cAAyC,MAAM;AAAA,EAC3C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAET,YAAY,YAAoB,QAAgB,OAAsB,UAAkB;AACtF,UAAM,WAAW,UAAU,OAAO,uBAAuB,mBAAmB,KAAK;AACjF;AAAA,MACE,oBAAoB,UAAU,UAAU,MAAM,KAAK,QAAQ,qCACtB,QAAQ,mHAEN,QAAQ,uBAAuB,QAAQ;AAAA,IAChF;AACA,SAAK,OAAO;AACZ,SAAK,aAAa;AAClB,SAAK,SAAS;AACd,SAAK,QAAQ;AACb,SAAK,WAAW;AAAA,EAClB;AACF;AAQO,SAAS,cAAc,YAAmC;AAC/D,QAAM,OAAO,UAAU,YAAY,CAAC,SAAS,GAAG;AAAA,IAC9C,OAAO,CAAC,UAAU,QAAQ,MAAM;AAAA,IAChC,UAAU;AAAA,IACV,SAAS;AAAA,EACX,CAAC;AACD,MAAI,KAAK,MAAO,OAAM,KAAK;AAC3B,QAAM,MAAM,KAAK,OAAO,KAAK;AAC7B,QAAM,SAAS,GAAG,kBAAkB;AACpC,MAAI,KAAK,WAAW,KAAK,CAAC,IAAI,WAAW,MAAM,EAAG,QAAO;AACzD,SAAO,IAAI,MAAM,OAAO,MAAM,EAAE,KAAK;AACvC;AAEA,SAAS,SAAwB;AAC/B,aAAW,QAAQ,QAAQ,IAAI,MAAM,KAAK,IAAI,MAAM,SAAS,GAAG;AAC9D,QAAI,CAAC,IAAK;AACV,UAAM,YAAYE,MAAK,KAAK,eAAe,CAAC;AAC5C,QAAI;AACF,iBAAW,WAAW,YAAY,IAAI;AAAA,IACxC,QAAQ;AACN;AAAA,IACF;AACA,QAAIC,UAAS,SAAS,EAAE,OAAO,EAAG,QAAO;AAAA,EAC3C;AACA,SAAO;AACT;AAEA,SAAS,YAAY,YAAoB,OAAqB;AAC5D,MAAI,CAAC,WAAW,UAAU,KAAK,CAACA,UAAS,UAAU,EAAE,OAAO,GAAG;AAC7D,UAAM,IAAI,MAAM,GAAG,KAAK,IAAI,UAAU,mCAAmC;AAAA,EAC3E;AACF;AAEA,eAAe,aACb,cACA,eAC2B;AAC3B,MAAI,cAAc;AAChB,gBAAY,cAAc,aAAa;AACvC,WAAO,CAAC,cAAc,sBAAsB;AAAA,EAC9C;AACA,QAAM,UAAU,QAAQ,IAAI,oBAAoB;AAChD,MAAI,SAAS;AACX,gBAAY,SAAS,oBAAoB;AACzC,WAAO,CAAC,SAAS,oBAAoB;AAAA,EACvC;AACA,QAAM,QAAQ,OAAO;AACrB,MAAI,MAAO,QAAO,CAAC,OAAO,MAAM;AAChC,QAAM,SAAS,iBAAiB;AAChC,MAAI,OAAQ,QAAO,CAAC,QAAQ,cAAc;AAE1C,MAAI,CAAC,eAAe;AAClB,UAAM,IAAI;AAAA,MACR,OAAO,kBAAkB,KAAK,cAAc,iDACzB,cAAc;AAAA,IACnC;AAAA,EACF;AAEA,QAAM,aAAa,QAAQ,IAAI,2BAA2B,GAAG,YAAY;AACzE,MAAI,eAAe,OAAO,eAAe,UAAU,eAAe,OAAO;AACvE,UAAM,IAAI;AAAA,MACR,gBAAgB,kBAAkB,KAAK,cAAc;AAAA;AAAA;AAAA,yBAGzB,cAAc;AAAA;AAAA,IAE5C;AAAA,EACF;AACA,SAAO,CAAC,MAAM,eAAe,GAAG,UAAU;AAC5C;AAYA,eAAsB,cACpB,cACA,UAAuC,CAAC,GACvB;AACjB,QAAM,CAAC,YAAY,MAAM,IAAI,MAAM,aAAa,cAAc,QAAQ,iBAAiB,IAAI;AAC3F,QAAM,QAAQ,cAAc,UAAU;AACtC,MAAI,UAAU,gBAAgB;AAC5B,UAAM,IAAI,2BAA2B,YAAY,QAAQ,OAAO,cAAc;AAAA,EAChF;AACA,SAAO;AACT;;;AE3HA,YAAYC,SAAQ;AACpB,YAAYC,WAAU;AAIf,IAAM,cAAc;AAKpB,IAAM,uBAAuB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,oCAgCA,WAAW;AAAA;AAAA;AAAA;AAMxC,IAAM,cACX;AAEK,IAAM,kBAAkB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AASxB,IAAM,2BAA2B;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,8BA8BV,WAAW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,qEAyB4B,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAU5E,SAAS,MAAM,MAAc,SAAuB;AAClD,MAAO,eAAW,IAAI,GAAG;AACvB,YAAQ,OAAO,MAAM,qBAAqB,IAAI;AAAA,CAAI;AAClD;AAAA,EACF;AACA,EAAG,cAAe,cAAQ,IAAI,GAAG,EAAE,WAAW,KAAK,CAAC;AACpD,EAAG,kBAAc,MAAM,OAAO;AAC9B,UAAQ,OAAO,MAAM,YAAY,IAAI;AAAA,CAAI;AAC3C;AAQO,SAAS,gBAAgB,WAAmB,UAA+B,CAAC,GAAS;AAC1F,QAAW,WAAK,WAAW,SAAS,kBAAkB,GAAG,oBAAoB;AAC7E,QAAW,WAAK,WAAW,oBAAoB,GAAG,eAAe;AACjE,MAAI,QAAQ,OAAO,UAAU;AAC3B,UAAW,WAAK,WAAW,WAAW,aAAa,YAAY,GAAG,wBAAwB;AAAA,EAC5F;AACA,UAAQ,OAAO;AAAA,IACb;AAAA,4DAA+D,WAAW;AAAA;AAAA,EAE5E;AACF;;;AJ5IA,IAAM,aAAa,oBAAI,IAAI,CAAC,MAAM,QAAQ,CAAC;AAE3C,IAAM,YACJ;AAUF,IAAM,aAAa;AAGnB,SAAS,QAAQ,MAAwB;AACvC,MAAI,KAAK,KAAK,CAAC,MAAM,WAAW,IAAI,CAAC,CAAC,GAAG;AACvC,YAAQ,OAAO,MAAM,GAAG,UAAU;AAAA,CAAI;AACtC,WAAO;AAAA,EACT;AACA,QAAM,SAAS,KAAK,KAAK,GAAG;AAC5B,MAAI,KAAK,SAAS,KAAK,WAAW,iBAAiB,WAAW,eAAe;AAC3E,YAAQ,OAAO,MAAM,GAAG,UAAU;AAAA,CAAI;AACtC,WAAO;AAAA,EACT;AACA,kBAAgB,QAAQ,IAAI,GAAG,KAAK,SAAS,IAAI,EAAE,IAAI,SAAS,IAAI,CAAC,CAAC;AACtE,SAAO;AACT;AAEA,IAAM,wBAAwB;AAE9B,SAAS,aAAa,KAAsB;AAC1C,SAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AACxD;AAQO,SAAS,eACd,QACA,YAAoB,uBACQ;AAC5B,QAAM,OAAOC,WAAU,QAAQ,CAAC,MAAM,GAAG;AAAA,IACvC,OAAO,CAAC,UAAU,QAAQ,MAAM;AAAA,IAChC,UAAU;AAAA,IACV,SAAS;AAAA,EACX,CAAC;AACD,MAAI,KAAK,SAAS,KAAK,WAAW,EAAG,QAAO;AAC5C,QAAM,QAAQ,KAAK,OAAO,MAAM,IAAI;AACpC,QAAM,QAAQ,MAAM,QAAQ,WAAW;AACvC,MAAI,UAAU,GAAI,QAAO;AACzB,QAAM,WAAW,oBAAI,IAAoB;AACzC,aAAW,QAAQ,MAAM,MAAM,QAAQ,CAAC,GAAG;AACzC,UAAM,QAAQ,oBAAoB,KAAK,IAAI;AAC3C,QAAI,QAAQ,CAAC,EAAG,UAAS,IAAI,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,IAAI,KAAK,CAAC;AAAA,EAChE;AACA,SAAO;AACT;AAGA,eAAe,YAA6B;AAC1C,UAAQ,OAAO,MAAM,GAAG,SAAS;AAAA,CAAI;AACrC,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,cAAc,QAAW,EAAE,eAAe,MAAM,CAAC;AAAA,EAClE,SAAS,KAAK;AACZ,UAAM,aAAa,aAAa,GAAG,EAAE,MAAM,IAAI,EAAE,CAAC,KAAK,IAAI,QAAQ,OAAO,EAAE;AAC5E,YAAQ,OAAO;AAAA,MACb,mCAAmC,SAAS,sDACzB,cAAc;AAAA;AAAA,IACnC;AACA,WAAO;AAAA,EACT;AACA,QAAM,WAAW,eAAe,MAAM;AACtC,MAAI,aAAa,MAAM;AACrB,YAAQ,OAAO;AAAA,MACb,oDAAoD,MAAM;AAAA;AAAA,IAE5D;AACA,WAAO;AAAA,EACT;AACA,QAAM,QAAQ,KAAK,IAAI,GAAG,GAAG,CAAC,GAAG,SAAS,KAAK,CAAC,EAAE,IAAI,CAAC,SAAS,KAAK,MAAM,CAAC;AAC5E,UAAQ,OAAO,MAAM,kCAAkC,cAAc,KAAK,MAAM;AAAA,CAAM;AACtF,aAAW,CAAC,MAAM,OAAO,KAAK,UAAU;AACtC,YAAQ,OAAO,MAAM,KAAK,KAAK,OAAO,KAAK,CAAC,KAAK,OAAO;AAAA,CAAI;AAAA,EAC9D;AACA,SAAO;AACT;AAGO,SAAS,WAAW,QAAuB,QAAuC;AACvF,MAAI,WAAW,KAAM,QAAO;AAC5B,MAAI,WAAW,KAAM,QAAO,OAAU,cAAU,QAAQ,MAAM,KAAK;AACnE,SAAO;AACT;AAEA,eAAe,QAAQ,MAAiC;AACtD,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,cAAc;AAAA,EAC/B,SAAS,KAAK;AACZ,YAAQ,OAAO,MAAM,WAAW,aAAa,GAAG,CAAC;AAAA,CAAI;AACrD,WAAO;AAAA,EACT;AACA,QAAM,UAAU,KAAK,CAAC,KAAK;AAC3B,QAAM,WAAW,eAAe,MAAM;AACtC,MAAI,aAAa,QAAS,YAAY,UAAU,CAAC,SAAS,IAAI,OAAO,GAAI;AACvE,YAAQ,OAAO;AAAA,MACb,4BAA4B,MAAM,aAAa,cAAc,uBAC/C,OAAO;AAAA;AAAA,IAGvB;AACA,WAAO;AAAA,EACT;AACA,QAAM,OAAOA,WAAU,QAAQ,MAAM,EAAE,OAAO,UAAU,CAAC;AACzD,MAAI,KAAK,MAAO,OAAM,KAAK;AAC3B,SAAO,WAAW,KAAK,QAAQ,KAAK,MAAM;AAC5C;AAGA,eAAsB,KAAK,OAAiB,QAAQ,KAAK,MAAM,CAAC,GAAoB;AAClF,QAAM,CAAC,OAAO,IAAI;AAClB,MAAI,YAAY,UAAa,WAAW,IAAI,OAAO,KAAM,YAAY,UAAU,KAAK,WAAW,GAAI;AACjG,WAAO,UAAU;AAAA,EACnB;AACA,MAAI,YAAY,aAAa;AAC3B,YAAQ,OAAO,MAAM,UAAU,OAAO,4BAA4B,cAAc;AAAA,CAAK;AACrF,WAAO;AAAA,EACT;AACA,MAAI,YAAY,QAAQ;AACtB,WAAO,QAAQ,KAAK,MAAM,CAAC,CAAC;AAAA,EAC9B;AACA,MAAI,QAAQ,WAAW,GAAG,GAAG;AAC3B,YAAQ,OAAO,MAAM,0BAA0B,OAAO;AAAA;AAAA,EAAO,SAAS,EAAE;AACxE,WAAO;AAAA,EACT;AACA,SAAO,QAAQ,IAAI;AACrB;AAGA,SAAS,oBAA6B;AACpC,QAAM,SAAS,QAAQ,KAAK,CAAC;AAC7B,MAAI,WAAW,UAAa,CAAI,eAAW,MAAM,EAAG,QAAO;AAC3D,SAAU,iBAAa,MAAM,MAAS,iBAAa,cAAc,YAAY,GAAG,CAAC;AACnF;AAEA,IAAI,kBAAkB,GAAG;AACvB,OAAK,EAAE;AAAA,IACL,CAAC,SAAS,QAAQ,KAAK,IAAI;AAAA,IAC3B,CAAC,QAAiB;AAChB,cAAQ,OAAO,MAAM,WAAW,aAAa,GAAG,CAAC;AAAA,CAAI;AACrD,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF;AACF;","names":["spawnSync","fs","os","statSync","join","platform","arch","join","statSync","fs","path","spawnSync"]}
|
package/dist/esm/index.d.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { InitializeResult, Trace, Assertion, EvaluateBatchResult, DriftReport, SimulatePersona, ConversationMessage, SimulateFaultConfig, Step, AssertionResult } from './proto/index.js';
|
|
2
|
-
export { AgentId, AssertionId, AssertionSpec, ConstraintSpec, ContentSpec, ERR_ASSERTION_ERROR, ERR_ENGINE_ERROR, ERR_INVALID_TRACE, ERR_PROVIDER_ERROR, ERR_SESSION_ERROR, ERR_TIMEOUT, EmbeddingSpec, EngineTimeoutError, ErrorData, EvaluateBatchParams, GenerateUserMessageParams, GenerateUserMessageResult, InitializeParams, LlmJudgeSpec, PluginSpec, ProtocolError, QueryDriftParams, QueryDriftResult, RPCError, RPCRequest, RPCResponse, STATUS_HARD_FAIL, STATUS_PASS, STATUS_SOFT_FAIL, STEP_AGENT_CALL, STEP_LLM_CALL, STEP_RETRIEVAL, STEP_TOOL_CALL, SchemaSpec, ShutdownResult, SubmitPluginResultParams, TYPE_CONSTRAINT, TYPE_CONTENT, TYPE_EMBEDDING, TYPE_LLM_JUDGE, TYPE_PLUGIN, TYPE_SCHEMA, TYPE_TRACE, TYPE_TRACE_TREE, TraceId, TraceMetadata, TraceSpec, TraceTreeSpec, agentId, assertionId, decodeResponse, encodeRequest, extractId, extractResult, traceId } from './proto/index.js';
|
|
1
|
+
import { InitializeResult, ProtocolLogger, ProtocolDiagnostic, Trace, Assertion, EvaluateBatchResult, DriftReport, SimulatePersona, ConversationMessage, SimulateFaultConfig, Step, AssertionResult, Outcome } from './proto/index.js';
|
|
2
|
+
export { AgentId, AssertionId, AssertionSpec, BiasProbe, ConstraintSpec, ContentSpec, ERR_ASSERTION_ERROR, ERR_ENGINE_ERROR, ERR_INVALID_TRACE, ERR_PROVIDER_ERROR, ERR_SESSION_ERROR, ERR_TIMEOUT, EmbeddingSpec, EngineTimeoutError, ErrorData, EvaluateBatchParams, FailureClass, GenerateUserMessageParams, GenerateUserMessageResult, InitializeParams, JudgeAgreement, JudgeMetadata, LlmJudgeSpec, OUTCOME_FAIL, OUTCOME_PASS, OUTCOME_SKIP, PluginSpec, ProtocolDesyncError, ProtocolDiagnosticBuffer, ProtocolDiagnosticKind, ProtocolError, QueryDriftParams, QueryDriftResult, REASON_MISSING_EVIDENCE, REASON_PROVIDER_NOT_CONFIGURED, REASON_PROVIDER_UNAVAILABLE, REASON_SHORT_CIRCUITED, REASON_SIMULATION, RPCError, RPCRequest, RPCResponse, STATUS_HARD_FAIL, STATUS_NOT_EVALUATED, STATUS_PASS, STATUS_SOFT_FAIL, STEP_AGENT_CALL, STEP_LLM_CALL, STEP_RETRIEVAL, STEP_TOOL_CALL, SchemaSpec, ShutdownResult, SubmitPluginResultParams, TYPE_CONSTRAINT, TYPE_CONTENT, TYPE_EMBEDDING, TYPE_LLM_JUDGE, TYPE_PLUGIN, TYPE_SCHEMA, TYPE_TRACE, TYPE_TRACE_TREE, TraceId, TraceMetadata, TraceSpec, TraceTreeSpec, agentId, assertionId, decodeResponse, encodeRequest, extractId, extractResult, layerForType, previewLine, traceId } from './proto/index.js';
|
|
3
3
|
import { ChildProcess } from 'node:child_process';
|
|
4
4
|
import { Interface } from 'node:readline';
|
|
5
5
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
6
6
|
import { EventEmitter } from 'node:events';
|
|
7
7
|
|
|
8
|
-
declare const VERSION = "0.
|
|
9
|
-
declare const ENGINE_VERSION = "0.
|
|
8
|
+
declare const VERSION = "1.0.0-rc.1";
|
|
9
|
+
declare const ENGINE_VERSION = "1.0.0-rc.1";
|
|
10
10
|
|
|
11
11
|
declare class EngineManager {
|
|
12
12
|
private enginePath;
|
|
@@ -21,6 +21,8 @@ declare class EngineManager {
|
|
|
21
21
|
logLevel?: string;
|
|
22
22
|
});
|
|
23
23
|
start(): Promise<InitializeResult>;
|
|
24
|
+
/** Send initialize and check the engine's version and capabilities. */
|
|
25
|
+
private initialize;
|
|
24
26
|
stop(): Promise<void>;
|
|
25
27
|
sendRequest(method: string, params: Record<string, unknown>): Promise<unknown>;
|
|
26
28
|
private sendRequestInternal;
|
|
@@ -31,6 +33,28 @@ declare class EngineManager {
|
|
|
31
33
|
get readlineInterface(): Interface | null;
|
|
32
34
|
}
|
|
33
35
|
|
|
36
|
+
interface AttestClientOptions {
|
|
37
|
+
/** Logger override. When omitted, warnings only emit if ATTEST_DEBUG_PROTOCOL=1. */
|
|
38
|
+
logger?: ProtocolLogger;
|
|
39
|
+
/** Listener invoked for every protocol diagnostic. */
|
|
40
|
+
onDiagnostic?: (diagnostic: ProtocolDiagnostic) => void;
|
|
41
|
+
/** Diagnostic ring-buffer capacity. Default 32. */
|
|
42
|
+
diagnosticBufferSize?: number;
|
|
43
|
+
/** Number of diagnostics in `desyncWindowMs` that triggers desync rejection. Default 3. */
|
|
44
|
+
desyncThreshold?: number;
|
|
45
|
+
/** Window length for desync detection in milliseconds. Default 1000. */
|
|
46
|
+
desyncWindowMs?: number;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* AttestClient owns request/response correlation over the engine's
|
|
50
|
+
* NDJSON protocol.
|
|
51
|
+
*
|
|
52
|
+
* Desync detection is a one-way latch: once the diagnostic rate breaches
|
|
53
|
+
* `desyncThreshold` within `desyncWindowMs` the client refuses further
|
|
54
|
+
* sends for the lifetime of the instance. There is no reset hook by
|
|
55
|
+
* design — recovery requires constructing a new client over a fresh
|
|
56
|
+
* engine subprocess.
|
|
57
|
+
*/
|
|
34
58
|
declare class AttestClient {
|
|
35
59
|
private readonly engine;
|
|
36
60
|
private requestId;
|
|
@@ -38,10 +62,23 @@ declare class AttestClient {
|
|
|
38
62
|
private writeChain;
|
|
39
63
|
private readerActive;
|
|
40
64
|
private lineHandler;
|
|
41
|
-
|
|
65
|
+
private readonly logger;
|
|
66
|
+
private readonly diagnosticBuffer;
|
|
67
|
+
private readonly diagnosticListeners;
|
|
68
|
+
private readonly desyncThreshold;
|
|
69
|
+
private readonly desyncWindowMs;
|
|
70
|
+
private desynced;
|
|
71
|
+
constructor(engine: EngineManager, options?: AttestClientOptions);
|
|
72
|
+
/** Subscribe to protocol diagnostics. Returns an unsubscribe function. */
|
|
73
|
+
onProtocolDiagnostic(listener: (diagnostic: ProtocolDiagnostic) => void): () => void;
|
|
74
|
+
/** Snapshot of the diagnostic ring buffer (most recent last). */
|
|
75
|
+
protocolDiagnostics(): readonly ProtocolDiagnostic[];
|
|
42
76
|
startReader(): void;
|
|
43
77
|
stopReader(): void;
|
|
78
|
+
private recordDiagnostic;
|
|
79
|
+
private maybeTriggerDesync;
|
|
44
80
|
private handleLine;
|
|
81
|
+
private handleDecodeFailure;
|
|
45
82
|
private failAll;
|
|
46
83
|
sendRequest(method: string, params: Record<string, unknown>, timeoutMs?: number): Promise<unknown>;
|
|
47
84
|
evaluateBatch(trace: Trace, assertions: readonly Assertion[], options?: {
|
|
@@ -57,6 +94,12 @@ declare class AttestClient {
|
|
|
57
94
|
timeout?: number;
|
|
58
95
|
}): Promise<string>;
|
|
59
96
|
}
|
|
97
|
+
/**
|
|
98
|
+
* Throw unless the engine advertised `simulation_mode` at initialize. An
|
|
99
|
+
* engine started without ATTEST_SIMULATION (or older than 1.0) would call
|
|
100
|
+
* real embedding and judge providers in simulation mode.
|
|
101
|
+
*/
|
|
102
|
+
declare function requireSimulationCapability(init: InitializeResult | null | undefined): void;
|
|
60
103
|
|
|
61
104
|
declare class TraceBuilder {
|
|
62
105
|
private traceId;
|
|
@@ -104,14 +147,60 @@ declare class AgentResult {
|
|
|
104
147
|
readonly totalCost: number;
|
|
105
148
|
readonly totalDurationMs: number;
|
|
106
149
|
constructor(trace: Trace, assertionResults?: readonly AssertionResult[], totalCost?: number, totalDurationMs?: number);
|
|
150
|
+
/** True if every assertion has status "pass". */
|
|
107
151
|
get passed(): boolean;
|
|
152
|
+
/**
|
|
153
|
+
* Test outcome after the whole batch. Any hard_fail fails; otherwise any
|
|
154
|
+
* not_evaluated skips (fails when `strict`); otherwise pass, including
|
|
155
|
+
* when only soft_fail rows are present (see `softFailures`).
|
|
156
|
+
*/
|
|
157
|
+
outcome(options?: {
|
|
158
|
+
strict?: boolean;
|
|
159
|
+
}): Outcome;
|
|
160
|
+
/** Failed assertions: hard_fail and soft_fail only. */
|
|
108
161
|
get failedAssertions(): readonly AssertionResult[];
|
|
109
162
|
get hardFailures(): readonly AssertionResult[];
|
|
110
163
|
get softFailures(): readonly AssertionResult[];
|
|
164
|
+
/** Assertions the engine did not evaluate; each carries a `reason`. */
|
|
165
|
+
get notEvaluated(): readonly AssertionResult[];
|
|
166
|
+
/** Count of not_evaluated assertions per reason, in first-seen order. */
|
|
167
|
+
get notEvaluatedReasons(): Readonly<Record<string, number>>;
|
|
168
|
+
/**
|
|
169
|
+
* Passed / evaluated assertions; null when nothing was evaluated.
|
|
170
|
+
* not_evaluated rows are excluded from the denominator.
|
|
171
|
+
*/
|
|
172
|
+
get passRate(): number | null;
|
|
111
173
|
get passCount(): number;
|
|
174
|
+
/** Number of failed assertions (hard_fail + soft_fail). */
|
|
112
175
|
get failCount(): number;
|
|
176
|
+
/** "not evaluated: <reason>[, <reason> ×n ...]" for skipped tests. */
|
|
177
|
+
skipReason(): string;
|
|
113
178
|
}
|
|
114
179
|
|
|
180
|
+
/**
|
|
181
|
+
* Terminal-grade rendering of Attest assertion failures.
|
|
182
|
+
*
|
|
183
|
+
* Mirrors the Python attest.cli.diagnostics module so PR comments,
|
|
184
|
+
* pytest output, and vitest output show the same fields. Use as the
|
|
185
|
+
* message argument when asserting the verdict, e.g.
|
|
186
|
+
* `expect(result.outcome(), renderDiagnostics(result)).not.toBe("fail")`, so
|
|
187
|
+
* reviewers see trace path, expected vs actual, and judge metadata.
|
|
188
|
+
*/
|
|
189
|
+
|
|
190
|
+
interface RenderOptions {
|
|
191
|
+
/** Emit ANSI colour escapes. Default false (plain text). */
|
|
192
|
+
color?: boolean;
|
|
193
|
+
/** Source location annotation appended after each block. */
|
|
194
|
+
testFile?: string;
|
|
195
|
+
/** Render passing assertions in addition to failures. Default false. */
|
|
196
|
+
includePassing?: boolean;
|
|
197
|
+
}
|
|
198
|
+
declare function renderAssertionFailure(result: AssertionResult, options?: RenderOptions): string;
|
|
199
|
+
declare function renderSummary(result: AgentResult, options?: {
|
|
200
|
+
color?: boolean;
|
|
201
|
+
}): string;
|
|
202
|
+
declare function renderDiagnostics(result: AgentResult, options?: RenderOptions): string;
|
|
203
|
+
|
|
115
204
|
declare class TraceTree {
|
|
116
205
|
readonly root: Trace;
|
|
117
206
|
constructor(root: Trace);
|
|
@@ -808,9 +897,11 @@ declare function tier(level: number): <T extends (...args: never[]) => unknown>(
|
|
|
808
897
|
/**
|
|
809
898
|
* Configure global SDK options.
|
|
810
899
|
*
|
|
811
|
-
* @param options.simulation - Enable simulation mode
|
|
812
|
-
*
|
|
813
|
-
*
|
|
900
|
+
* @param options.simulation - Enable simulation mode: the engine evaluates
|
|
901
|
+
* layers 1-4 and reports layers 5-6 as not evaluated (reason "simulation")
|
|
902
|
+
* without calling any provider. The engine reads `ATTEST_SIMULATION` when it
|
|
903
|
+
* starts; the Vitest fixtures set it from this flag. Equivalent to setting
|
|
904
|
+
* `ATTEST_SIMULATION=1` in the environment.
|
|
814
905
|
* @param options.sampleRate - Sampling rate for continuous evaluation (0.0–1.0).
|
|
815
906
|
* Equivalent to setting `ATTEST_SAMPLE_RATE` in the environment.
|
|
816
907
|
*/
|
|
@@ -823,6 +914,11 @@ declare function config(options: {
|
|
|
823
914
|
* or the `ATTEST_SIMULATION` environment variable.
|
|
824
915
|
*/
|
|
825
916
|
declare function isSimulationMode(): boolean;
|
|
917
|
+
/**
|
|
918
|
+
* True when `ATTEST_SIMULATION` is set for this process (and so for an
|
|
919
|
+
* engine it spawns): "1", "true" or "yes", trimmed and case-insensitive.
|
|
920
|
+
*/
|
|
921
|
+
declare function simulationEnvEnabled(): boolean;
|
|
826
922
|
/**
|
|
827
923
|
* Returns the sampling rate for continuous evaluation. Priority:
|
|
828
924
|
* programmatic `config({ sampleRate })` > `ATTEST_SAMPLE_RATE` env var > 0.
|
|
@@ -834,13 +930,6 @@ declare function getSampleRate(): number;
|
|
|
834
930
|
*/
|
|
835
931
|
declare function resetConfig(): void;
|
|
836
932
|
|
|
837
|
-
/**
|
|
838
|
-
* Return a deterministic all-pass result for every assertion without
|
|
839
|
-
* contacting the engine or any LLM provider. Used when simulation mode
|
|
840
|
-
* is active (`ATTEST_SIMULATION=1` or `config({ simulation: true })`).
|
|
841
|
-
*/
|
|
842
|
-
declare function simulationEvaluateBatch(assertions: readonly Assertion[]): EvaluateBatchResult;
|
|
843
|
-
|
|
844
933
|
interface PluginResult {
|
|
845
934
|
readonly status: string;
|
|
846
935
|
readonly score: number;
|
|
@@ -903,6 +992,59 @@ declare class ContinuousEvalRunner {
|
|
|
903
992
|
private scheduleNext;
|
|
904
993
|
}
|
|
905
994
|
|
|
995
|
+
/**
|
|
996
|
+
* Calibration helpers — TS parity with engine/internal/assertion/judge/calibration.go
|
|
997
|
+
* and sdks/python/src/attest/calibration.py.
|
|
998
|
+
*
|
|
999
|
+
* Computes Cohen's κ, agreement %, and ROC-AUC over (human, judge) score
|
|
1000
|
+
* pairs and parses CSV/JSONL labels files. Metrics match the engine
|
|
1001
|
+
* byte-for-byte on the same input.
|
|
1002
|
+
*/
|
|
1003
|
+
interface LabelPair {
|
|
1004
|
+
readonly human: number;
|
|
1005
|
+
readonly judge: number;
|
|
1006
|
+
}
|
|
1007
|
+
interface AgreementResult {
|
|
1008
|
+
readonly threshold: number;
|
|
1009
|
+
readonly n: number;
|
|
1010
|
+
readonly agreement: number;
|
|
1011
|
+
readonly cohenKappa: number;
|
|
1012
|
+
readonly rocAuc: number;
|
|
1013
|
+
}
|
|
1014
|
+
interface LabeledRecord {
|
|
1015
|
+
input: string;
|
|
1016
|
+
humanLabel: number;
|
|
1017
|
+
judgeScore: number;
|
|
1018
|
+
judgeKnown: boolean;
|
|
1019
|
+
}
|
|
1020
|
+
/**
|
|
1021
|
+
* Compute Cohen's κ, agreement %, and ROC-AUC over labeled pairs.
|
|
1022
|
+
*
|
|
1023
|
+
* Throws when pairs is empty or threshold is outside (0, 1). Cohen's κ
|
|
1024
|
+
* degenerates to 0 when all labels land in a single class for both
|
|
1025
|
+
* raters; ROC-AUC is reported as 0 when only one human-class is
|
|
1026
|
+
* present.
|
|
1027
|
+
*/
|
|
1028
|
+
declare function computeAgreement(pairs: readonly LabelPair[], threshold: number): AgreementResult;
|
|
1029
|
+
/**
|
|
1030
|
+
* Parse a 2- or 3-column CSV (input, human_label[, judge_score]).
|
|
1031
|
+
* Header detected when row[0][1] does not parse as a float. Lines
|
|
1032
|
+
* beginning with `#` are skipped.
|
|
1033
|
+
*/
|
|
1034
|
+
declare function loadLabelsCSV(text: string): LabeledRecord[];
|
|
1035
|
+
/**
|
|
1036
|
+
* Parse a newline-delimited JSON file. Each line is an object with the
|
|
1037
|
+
* shape {"input": "...", "human_label": 0.9, "judge_score": 0.8}.
|
|
1038
|
+
* `judge_score` is optional.
|
|
1039
|
+
*/
|
|
1040
|
+
declare function loadLabelsJSONL(text: string): LabeledRecord[];
|
|
1041
|
+
/**
|
|
1042
|
+
* Return the 16-character SHA-256 prefix used by the engine for prompt
|
|
1043
|
+
* hashes. Mirrors promptHash in engine/internal/assertion/judge_eval.go
|
|
1044
|
+
* so SDK-recorded calibration rows align with JudgeMetadata.prompt_hash.
|
|
1045
|
+
*/
|
|
1046
|
+
declare function promptHash(text: string): string;
|
|
1047
|
+
|
|
906
1048
|
declare const FRIENDLY_USER: SimulatePersona;
|
|
907
1049
|
declare const ADVERSARIAL_USER: SimulatePersona;
|
|
908
1050
|
declare const CONFUSED_USER: SimulatePersona;
|
|
@@ -1108,4 +1250,4 @@ interface TraceAdapter {
|
|
|
1108
1250
|
traceFromResponse(response: unknown, options?: Record<string, unknown>): Trace;
|
|
1109
1251
|
}
|
|
1110
1252
|
|
|
1111
|
-
export { ADVERSARIAL_USER, Agent, AgentResult, AlertDispatcher, type AlertPayload, AnthropicAdapter, Assertion, AssertionResult, AttestClient, type AttestPlugin, CONFUSED_USER, COOPERATIVE_USER, ContinuousEvalRunner, ConversationMessage, DriftReport, ENGINE_VERSION, EngineManager, EvaluateBatchResult, ExpectChain, FRIENDLY_USER, GeminiAdapter, InitializeResult, LangChainAdapter, ManualAdapter, OTelAdapter, OllamaAdapter, OpenAIAdapter, PluginRegistry, type PluginResult, Sampler, SimulateFaultConfig, SimulatePersona, Step, TIER_1, TIER_2, TIER_3, Trace, type TraceAdapter, TraceBuilder, TraceTree, VERSION, activeBuilder, agent, attestExpect, config, delegate, executePluginAssertion, getSampleRate, isSimulationMode, resetConfig,
|
|
1253
|
+
export { ADVERSARIAL_USER, Agent, AgentResult, type AgreementResult, AlertDispatcher, type AlertPayload, AnthropicAdapter, Assertion, AssertionResult, AttestClient, type AttestClientOptions, type AttestPlugin, CONFUSED_USER, COOPERATIVE_USER, ContinuousEvalRunner, ConversationMessage, DriftReport, ENGINE_VERSION, EngineManager, EvaluateBatchResult, ExpectChain, FRIENDLY_USER, GeminiAdapter, InitializeResult, type LabelPair, type LabeledRecord, LangChainAdapter, ManualAdapter, OTelAdapter, OllamaAdapter, OpenAIAdapter, Outcome, PluginRegistry, type PluginResult, ProtocolDiagnostic, ProtocolLogger, type RenderOptions, Sampler, SimulateFaultConfig, SimulatePersona, Step, TIER_1, TIER_2, TIER_3, Trace, type TraceAdapter, TraceBuilder, TraceTree, VERSION, activeBuilder, agent, attestExpect, computeAgreement, config, delegate, executePluginAssertion, getSampleRate, isSimulationMode, loadLabelsCSV, loadLabelsJSONL, promptHash, renderAssertionFailure, renderDiagnostics, renderSummary, requireSimulationCapability, resetConfig, simulationEnvEnabled, tier };
|