@reclaimprotocol/agent 0.1.36 → 0.1.38

package/src/proof/check-matchers.ts

@@ -0,0 +1,227 @@
+/**
+ * Offline matcher check: run a recipe's `responseRedactions` +
+ * `responseMatches` against a captured response with attestor-core's own HTTP
+ * provider, without a network call or a TLS session.
+ *
+ * It replays the two halves of a real proof on a synthetic transcript:
+ *   1. prover side: `providers.http.createRequest` builds the request and
+ *      `getResponseRedactions` picks the response slices to reveal (xPath /
+ *      jsonPath / regex, exactly as at proof time);
+ *   2. attestor side: `providers.http.assertValidProviderReceipt` checks the
+ *      redacted transcript against `responseMatches` and extracts params.
+ *
+ * Hashed (OPRF) slices stay in plaintext here: OPRF needs the attestor, so the
+ * check treats the value as its own nullifier. Everything else is the same
+ * engine the attestor runs.
+ */
+import {
+	CURRENT_ATTESTOR_VERSION,
+	type Logger,
+	PROVIDER_CTX,
+	providers,
+	type RedactedOrHashedArraySlice,
+	redactSlices,
+} from '@reclaimprotocol/attestor-core'
+// Side-effect: make external domhandler nodes DOM-like so attestor-core's
+// bundled `xpath` accepts them (see patch-domhandler.ts).
+import './patch-domhandler.ts'
+import { LOGGER } from '../logger.ts'
+import type { CapturedRequest, ReclaimProvider } from '../provider/schema.ts'
+import { buildHttpParams } from './proof-core.ts'
+
+/** Longest revealed slice returned; the full slice is still checked. */
+const MAX_SLICE_CHARS = 400
+/** Most revealed slices returned. */
+const MAX_SLICES = 10
+/** Longest error message returned (attestor errors can embed the base64 of
+ *  a whole element). */
+const MAX_REASON_CHARS = 600
+
+export interface RevealedSlice {
+	/** Byte offsets in the response body. */
+	from: number
+	to: number
+	/** Revealed text, truncated to {@link MAX_SLICE_CHARS}. */
+	text: string
+	/** Set when the slice is an OPRF-hashed value (plaintext here). */
+	hashed?: true
+}
+
+export interface MatcherCheckResult {
+	/** True when the attestor would accept this response. */
+	ok: boolean
+	/** Where it failed: building the request, selecting what to reveal, or
+	 *  checking `responseMatches` on the revealed bytes. */
+	failedAt?: 'request' | 'redaction' | 'match'
+	/** Attestor-core's error, truncated. */
+	reason?: string
+	/** Params the attestor would extract (regex groups + `paramValues` the
+	 *  matches use). */
+	extractedParameters: Record<string, string>
+	/** The response body slices revealed to the attestor. */
+	revealed: RevealedSlice[]
+	revealedSliceCount: number
+	revealedBytes: number
+	bodyBytes: number
+	/** Set when an OPRF-hashed redaction was checked in plaintext. */
+	note?: string
+}
+
+/** Placeholder secret header: the request builder needs one secret, and the
+ *  check never sends the request. */
+const OFFLINE_SECRET = { 'X-Reclaim-Offline-Check': '1' }
+
+/**
+ * Check `provider`'s matchers against `captured`'s response body, offline.
+ * Uses the captured status and content type, so a non-2xx capture fails the
+ * same way it would at proof time.
+ */
+export function checkMatchersOffline(
+	provider: ReclaimProvider,
+	captured: Pick<CapturedRequest, 'status' | 'contentType' | 'responseBody'>,
+): MatcherCheckResult {
+	const http = providers.http
+	const params = buildHttpParams(provider)
+	const logger = silentLogger()
+	const body = Buffer.from(captured.responseBody ?? '', 'utf8')
+	const head = Buffer.from(
+		`HTTP/1.1 ${captured.status} ${captured.status < 300 ? 'OK' : 'ERR'}\r\n`
+		+ `Content-Type: ${captured.contentType || 'text/plain'}\r\n`
+		+ `Content-Length: ${body.length}\r\n\r\n`,
+	)
+	const response = new Uint8Array(Buffer.concat([head, body]))
+	const empty = {
+		extractedParameters: {},
+		revealed: [],
+		revealedSliceCount: 0,
+		revealedBytes: 0,
+		bodyBytes: body.length,
+	}
+
+	let request: Uint8Array
+	try {
+		const req = http.createRequest(
+			{ headers: OFFLINE_SECRET }, structuredClone(params), logger,
+		)
+		const data = typeof req.data === 'string'
+			? new Uint8Array(Buffer.from(req.data))
+			: req.data
+		request = redactSlices(data, req.redactions)
+	} catch(err) {
+		return { ...empty, ok: false, failedAt: 'request', reason: reasonOf(err) }
+	}
+
+	let slices: RedactedOrHashedArraySlice[]
+	try {
+		slices = http.getResponseRedactions?.({
+			response, params: structuredClone(params), logger, ctx: PROVIDER_CTX,
+		}) ?? []
+	} catch(err) {
+		return { ...empty, ok: false, failedAt: 'redaction', reason: reasonOf(err) }
+	}
+
+	const hidden = slices.filter((s) => !s.hash)
+	const hashed = slices.filter((s) => s.hash)
+	const view = revealedSlices(head.length, response.length, hidden, hashed)
+	const redacted = redactSlices(response, hidden)
+	const seen = {
+		revealed: view.slice(0, MAX_SLICES).map((s) => ({
+			from: s.from,
+			to: s.to,
+			text: truncate(
+				Buffer.from(response.subarray(head.length + s.from, head.length + s.to))
+					.toString('utf8'),
+				MAX_SLICE_CHARS,
+			),
+			...(s.hashed ? { hashed: true as const } : {}),
+		})),
+		revealedSliceCount: view.length,
+		revealedBytes: view.reduce((n, s) => n + s.to - s.from, 0),
+		bodyBytes: body.length,
+		...(hashed.length
+			? {
+				note: 'OPRF-hashed values were checked in plaintext; at proof time '
+					+ 'the attestor sees their nullifier instead.',
+			}
+			: {}),
+	}
+
+	try {
+		const out = http.assertValidProviderReceipt({
+			clientVersion: CURRENT_ATTESTOR_VERSION,
+			receipt: [
+				{ sender: 'client', message: request },
+				{ sender: 'server', message: redacted },
+			],
+			params: structuredClone(params),
+			logger,
+			ctx: PROVIDER_CTX,
+		})
+		// The http provider validates synchronously; a thenable here would mean
+		// a provider we don't run offline.
+		const extracted = out instanceof Promise ? undefined : out
+		return {
+			ok: true,
+			extractedParameters: extracted?.extractedParameters ?? {},
+			...seen,
+		}
+	} catch(err) {
+		return {
+			ok: false,
+			failedAt: 'match',
+			reason: reasonOf(err),
+			extractedParameters: {},
+			...seen,
+		}
+	}
+}
+
+/** Body-relative revealed ranges: the complement of `hidden` within the body,
+ *  merged with the hashed slices (revealed as their nullifier at proof time).
+ */
+function revealedSlices(
+	bodyStart: number,
+	total: number,
+	hidden: RedactedOrHashedArraySlice[],
+	hashed: RedactedOrHashedArraySlice[],
+): { from: number, to: number, hashed?: true }[] {
+	const out: { from: number, to: number, hashed?: true }[] = []
+	let cursor = bodyStart
+	const sorted = [...hidden].sort((a, b) => a.fromIndex - b.fromIndex)
+	for(const s of sorted) {
+		if(s.fromIndex > cursor) {
+			out.push({ from: cursor - bodyStart, to: s.fromIndex - bodyStart })
+		}
+
+		cursor = Math.max(cursor, s.toIndex)
+	}
+
+	if(cursor < total) {
+		out.push({ from: cursor - bodyStart, to: total - bodyStart })
+	}
+
+	for(const h of hashed) {
+		out.push({
+			from: h.fromIndex - bodyStart, to: h.toIndex - bodyStart, hashed: true,
+		})
+	}
+
+	return out
+		.filter((s) => s.to > s.from && s.from >= 0)
+		.sort((a, b) => a.from - b.from)
+}
+
+function reasonOf(err: unknown): string {
+	return truncate(err instanceof Error ? err.message : String(err),
+		MAX_REASON_CHARS)
+}
+
+function truncate(s: string, max: number): string {
+	return s.length > max ? `${s.slice(0, max)}… (${s.length} chars)` : s
+}
+
+/** A logger for attestor-core calls that records nothing: the check reports
+ *  its outcome in the result, and the provider logs whole responses. */
+function silentLogger(): Logger {
+	return LOGGER.child({ scope: 'check-matchers' }, { level: 'silent' })
+}

package/src/proof/zk-files.ts

@@ -1,84 +1,376 @@
-// Lazy download of the @reclaimprotocol/zk-symmetric-crypto circuit resources
-// the proof engines need (the downloader fetches every engine — stwo, snarkjs,
-// gnark — as one bundle). These are ~280MB and are NOT fetched in a
-// `postinstall` (that blocked the MCP server's first launch past the connect
-// timeout — see the install notes). Instead we fetch them on the first proof,
-// once, so the server starts immediately and only `run_proof` pays the cost.
-
-import { spawn } from 'node:child_process'
-import { existsSync } from 'node:fs'
+// Download of the @reclaimprotocol/zk-symmetric-crypto circuit resources the
+// proof engines read from disk. Never a `postinstall`: that blocked the MCP
+// server's first launch past the connect timeout. Instead the MCP server starts
+// the download in the background at launch (`startZkFilesDownload`), and every
+// proof awaits it (`ensureZkFiles`) as the safety net.
+//
+// Proofs default to the stwo engine. Its resources are ~1.5MB, so we fetch only
+// `resources/stwo` file by file from the package's pinned commit and check each
+// file's git blob hash against the listing for that commit. When that fails
+// (GitHub API rate limit, a changed layout), we fall back to the package's own
+// downloader, which fetches the whole repository archive (~350MB).
+// (In zk-symmetric-crypto 5.1.x the stwo prover loads its WASM from the npm
+// package itself; `resources/stwo` is kept as the readiness signal.)
+
+import { badGateway, internal } from '@hapi/boom'
+import assert from 'node:assert'
+import { spawn as nodeSpawn, type StdioOptions } from 'node:child_process'
+import { createHash } from 'node:crypto'
+import { existsSync, statSync } from 'node:fs'
+import {
+	mkdir,
+	mkdtemp,
+	rename,
+	rm,
+	writeFile
+} from 'node:fs/promises'
 import { createRequire } from 'node:module'
 import { dirname, join } from 'node:path'
+import { pathToFileURL } from 'node:url'
 
-const require = createRequire(import.meta.url)
+const REPO = 'reclaimprotocol/zk-symmetric-crypto'
+/** The stwo resource dir; its presence means the circuits are ready. */
+const STWO_DIR = join('resources', 'stwo')
+/** Where the package's downloader writes the archive, relative to its cwd. */
+const ARCHIVE_PATH = join('zk-symmetric-crypto-download', 'repo.zip')
 
-let inFlight: Promise<void> | undefined
+export type ZkFilesState =
+	| { status: 'idle' }
+	| {
+		status: 'downloading'
+		/** `stwo-files`: the ~1.5MB stwo files. `full-archive`: the ~350MB
+		 *  repository archive (fallback). */
+		method: 'stwo-files' | 'full-archive'
+		elapsedMs: number
+		bytesDownloaded?: number
+		totalBytes?: number
+	}
+	| { status: 'ready' }
+	| { status: 'failed', reason: string }
+
+/** Minimal child process surface `download-files.js` is driven through. */
+export interface ChildLike {
+	once(event: 'exit', listener: (code: number | null) => void): unknown
+	once(event: 'error', listener: (err: Error) => void): unknown
+}
+
+interface Progress {
+	method: 'stwo-files' | 'full-archive'
+	startedAt: number
+	bytesDownloaded?: number
+	totalBytes?: number
+	/** The archive file whose size is the progress (full-archive only). */
+	archivePath?: string
+}
+
+export interface ZkFilesDeps {
+	/** The zk-symmetric-crypto package root, or undefined when not installed. */
+	resolvePackageDir?: () => string | undefined
+	/** The pinned repository commit the package's resources come from. */
+	readCommit?: (pkgDir: string) => Promise<string>
+	fetch?: typeof fetch
+	spawn?: (
+		command: string,
+		args: string[],
+		options: { cwd: string, stdio: StdioOptions },
+	) => ChildLike
+	/** Progress lines. Defaults to stderr: stdout carries MCP JSON-RPC. */
+	log?: (message: string) => void
+}
 
 /**
- * Ensure the zk circuit resources are on disk before a proof runs. Idempotent
- * and concurrency-safe: the download runs at most once per process, and a
- * failed attempt clears the latch so a later proof can retry.
- *
- * Resolves silently when the resources are already present (the warm npx cache,
- * or after a manual `npm run download:zk-files`) and when
- * `@reclaimprotocol/zk-symmetric-crypto` isn't installed at all (SDK-only
- * consumers have nothing to prove).
- *
- * All output goes to stderr (fd 2): the MCP server speaks JSON-RPC over stdout,
- * so a stray write to fd 1 mid-session would corrupt the protocol.
+ * Build a downloader with its own state. The module exports one default
+ * instance; tests build their own with injected `spawn` and `fetch`.
  */
-export function ensureZkFiles() {
-	inFlight ??= download().catch((err) => {
-		inFlight = undefined
-		throw err
+export function makeZkFiles(deps: ZkFilesDeps = {}) {
+	const resolvePackageDir = deps.resolvePackageDir ?? resolveZkPackageDir
+	const readCommit = deps.readCommit ?? readPinnedCommit
+	const fetchFn = deps.fetch ?? fetch
+	const spawn = deps.spawn ?? nodeSpawn
+	const log = deps.log ?? ((message: string) => {
+		process.stderr.write(`[reclaim-agent] ${message}\n`)
 	})
-	return inFlight
-}
 
-async function download() {
-	let mainPath: string
-	try {
-		mainPath = require.resolve('@reclaimprotocol/zk-symmetric-crypto')
-	} catch{
-		return
+	let inFlight: Promise<void> | undefined
+	let failure: string | undefined
+	let ready = false
+	let progress: Progress | undefined
+
+	return { ensure, start, state }
+
+	/**
+	 * Ensure the circuit resources are on disk. Idempotent and
+	 * concurrency-safe: one download runs at a time per process, and a failed
+	 * attempt clears the latch so a later call retries.
+	 */
+	function ensure() {
+		inFlight ??= download()
+			.then(() => {
+				ready = true
+				failure = undefined
+			}, (err: unknown) => {
+				inFlight = undefined
+				failure = messageOf(err)
+				throw err
+			})
+			.finally(() => {
+				progress = undefined
+			})
+		return inFlight
 	}
 
-	// mainPath is <pkg>/lib/index.js → resources live at <pkg>/resources, the
-	// downloader at <pkg>/lib/scripts/download-files.js. Proofs default to the
-	// stwo (WASM) engine, so its resource dir being present is the signal that
-	// the circuits are ready (the downloader fetches all engines together).
-	const libDir = dirname(mainPath)
-	const pkgDir = dirname(libDir)
-	if(existsSync(join(pkgDir, 'resources', 'stwo'))) {
-		return
+	/** Start `ensure` in the background; `state` reports the outcome. */
+	function start() {
+		ensure().catch(() => {})
 	}
 
-	const downloadScript = join(libDir, 'scripts', 'download-files.js')
-	if(!existsSync(downloadScript)) {
-		throw new Error(
-			'zk circuit downloader not found in '
-			+ '@reclaimprotocol/zk-symmetric-crypto; '
-			+ 'run `npm run download:zk-files` and retry.',
+	function state(): ZkFilesState {
+		if(ready) {
+			return { status: 'ready' }
+		}
+
+		if(progress) {
+			const bytesDownloaded = progress.archivePath
+				? fileSize(progress.archivePath)
+				: progress.bytesDownloaded
+			return {
+				status: 'downloading',
+				method: progress.method,
+				elapsedMs: Date.now() - progress.startedAt,
+				...(bytesDownloaded !== undefined ? { bytesDownloaded } : {}),
+				...(progress.totalBytes !== undefined
+					? { totalBytes: progress.totalBytes }
+					: {}),
+			}
+		}
+
+		if(failure !== undefined) {
+			return { status: 'failed', reason: failure }
+		}
+
+		return { status: 'idle' }
+	}
+
+	async function download() {
+		const pkgDir = resolvePackageDir()
+		// SDK-only consumers don't install the package: nothing to prove.
+		if(!pkgDir || existsSync(join(pkgDir, STWO_DIR))) {
+			return
+		}
+
+		try {
+			progress = { method: 'stwo-files', startedAt: Date.now() }
+			await fetchStwoFiles(pkgDir, progress)
+		} catch(err) {
+			const stwoError = messageOf(err)
+			log(
+				`fetching the stwo proof files failed (${stwoError}); `
+				+ 'downloading the full circuit archive (~350MB, one-time)…',
+			)
+			progress = {
+				method: 'full-archive',
+				startedAt: Date.now(),
+				archivePath: join(pkgDir, ARCHIVE_PATH),
+			}
+			try {
+				await runArchiveDownloader(pkgDir)
+			} catch(archiveErr) {
+				throw badGateway(
+					`stwo files: ${stwoError}; full archive: ${messageOf(archiveErr)}`,
+				)
+			}
+		}
+
+		assert(
+			existsSync(join(pkgDir, STWO_DIR)),
+			internal(
+				`zk circuit download finished but ${join(pkgDir, STWO_DIR)} `
+				+ 'is missing; run `npm run download:zk-files` and retry.',
+			),
 		)
 	}
 
-	process.stderr.write(
-		'[reclaim-agent] downloading zkTLS proof circuits '
-		+ '(~280MB, one-time)…\n',
-	)
-	// stdio fd 1→2: send the downloader's stdout to our stderr so its progress
-	// never lands on the MCP server's JSON-RPC stdout.
-	await new Promise<void>((resolve, reject) => {
-		const child = spawn(process.execPath, [downloadScript], {
-			stdio: ['ignore', 2, 2],
-		})
-		child.on('error', reject)
-		child.on('exit', (code) => {
-			if(code === 0) {
-				resolve()
-			} else {
-				reject(new Error(`zk download exited ${code}`))
+	/**
+	 * Fetch `resources/stwo` at the pinned commit, file by file, into a temp
+	 * dir inside the package, then rename it into place (same filesystem).
+	 */
+	async function fetchStwoFiles(pkgDir: string, stats: Progress) {
+		const commit = await readCommit(pkgDir)
+		assert(/^[0-9a-f]{40}$/.test(commit), internal('invalid pinned commit'))
+		const files = await listStwoFiles(commit)
+		stats.bytesDownloaded = 0
+		stats.totalBytes = files.reduce((sum, f) => sum + f.size, 0)
+		log(
+			`downloading zkTLS stwo proof files (${stats.totalBytes} bytes, `
+			+ 'one-time)…',
+		)
+
+		const tmp = await mkdtemp(join(pkgDir, '.stwo-download-'))
+		try {
+			for(const file of files) {
+				const res = await fetchFn(
+					`https://raw.githubusercontent.com/${REPO}/${commit}/`
+					+ `resources/stwo/${file.name}`,
+				)
+				assert(
+					res.ok,
+					badGateway(`stwo file ${file.name}: HTTP ${res.status}`),
+				)
+				const bytes = new Uint8Array(await res.arrayBuffer())
+				assert(
+					gitBlobSha(bytes) === file.sha,
+					badGateway(`stwo file ${file.name} failed its integrity check`),
+				)
+				await writeFile(join(tmp, file.name), bytes)
+				stats.bytesDownloaded = (stats.bytesDownloaded ?? 0) + bytes.length
 			}
+
+			const dest = join(pkgDir, STWO_DIR)
+			await mkdir(dirname(dest), { recursive: true })
+			// Another process may have finished first; its copy is as good.
+			await rename(tmp, dest).catch((err: unknown) => {
+				if(!existsSync(dest)) {
+					throw err
+				}
+			})
+		} finally {
+			await rm(tmp, { recursive: true, force: true })
+		}
+	}
+
+	/** The file listing of `resources/stwo` at `commit`, with blob hashes. */
+	async function listStwoFiles(commit: string) {
+		const res = await fetchFn(
+			`https://api.github.com/repos/${REPO}/contents/resources/stwo`
+			+ `?ref=${commit}`,
+			{ headers: { Accept: 'application/vnd.github+json' } },
+		)
+		assert(res.ok, badGateway(`stwo file listing: HTTP ${res.status}`))
+		const listing: unknown = await res.json()
+		assert(
+			Array.isArray(listing) && listing.length > 0,
+			badGateway('stwo file listing is empty'),
+		)
+		return listing.map((entry: unknown) => {
+			assert(
+				typeof entry === 'object' && entry !== null
+				&& 'type' in entry && entry.type === 'file'
+				&& 'name' in entry && typeof entry.name === 'string'
+				&& /^[\w.-]+$/.test(entry.name) && !entry.name.startsWith('.')
+				&& 'size' in entry && typeof entry.size === 'number'
+				&& 'sha' in entry && typeof entry.sha === 'string',
+				badGateway('unexpected entry in the stwo file listing'),
+			)
+			return { name: entry.name, size: entry.size, sha: entry.sha }
 		})
-	})
+	}
+
+	/**
+	 * Run the package's `download-files.js`. Its temp paths are relative, so
+	 * cwd is the package dir: the archive and extracted tree then sit on the
+	 * same filesystem as the final `rename()` (no EXDEV) and never land in the
+	 * developer's project.
+	 */
+	async function runArchiveDownloader(pkgDir: string) {
+		const script = join(pkgDir, 'lib', 'scripts', 'download-files.js')
+		assert(
+			existsSync(script),
+			internal(
+				'zk circuit downloader not found in '
+				+ '@reclaimprotocol/zk-symmetric-crypto; '
+				+ 'run `npm run download:zk-files` and retry.',
+			),
+		)
+
+		await new Promise<void>((resolve, reject) => {
+			// stdio fd 1→2: the downloader's progress must never reach the MCP
+			// server's JSON-RPC stdout.
+			const child = spawn(process.execPath, [script], {
+				cwd: pkgDir,
+				stdio: ['ignore', 2, 2],
+			})
+			child.once('error', reject)
+			child.once('exit', (code) => {
+				if(code === 0) {
+					resolve()
+				} else {
+					reject(internal(`zk circuit download exited with code ${code}`))
+				}
+			})
+		})
+	}
+}
+
+const DEFAULT = makeZkFiles()
+
+/**
+ * Ensure the zk circuit resources are on disk before a proof runs. Resolves
+ * at once when they're present or the package isn't installed.
+ */
+export function ensureZkFiles() {
+	return DEFAULT.ensure()
+}
+
+/** Start the download in the background (MCP server launch). */
+export function startZkFilesDownload() {
+	DEFAULT.start()
+}
+
+/** Where the download stands: idle, downloading, ready, or failed. */
+export function zkFilesState() {
+	return DEFAULT.state()
+}
+
+/**
+ * The zk-symmetric-crypto copy attestor-core loads (it may be nested under
+ * attestor-core), else the hoisted one.
+ */
+function resolveZkPackageDir() {
+	const require = createRequire(import.meta.url)
+	for(const from of [
+		() => createRequire(require.resolve('@reclaimprotocol/attestor-core')),
+		() => require,
+	]) {
+		try {
+			// <pkg>/lib/index.js → <pkg>
+			return dirname(dirname(
+				from().resolve('@reclaimprotocol/zk-symmetric-crypto'),
+			))
+		} catch{
+			// try the next resolution root
+		}
+	}
+
+	return undefined
+}
+
+async function readPinnedCommit(pkgDir: string) {
+	const mod: { GIT_COMMIT_HASH?: unknown } = await import(
+		pathToFileURL(join(pkgDir, 'lib', 'index.js')).href
+	)
+	assert(
+		typeof mod.GIT_COMMIT_HASH === 'string',
+		internal('zk-symmetric-crypto exports no GIT_COMMIT_HASH'),
+	)
+	return mod.GIT_COMMIT_HASH
+}
+
+/** Git's blob object id: sha1 of `blob <size>\0<content>`. */
+function gitBlobSha(bytes: Uint8Array) {
+	return createHash('sha1')
+		.update(`blob ${bytes.length}\0`)
+		.update(bytes)
+		.digest('hex')
+}
+
+function fileSize(path: string) {
+	try {
+		return statSync(path).size
+	} catch{
+		return undefined
+	}
+}
+
+function messageOf(err: unknown) {
+	return err instanceof Error ? err.message : String(err)
 }