@reclaimprotocol/agent 0.1.5 → 0.1.6

package/src/proof/patch-domhandler.ts

@@ -0,0 +1,68 @@
+// Side-effect module: make `domhandler` nodes DOM-like for the `xpath` engine.
+//
+// WHY THIS EXISTS
+// `@reclaimprotocol/attestor-core`'s published `lib` bundles its own inlined
+// copy of `domhandler` and applies a DOM-compat patch (nodeName / localName +
+// a NamedNodeMap `item()` shim) to THAT copy. But at proof time the HTML tree
+// is built by the *external* `parse5-htmlparser2-tree-adapter`, which uses the
+// separately-installed `domhandler` — a different, UNPATCHED copy. The `xpath`
+// engine then fails `isNodeLike()` on every node (it needs a string `nodeName`)
+// and throws "Context node does not appear to be a valid DOM node", so ALL
+// xPath / xPath+jsonPath response redactions fail during `runAttestorProof`'s
+// receipt validation. (Regex-only redactions escape it — they never touch the
+// parse5/xpath path.) Running attestor-core from source works because there is
+// then a single shared `domhandler`; the bug is only in the bundled `lib`.
+//
+// Rather than fork attestor-core, apply the identical patch to the external
+// `domhandler` the adapter actually uses, so its tree nodes are DOM-like. This
+// mirrors attestor-core's own `patch-parse5-tree`. Idempotent and
+// version-agnostic. Import it before any proving path runs.
+//
+// Remove once attestor-core externalizes `domhandler` in its build (one shared
+// copy → its own patch suffices).
+import { Element, Node } from 'domhandler'
+
+const PATCH_FLAG = Symbol.for('reclaim.domhandler.domCompatPatched')
+const proto = Node.prototype as unknown as Record<PropertyKey, unknown>
+
+if(!proto[PATCH_FLAG]) {
+	// `xpath`'s isNodeLike() requires a string `nodeName`; domhandler nodes carry
+	// the tag under `.name`.
+	Object.defineProperty(Node.prototype, 'nodeName', {
+		configurable: true,
+		get(this: { name?: string }) {
+			return this.name
+		},
+	})
+	Object.defineProperty(Node.prototype, 'localName', {
+		configurable: true,
+		get(this: { name?: string }) {
+			return this.name
+		},
+	})
+
+	// Attribute-predicate steps (`[@id=…]`, `contains(@class,…)`) walk the
+	// NamedNodeMap via `.item(i)`; give each returned attribute node the DOM
+	// shape `xpath` expects (nodeType 2 + localName).
+	const attrsGetter =
+		Object.getOwnPropertyDescriptor(Element.prototype, 'attributes')?.get
+	if(attrsGetter) {
+		Object.defineProperty(Element.prototype, 'attributes', {
+			configurable: true,
+			get(this: unknown) {
+				const attrs = attrsGetter.call(this) as
+					(Record<number, { name: string }> & {
+						item?: (i: number) => unknown
+					})
+				attrs.item = (i: number) => {
+					const attr = attrs[i]
+					return { ...attr, nodeType: 2, localName: attr.name }
+				}
+
+				return attrs
+			},
+		})
+	}
+
+	proto[PATCH_FLAG] = true
+}

package/src/proof/zk-files.ts

@@ -1,6 +1,7 @@
 // Lazy download of the @reclaimprotocol/zk-symmetric-crypto circuit resources
-// that the snarkjs proof engine needs. These are ~280MB and are NOT fetched in
-// a `postinstall` (that blocked the MCP server's first launch past the connect
+// the proof engines need (the downloader fetches every engine — stwo, snarkjs,
+// gnark — as one bundle). These are ~280MB and are NOT fetched in a
+// `postinstall` (that blocked the MCP server's first launch past the connect
 // timeout — see the install notes). Instead we fetch them on the first proof,
 // once, so the server starts immediately and only `run_proof` pays the cost.
 
@@ -43,12 +44,12 @@ async function download(): Promise<void> {
 	}
 
 	// mainPath is <pkg>/lib/index.js → resources live at <pkg>/resources, the
-	// downloader at <pkg>/lib/scripts/download-files.js. We only use the snarkjs
-	// engine (createClaimOnAttestor zkEngine: 'snarkjs'), so its presence is the
-	// signal that the circuits are ready.
+	// downloader at <pkg>/lib/scripts/download-files.js. Proofs default to the
+	// stwo (WASM) engine, so its resource dir being present is the signal that
+	// the circuits are ready (the downloader fetches all engines together).
 	const libDir = dirname(mainPath)
 	const pkgDir = dirname(libDir)
-	if(existsSync(join(pkgDir, 'resources', 'snarkjs'))) {
+	if(existsSync(join(pkgDir, 'resources', 'stwo'))) {
 		return
 	}
 

package/src/provider/dom-xpath.ts

@@ -0,0 +1,225 @@
+// Generate a verified xPath to the element holding a target value, using the
+// SAME parser the attestor uses (parse5 + parse5-htmlparser2-tree-adapter) and
+// self-verifying each candidate with the same `xpath` engine. Because the path
+// is built from — and checked against — the attestor's own parse of the raw
+// response body, it provably resolves to that element at proof time, on any
+// site (no per-endpoint logic, no fragile byte-offset counting).
+//
+// The domhandler patch makes the tree's nodes DOM-like for `xpath` (see
+// patch-domhandler.ts — mirrors attestor-core's patch-parse5-tree).
+import { type Element, isTag, isText, type Node } from 'domhandler'
+import { parse } from 'parse5'
+import { adapter as htmlAdapter } from 'parse5-htmlparser2-tree-adapter'
+import xpath from 'xpath'
+import '../proof/patch-domhandler.ts'
+
+/** Identifying attributes stable enough to anchor an xPath on (value-bearing
+ *  attrs like `content`/`href`/`value` are excluded). */
+const IDENT_ATTRS =
+	['name', 'property', 'itemprop', 'data-testid', 'rel', 'for']
+
+export interface DomAnchor {
+	xPath: string
+	/** Byte range of the element in the source html (parse5 source locations),
+	 *  matching the slice the attestor reveals for this xPath. */
+	start: number
+	end: number
+}
+
+/**
+ * A verified xPath to the tightest element containing one of `targets`, or
+ * undefined if the value isn't present in the parsed tree. Prefers a short
+ * stable selector (unique id → identifying attr → class), falling back to an
+ * absolute child-indexed path (`/html[1]/body[1]/…/option[1]`). Every candidate
+ * is checked with the real `xpath` engine and only returned if it resolves,
+ * first-match, to that exact element.
+ */
+export function elementAnchor(
+	html: string,
+	targets: string[],
+): DomAnchor | undefined {
+	let doc: unknown
+	try {
+		doc = parse(html, {
+			treeAdapter: htmlAdapter,
+			sourceCodeLocationInfo: true,
+		})
+	} catch{
+		return undefined
+	}
+
+	// `xpath` needs a named root node to start from (as attestor-core does).
+	;(doc as { name: string }).name = 'root'
+
+	const el = findElement(doc as Node, targets)
+	if(!el) {
+		return undefined
+	}
+
+	const { startIndex, endIndex } = el
+	if(startIndex === null || endIndex === null) {
+		return undefined
+	}
+
+	for(const candidate of candidateXPaths(el)) {
+		if(matchesUniquely(doc, candidate, el)) {
+			// `[startIndex, endIndex)` is exactly the slice the attestor reveals for
+			// this xPath (its getNodeRange uses these same parse5 source locations).
+			return { xPath: candidate, start: startIndex, end: endIndex }
+		}
+	}
+
+	return undefined
+}
+
+/** The tightest element that DIRECTLY holds a target. Prefers a COMPLETE-value
+ *  occurrence — an attribute whose value IS the target, or a text child that IS
+ *  the target — over one where the target is merely embedded in a longer string
+ *  (e.g. a userid inside a `1234567+user@…` noreply address). The complete
+ *  field is the semantic source and yields a portable anchor; the embedded one
+ *  is incidental. Smallest byte span wins within the chosen tier. */
+function findElement(root: Node, targets: string[]): Element | undefined {
+	const complete: Element[] = []
+	const embedded: Element[] = []
+
+	const walk = (node: Node): void => {
+		if(isTag(node)) {
+			const el = node
+			const texts = el.children.filter(isText).map((c) => c.data)
+			const attrs = Object.values(el.attribs ?? {})
+			const isTarget = (v: string): boolean => targets.some((t) => v === t)
+			const hasTarget =
+				(v: string): boolean => targets.some((t) => v.includes(t))
+			if(texts.some((t) => isTarget(t.trim())) || attrs.some(isTarget)) {
+				complete.push(el)
+			} else if(texts.some(hasTarget) || attrs.some(hasTarget)) {
+				embedded.push(el)
+			}
+		}
+
+		if('children' in node && Array.isArray(node.children)) {
+			for(const child of node.children) {
+				walk(child)
+			}
+		}
+	}
+
+	walk(root)
+	return smallestElement(complete.length ? complete : embedded)
+}
+
+/** Byte span of an element (0 when source locations are missing). */
+function elementSpan(e: Element): number {
+	return (e.endIndex ?? 0) - (e.startIndex ?? 0)
+}
+
+/** The element with the smallest byte span (the tightest / deepest match). */
+function smallestElement(els: Element[]): Element | undefined {
+	return els
+		.filter((e) => e.startIndex !== null && e.endIndex !== null)
+		.sort((a, b) => elementSpan(a) - elementSpan(b))
+		.at(0)
+}
+
+/** Candidate xPaths for `el`, best (most stable / shortest) first. */
+function* candidateXPaths(el: Element): Generator<string> {
+	const tag = el.name
+	const a = el.attribs ?? {}
+	if(a.id) {
+		yield `//${tag}[@id=${xpathLiteral(a.id)}]`
+	}
+
+	for(const name of IDENT_ATTRS) {
+		if(a[name]) {
+			yield `//${tag}[@${name}=${xpathLiteral(a[name])}]`
+		}
+	}
+
+	if(a.class) {
+		const distinctive = a.class
+			.split(/\s+/)
+			.filter(Boolean)
+			.sort((x, y) => y.length - x.length)[0]
+		if(distinctive) {
+			yield `//${tag}[contains(@class, ${xpathLiteral(distinctive)})]`
+		}
+	}
+
+	// A uniquely-identified DIRECT child pins the parent without a positional
+	// path (`//span[span[@id="labels"]]`). Self-verification (firstSelected)
+	// discards it when the child id/attr isn't unique — e.g. GitHub reuses
+	// `id="labels"` per email row — so it never silently matches the wrong node.
+	for(const child of el.children) {
+		if(!isTag(child)) {
+			continue
+		}
+
+		const ca = child.attribs ?? {}
+		if(ca.id) {
+			yield `//${tag}[${child.name}[@id=${xpathLiteral(ca.id)}]]`
+		}
+
+		for(const name of IDENT_ATTRS) {
+			if(ca[name]) {
+				yield `//${tag}[${child.name}[@${name}=${xpathLiteral(ca[name])}]]`
+			}
+		}
+	}
+
+	yield absolutePath(el)
+}
+
+/** Chrome "full XPath"-style absolute path: `/html[1]/body[1]/…/tag[n]`, where
+ *  n is the element's 1-based position among same-tag siblings. */
+function absolutePath(el: Element): string {
+	const parts: string[] = []
+	let node: Node | null = el
+	while(node && isTag(node)) {
+		const tag = node.name
+		let index = 1
+		for(let sib = node.prev; sib; sib = sib.prev) {
+			if(isTag(sib) && sib.name === tag) {
+				index++
+			}
+		}
+
+		parts.unshift(`/${tag}[${index}]`)
+		node = node.parent
+	}
+
+	return parts.join('')
+}
+
+/** True when `expression` selects EXACTLY `el` (one match) against `doc`. The
+ *  attestor applies the redaction regex to EVERY xPath match and throws if any
+ *  fails, so a non-unique selector (e.g. a class shared by sibling rows) is
+ *  unusable — it must resolve to the single target element. */
+function matchesUniquely(doc: unknown, expression: string, el: Node): boolean {
+	try {
+		const engine = xpath as unknown as {
+			parse(e: string): { select(o: unknown): Node[] }
+		}
+		const nodes = engine.parse(expression).select({
+			node: doc,
+			allowAnyNamespaceForNoPrefix: true,
+		})
+		return nodes.length === 1 && nodes[0] === el
+	} catch{
+		return false
+	}
+}
+
+/** Quote a string as an XPath 1.0 string literal (concat() when it holds both
+ *  quote characters). */
+function xpathLiteral(s: string): string {
+	if(!s.includes('"')) {
+		return `"${s}"`
+	}
+
+	if(!s.includes("'")) {
+		return `'${s}'`
+	}
+
+	const parts = s.split('"').map((p) => `"${p}"`)
+	return `concat(${parts.join(', \'"\', ')})`
+}

package/src/provider/draft.ts

@@ -3,35 +3,129 @@ import type {
 	CapturedRequest,
 	OprfMode,
 	ReclaimProvider,
+	ResponseMatch,
+	ResponseRedaction,
 	SecretParamRef,
 } from './schema.ts'
 import { splitSecrets } from './secrets.ts'
 
+/** One value to extract from the request: the captured `value`, a `name` (its
+ *  param / capture-group), and an optional per-target OPRF `hash`. */
+export interface DraftTarget {
+	value: string
+	name: string
+	hash?: OprfMode
+}
+
+/** The realized extraction for one target — its value paired with the
+ *  redaction that reveals it (absent when matched literally). Lets a consumer
+ *  validate each value against its own selector (e.g. the cloud replay). */
+export interface Extraction {
+	name: string
+	value: string
+	redaction?: ResponseRedaction
+}
+
 export interface DraftResult {
 	provider: ReclaimProvider
 	secretRefs: SecretParamRef[]
+	/** One entry per target, in `responseMatches` order. */
+	extractions: Extraction[]
 }
 
-/** Replace any existing `accept-encoding` (case-insensitive) with a
- *  canonical `Accept-Encoding: identity`. Strips browser-captured
- *  encodings like `gzip, deflate, br, zstd` that would otherwise make
- *  the attestor match against compressed response bytes. */
-function withIdentityEncoding(
+/** Request headers that must NOT survive into the provider params. Two kinds:
+ *  - Conditional headers (`if-none-match`/`if-modified-since`/…): a captured
+ *    cached ETag/date makes the origin answer 304/412 when the attestor
+ *    re-issues, and the attestor treats non-2xx as failure. We always want the
+ *    full fresh response.
+ *  - Transfer headers the ATTESTOR sets itself (`connection: close`,
+ *    `accept-encoding: identity`). Passing our own duplicates/collides with the
+ *    request the attestor builds, so leave them out — the attestor already
+ *    forces `identity` (uncompressed, so its byte-matching works). */
+const STRIP_REQUEST_HEADERS = new Set([
+	'if-none-match',
+	'if-modified-since',
+	'if-match',
+	'if-unmodified-since',
+	'if-range',
+	'connection',
+	'accept-encoding',
+])
+
+/** Drop the headers the attestor manages or that would 304 at proof time
+ *  (see {@link STRIP_REQUEST_HEADERS}); keep every other captured header so the
+ *  attestor's replay looks like the browser's request. */
+function sanitizeRequestHeaders(
 	headers: Record<string, string>,
 ): Record<string, string> {
 	const out: Record<string, string> = {}
 	for(const [k, v] of Object.entries(headers)) {
-		if(k.toLowerCase() === 'accept-encoding') {
+		if(STRIP_REQUEST_HEADERS.has(k.toLowerCase())) {
 			continue
 		}
 
 		out[k] = v
 	}
 
-	out['Accept-Encoding'] = 'identity'
 	return out
 }
 
+/** Reject a draft that bakes a captured per-user value as a LITERAL into a
+ *  match or redaction — either a neighbouring target's value or a target's own
+ *  value left un-templated. The matcher builds each target's window
+ *  independently and drops the per-user WORDS it can see, but it cannot know a
+ *  neighbouring value is itself user-specific (e.g. a name sharing a text node
+ *  with an id — a `<b>{{id}}</b>` sitting beside a `<td>{{name}}</td>`).
+ *  Cross-checking every value against every match here catches that: a portable
+ *  provider contains NO captured value as a literal — each appears only as its
+ *  `{{param}}`. Throwing discards the draft so the agent re-captures or
+ *  re-proposes rather than shipping a provider pinned to this one user. */
+export function assertPortableMatches(
+	matches: ResponseMatch[],
+	redactions: ResponseRedaction[],
+	targets: DraftTarget[],
+): void {
+	// Guard only values long enough to be plausibly user-specific; shorter runs
+	// (single digits, 2-3 char tokens) collide with ordinary structural bytes.
+	const MIN_ALNUM = 4
+	const guarded = targets
+		.map((t) => t.value)
+		.filter((v) => v.replace(/[^A-Za-z0-9]/g, '').length >= MIN_ALNUM)
+	if(!guarded.length) {
+		return
+	}
+
+	const stripParams = (s: string): string => s.replace(/\{\{[^}]*\}\}/g, '')
+	const scan = (text: string, where: string, own: string | null): void => {
+		const bare = stripParams(text)
+		for(const v of guarded) {
+			if(bare.includes(v)) {
+				const kind = v === own
+					? 'its own value un-templated'
+					: "a neighbouring target's per-user value"
+				throw new Error(
+					`drafted ${where} bakes ${kind} ("${v}") as a literal — the `
+					+ 'provider would be pinned to this user. Re-capture or re-propose.',
+				)
+			}
+		}
+	}
+
+	for(const [i, m] of matches.entries()) {
+		scan(
+			m.value,
+			`responseMatch for "${targets[i]?.name ?? '?'}"`,
+			targets[i]?.value ?? null,
+		)
+	}
+
+	for(const r of redactions) {
+		if(r.regex) {
+			scan(r.regex, 'responseRedaction regex', null)
+		}
+	}
+}
+
 /** Derive a `paramValues` key (and capture-group name) from the provider
  *  name's last alphanumeric token. `"GitHub Username"` → `"username"`,
  *  `"x-followers"` → `"followers"`. */
@@ -43,7 +137,7 @@ function deriveParamName(providerName: string): string {
 
 export function draftProvider(
 	req: CapturedRequest,
-	target: string,
+	targets: DraftTarget[],
 	name: string,
 	hash?: OprfMode,
 ): DraftResult {
@@ -51,31 +145,70 @@ export function draftProvider(
 		throw new Error(`captured request ${req.requestId} has no response body`)
 	}
 
-	const paramName = deriveParamName(name)
-	const synthesis = synthesizeMatcher(
-		req.responseBody,
-		req.contentType,
-		target,
-		paramName,
-		hash,
-	)
-	if(!synthesis) {
-		throw new Error(
-			`target value "${target}" not found in` +
-			` response body for request ${req.requestId}`,
+	if(targets.length === 0) {
+		throw new Error('draftProvider needs at least one target')
+	}
+
+	// One request → N matches + N redactions. Synthesize each target against the
+	// SAME response body; a distinct param name per target keeps their capture
+	// groups + paramValues from colliding.
+	const matches: ResponseMatch[] = []
+	const redactions: ResponseRedaction[] = []
+	const extractions: Extraction[] = []
+	let paramValues: Record<string, string> | undefined
+	const seenParams = new Set<string>()
+	for(const t of targets) {
+		const paramName = deriveParamName(t.name)
+		if(seenParams.has(paramName)) {
+			throw new Error(
+				`two targets resolve to the same param "${paramName}"; `
+				+ 'give them distinct names',
+			)
+		}
+
+		seenParams.add(paramName)
+		const synthesis = synthesizeMatcher(
+			req.responseBody,
+			req.contentType,
+			t.value,
+			paramName,
+			t.hash ?? hash,
 		)
+		if(!synthesis) {
+			throw new Error(
+				`target value "${t.value}" not found in`
+				+ ` response body for request ${req.requestId}`,
+			)
+		}
+
+		matches.push(synthesis.match)
+		if(synthesis.redaction) {
+			redactions.push(synthesis.redaction)
+		}
+
+		if(synthesis.paramValues) {
+			paramValues = { ...paramValues, ...synthesis.paramValues }
+		}
+
+		extractions.push({
+			name: paramName,
+			value: t.value,
+			...(synthesis.redaction ? { redaction: synthesis.redaction } : {}),
+		})
 	}
 
+	// Portability gate: never ship a draft that bakes a per-user value literally.
+	assertPortableMatches(matches, redactions, targets)
+
 	const { secrets, publics } = splitSecrets(req.requestHeaders)
-	// Force the server to respond uncompressed. Captured browser requests
-	// usually carry `accept-encoding: gzip, deflate, br, zstd`; the attestor
-	// matches against the raw response bytes (no transparent decompression
-	// in the TLS replay path), so gzip/br/zstd payloads make every
-	// xPath/jsonPath/regex redaction miss against unparseable bytes.
-	// Replacing here also benefits `replay_request` — Node's `fetch` does
-	// auto-decompress, but pinning the encoding keeps replay and proof
-	// looking at the same body shape end-to-end.
-	const headersForProvider = withIdentityEncoding(publics)
+	// Sanitize the public headers before they enter the provider. Two problems
+	// a captured browser request carries: (1) `accept-encoding: gzip, …`, which
+	// makes the attestor match against compressed bytes (it does no transparent
+	// decompression in the TLS replay path) — pin it to identity; (2) conditional
+	// headers (`if-none-match`/`if-modified-since`), which make the origin answer
+	// 304 at proof time and fail the proof. Sanitizing here also keeps
+	// `replay_request` and the proof looking at the same full-200 body shape.
+	const headersForProvider = sanitizeRequestHeaders(publics)
 	const secretRefs: SecretParamRef[] = Object.keys(secrets).map((name) => ({
 		name,
 		source: name.toLowerCase() === 'cookie' ? 'cookie' : 'header',
@@ -87,19 +220,19 @@ export function draftProvider(
 		url: req.url,
 		method: req.method as ReclaimProvider['method'],
 		headers: headersForProvider,
-		responseMatches: [synthesis.match],
+		responseMatches: matches,
 	}
-	if(synthesis.redaction) {
-		provider.responseRedactions = [synthesis.redaction]
+	if(redactions.length) {
+		provider.responseRedactions = redactions
 	}
 
-	if(synthesis.paramValues) {
-		provider.paramValues = synthesis.paramValues
+	if(paramValues) {
+		provider.paramValues = paramValues
 	}
 
 	if(req.requestBody) {
 		provider.body = req.requestBody
 	}
 
-	return { provider, secretRefs }
+	return { provider, secretRefs, extractions }
 }

package/src/provider/index.ts

@@ -1,6 +1,11 @@
 // Provider primitives shared with the app's capture pipeline:
 // the attestor-derived recipe types, secret classification, and the GraphQL
 // op parser. Exposed as `@reclaimprotocol/agent/provider`.
+// Provider synthesis from a captured request (the `propose_provider` core):
+// drafts the canonical responseMatches/redactions + paramValues + secret refs.
+export { draftProvider } from './draft.ts'
+export type { DraftResult } from './draft.ts'
+export { detectConcerns } from './constraints.ts'
 export { extractGraphqlOp, looksLikeGraphqlEndpoint } from './graphql.ts'
 export type {
 	CapturedRequest,
@@ -13,6 +18,9 @@ export type {
 	SecretParamRef,
 } from './schema.ts'
 export { classifyHeader, splitSecrets } from './secrets.ts'
+// Recipe → version `requests[]` (a `RequestSelection`) — the same mapping the
+// MCP publish uses, so the cloud authoring publish stays byte-identical.
+export { providerToRequestSelection } from './to-version.ts'
 // Re-exported attestor-core provider primitives (the registry, the param hash,
 // and the param type) so the builder imports them via the agent rather than
 // depending on @reclaimprotocol/attestor-core directly.

package/src/provider/injection-doc.ts

@@ -11,6 +11,12 @@
  * standard APIs; the client already intercepts page requests) and treats
  * `window.Reclaim` as a rarely-needed edge-case bridge, pointing at the
  * authoritative `src/support/env.d.ts` rather than reproducing its surface.
+ * It also covers the HAWKEYE-style live-interception path (`injectionType` /
+ * `interceptorType`, `disableRequestReplay` / `isDocumentRequestReplayEnabled`)
+ * and `bodySniff`, reverse-engineered from the old-devtools Prisma schema,
+ * controller code, and dashboard field hints — neither publish tool exposes
+ * these as inputs (both only build the standard replay flow), so this is
+ * read-the-field-and-recognize-it documentation, not authoring guidance.
  */
 import { readFileSync } from 'node:fs'
 import { fileURLToPath } from 'node:url'