@reclaimprotocol/agent 0.1.36 → 0.1.38

package/src/provider/draft.ts

@@ -1,12 +1,15 @@
+import { badRequest } from '@hapi/boom'
+import assert from 'node:assert'
 import { isOprfHashable, OPRF_MAX_BYTES, synthesizeMatcher } from './matcher.ts'
 import { normalizeCandidates } from './normalize.ts'
-import type {
-	CapturedRequest,
-	OprfMode,
-	ReclaimProvider,
-	ResponseMatch,
-	ResponseRedaction,
-	SecretParamRef,
+import {
+	type CapturedRequest,
+	DYNAMIC_GEO,
+	type OprfMode,
+	type ReclaimProvider,
+	type ResponseMatch,
+	type ResponseRedaction,
+	type SecretParamRef,
 } from './schema.ts'
 import {
 	secretUrlCharBudget,
@@ -433,3 +436,208 @@ export function draftProvider(
 
 	return { provider, secretRefs, extractions }
 }
+
+/** Fields `update_draft` may replace on a drafted provider. */
+export interface DraftEdit {
+	responseMatches?: ResponseMatch[]
+	responseRedactions?: ResponseRedaction[]
+	paramValues?: Record<string, string>
+	/** ISO country the request goes out from, or `{{DYNAMIC_GEO}}` /
+	 *  `{{DYNAMIC_GEO_SECRET}}` for the verifying user's own country. */
+	geoLocation?: string
+}
+
+const GEO_TEMPLATES = new Set([DYNAMIC_GEO, '{{DYNAMIC_GEO_SECRET}}'])
+
+const MATCH_TYPES = new Set(['contains', 'regex'])
+const OPRF_MODES = new Set(['oprf', 'oprf-raw', 'oprf-mpc'])
+
+/**
+ * Apply an agent's edit to a drafted provider: validate the new matchers the
+ * way the attestor reads them (regexes compile with its `sgi` flags, every
+ * redaction has a selector, a hashed redaction has exactly one named group),
+ * then run the same portability gate as {@link draftProvider} so an edit can't
+ * bake a captured value in as a literal. Returns the updated provider, its
+ * extractions (re-paired with the redaction that names each param), and
+ * advisory warnings (for example, a `{{param}}` nothing resolves).
+ */
+export function editDraftProvider(
+	provider: ReclaimProvider,
+	extractions: Extraction[],
+	edit: DraftEdit,
+): {
+	provider: ReclaimProvider
+	extractions: Extraction[]
+	warnings: string[]
+} {
+	const matches = edit.responseMatches ?? provider.responseMatches
+	const redactions = edit.responseRedactions
+		?? provider.responseRedactions ?? []
+	const paramValues = edit.paramValues ?? provider.paramValues ?? {}
+	const geoLocation = GEO_TEMPLATES.has(edit.geoLocation ?? '')
+		? edit.geoLocation
+		: edit.geoLocation?.toUpperCase()
+	assert(
+		geoLocation === undefined || GEO_TEMPLATES.has(geoLocation)
+		|| /^[A-Z]{2}$/.test(geoLocation),
+		badRequest(
+			'geoLocation must be a 2-letter ISO country code, {{DYNAMIC_GEO}}, '
+			+ 'or {{DYNAMIC_GEO_SECRET}}',
+		),
+	)
+	assert(
+		matches.length > 0,
+		badRequest('responseMatches must have at least one entry'),
+	)
+	for(const [i, m] of matches.entries()) {
+		assert(
+			MATCH_TYPES.has(m.type) && typeof m.value === 'string' && m.value,
+			badRequest(
+				`responseMatches[${i}] needs type "contains" or "regex" and a `
+				+ 'non-empty value',
+			),
+		)
+		if(m.type === 'regex') {
+			assertRegex(m.value, `responseMatches[${i}].value`)
+		}
+	}
+
+	for(const [i, r] of redactions.entries()) {
+		assert(
+			r.xPath || r.jsonPath || r.regex,
+			badRequest(`responseRedactions[${i}] needs an xPath, jsonPath, or regex`),
+		)
+		if(r.regex) {
+			assertRegex(r.regex, `responseRedactions[${i}].regex`)
+		}
+
+		assert(
+			!r.hash || (OPRF_MODES.has(r.hash) && groupNames(r.regex).length === 1),
+			badRequest(
+				`responseRedactions[${i}] is hashed (${r.hash}), so it needs a hash `
+				+ 'of oprf, oprf-raw, or oprf-mpc and a regex with exactly one named '
+				+ 'capture group',
+			),
+		)
+	}
+
+	// An edit can't lower privacy: a draft that redacts keeps redacting, and
+	// every group it hashed stays hashed with the same mode.
+	const original = provider.responseRedactions ?? []
+	assert(
+		!original.length || redactions.length,
+		badRequest(
+			'responseRedactions can\'t be emptied: the attestor would see the '
+			+ 'whole response',
+		),
+	)
+	for(const r of original.filter((o) => o.hash)) {
+		for(const name of groupNames(r.regex)) {
+			const kept = redactions.find((e) => groupNames(e.regex).includes(name))
+			assert(
+				kept?.hash === r.hash,
+				badRequest(
+					`capture group ${name} is ${r.hash}-hashed in the draft; keep a `
+					+ 'redaction for it with the same hash',
+				),
+			)
+		}
+	}
+
+	const groups = matcherGroups(matches, redactions)
+	// A group the draft never extracted has a value nothing below checks, and
+	// no extraction would track it: new data points need a fresh draft.
+	const known = new Set([
+		...extractions.map((e) => e.name),
+		...matcherGroups(
+			provider.responseMatches,
+			provider.responseRedactions ?? [],
+		),
+	])
+	const unknown = [...groups].filter((name) => !known.has(name))
+	assert(
+		!unknown.length,
+		badRequest(
+			`capture group(s) ${unknown.join(', ')} are not drafted extractions; `
+			+ 'propose a new draft to add a data point',
+		),
+	)
+
+	// Always the captured values, so editing a paramValues entry can't hide
+	// one from the portability gate; edited values are checked as well.
+	const targets: DraftTarget[] = [
+		...extractions.map(({ name, value }) => ({ name, value })),
+		...Object.entries(paramValues)
+			.filter(([name, value]) => !extractions
+				.some((e) => e.name === name && e.value === value))
+			.map(([name, value]) => ({ name, value })),
+	]
+	assertPortableMatches(matches, redactions, targets)
+
+	// Same OPRF gate as draftProvider, against the URL as the attestor signs it.
+	const hashed = redactions.flatMap(({ regex, hash }) => hash
+		? groupNames(regex).flatMap((name) => targets
+			.filter((t) => t.name === name)
+			.map((t) => ({ ...t, hash })))
+		: [])
+	const url = provider.url
+		.replace(/\{\{([^{}]+)\}\}/g, (t, name: string) => paramValues[name] ?? t)
+	assertNoOprfUrlConflict(url, hashed, hashed.map((t) => t.hash))
+
+	const { responseRedactions: _r, paramValues: _p, ...rest } = provider
+	void _r
+	void _p
+	const next: ReclaimProvider = {
+		...rest,
+		responseMatches: matches,
+		...(redactions.length ? { responseRedactions: redactions } : {}),
+		...(Object.keys(paramValues).length ? { paramValues } : {}),
+		...(geoLocation ? { geoLocation } : {}),
+	}
+	const warnings = matches
+		.flatMap((m) => [...m.value.matchAll(/\{\{([^{}]+)\}\}/g)].map((t) => t[1]))
+		.filter((name) => !(name in paramValues) && !groups.has(name))
+		.map((name) => `{{${name}}} has no paramValues entry or named capture `
+			+ 'group, so the attestor would look for it literally.')
+	const nextExtractions = extractions.map((e) => {
+		const redaction = redactions
+			.find((r) => groupNames(r.regex).includes(e.name))
+		// The captured value, never an edited one, so a later edit's gate still
+		// sees it.
+		return {
+			name: e.name,
+			value: e.value,
+			...(redaction ? { redaction } : {}),
+		}
+	})
+	return { provider: next, extractions: nextExtractions, warnings }
+}
+
+/** Named capture groups across regex matches and redactions. */
+function matcherGroups(
+	matches: ResponseMatch[],
+	redactions: ResponseRedaction[],
+) {
+	return new Set([
+		...matches.flatMap((m) => m.type === 'regex' ? groupNames(m.value) : []),
+		...redactions.flatMap((r) => groupNames(r.regex)),
+	])
+}
+
+/** Named capture groups in `regex` (none when absent). */
+function groupNames(regex: string | undefined): string[] {
+	return [...(regex ?? '').matchAll(/\(\?<([A-Za-z_][A-Za-z0-9_]*)>/g)]
+		.map((m) => m[1])
+}
+
+/** Throw a 400 unless `regex` compiles the way the attestor compiles it. */
+function assertRegex(regex: string, where: string) {
+	let error: string | undefined
+	try {
+		new RegExp(regex, 'sgi')
+	} catch(err) {
+		error = err instanceof Error ? err.message : String(err)
+	}
+
+	assert(!error, badRequest(`${where} is not a valid regex: ${error}`))
+}

package/src/provider/matcher.ts

@@ -1,3 +1,5 @@
+import { badRequest } from '@hapi/boom'
+import assert from 'node:assert'
 import { elementAnchor } from './dom-xpath.ts'
 import { normalizeCandidates } from './normalize.ts'
 import type { OprfMode, ResponseMatch, ResponseRedaction } from './schema.ts'
@@ -81,6 +83,13 @@ export function synthesizeMatcher(
 		}
 	}
 
+	// Inline JSON / JS-object text outside an HTML element (a JS bundle, a
+	// non-JSON text body): anchor on the key name right before the value.
+	const inline = inlineKeyMatcher(body, target, paramName, valueClass, hash)
+	if(inline) {
+		return inline
+	}
+
 	// No JSON key and no stable xPath anchor: a content-anchored regex, matched
 	// by CONTENT so it reproduces regardless of DOM position at proof time (a
 	// positional `(//tag)[N]` resolves to the wrong node at proof — see 9fd7770).
@@ -430,9 +439,186 @@ function htmlMatcher(
 		}
 	}
 
-	return elementResult(
+	// Inline JS-object / JSON text inside the element (a SvelteKit/Next/Nuxt
+	// hydration `<script>` that isn't valid JSON): anchor on the key name
+	// nearest the value (`currentTierDescription:"{{v}}"`), scoped by the xPath.
+	const inline = inlineKeyMatcher(
+		revealSlice, target, paramName, valueClass, hash, anchor.xPath,
+	)
+	if(inline) {
+		return inline
+	}
+
+	const result = elementResult(
 		anchor.xPath, revealSlice, target, paramName, valueClass, hash,
 	)
+	const [left = '', right = ''] = result?.match.value
+		.split(`{{${paramName}}}`) ?? []
+	const hits = result?.redaction?.regex
+		? countMatches(revealSlice, result.redaction.regex)
+		: 0
+	assert(
+		!result || !isGenericWindow(left, right) || hits <= 1,
+		badRequest(
+			`target value "${target}" sits in a <${elementTag(revealSlice)}> `
+			+ 'with no key name or markup next to it. The only literal context, '
+			+ `"${result?.match.value}", matches ${hits} values in that element, `
+			+ 'so it would not pin this one. Pick a request where the value has '
+			+ 'a key (for example, a JSON API response) or sits in its own '
+			+ 'element.',
+		),
+	)
+	return result
+}
+
+/** How many times `regex` matches in `text` (0 for an invalid regex). */
+function countMatches(text: string, regex: string): number {
+	try {
+		return [...text.matchAll(new RegExp(regex, 'g'))].length
+	} catch{
+		return 0
+	}
+}
+
+/** Tag name of the element markup starting `markup` (`script`, `div`, …). */
+function elementTag(markup: string): string {
+	return /^<([A-Za-z][\w-]*)/.exec(markup)?.[1] ?? 'element'
+}
+
+/** A literal window with no letter or digit on either side (for example,
+ *  `:"{{v}}",`): it pins a value only when the element holds nothing else. */
+function isGenericWindow(left: string, right: string): boolean {
+	return !/[A-Za-z0-9]/.test(left + right)
+}
+
+// ---------------------------------------------------------------------------
+// Inline key: `key:"value"` / `"key":"value"` in JS-object or JSON text
+// ---------------------------------------------------------------------------
+
+/** How far left of a value to look for its key. */
+const MAX_KEY_LOOKBACK = 128
+
+/** The key directly before a value: JSON `"key":`, `'key':`, or a bare JS
+ *  identifier `key:`, then the value's opening quote (if any) at the end. */
+const KEY_BEFORE_VALUE = new RegExp(
+	'(?:"([A-Za-z_$][\\w$-]*)"|\'([A-Za-z_$][\\w$-]*)\''
+	+ '|(?<![\\w$"\'])([A-Za-z_$][\\w$]*))\\s*:\\s*(["\']?)$',
+)
+
+/**
+ * Matcher for a value written as `key:"value"` (JS object literal, as SvelteKit
+ * and similar frameworks inline into SSR pages) or `"key":"value"` inside text
+ * that isn't parseable JSON. The window is the key name plus the value's
+ * quotes, so the match pins this field and nothing user-specific:
+ * `currentTierDescription:"{{v}}"`.
+ *
+ * Rejects a key that appears more than once in `scope` with a different value:
+ * the regex would extract the first one, which may not be this field. When the
+ * value has a key but it's ambiguous, throws rather than falling back to a
+ * looser window. Returns undefined when no occurrence has a key before it.
+ */
+function inlineKeyMatcher(
+	scope: string,
+	target: string,
+	paramName: string,
+	valueClass: string,
+	hash?: OprfMode,
+	xPath?: string,
+): MatcherSynthesis | undefined {
+	const candidates = [target, ...normalizeCandidates(target)]
+		.filter((c) => c && !c.startsWith('"'))
+	let ambiguous: { key: string, count: number } | undefined
+	for(const cand of new Set(candidates)) {
+		for(
+			let idx = scope.indexOf(cand);
+			idx >= 0;
+			idx = scope.indexOf(cand, idx + 1)
+		) {
+			const kv = keyBefore(scope, idx, cand)
+			if(!kv) {
+				continue
+			}
+
+			const others = valuesForKey(scope, kv.key)
+				.filter((v) => v !== cand)
+			if(others.length) {
+				ambiguous ??= { key: kv.key, count: others.length + 1 }
+				continue
+			}
+
+			// A single-quoted free-form value must stop at its own quote.
+			const cls = kv.close === '\'' && valueClass === '[^"<>\\n]+'
+				? '[^\'<>\\n]+'
+				: valueClass
+			return keyValueResult(
+				kv.prefix, kv.close, cand, paramName, cls,
+				xPath ? { xPath } : {}, hash,
+			)
+		}
+	}
+
+	assert(
+		!ambiguous,
+		badRequest(
+			`target value "${target}" sits under key "${ambiguous?.key}", but that `
+			+ `key occurs ${ambiguous?.count} times with different values, so a `
+			+ 'match on it would not pin this value. Pick a request where the key '
+			+ 'is unique (for example, a JSON API that returns only this object).',
+		),
+	)
+	return undefined
+}
+
+/** The key + opening-quote prefix directly before the value at `idx`, when the
+ *  value is a complete quoted string (or a bare token) under that key. */
+function keyBefore(
+	scope: string,
+	idx: number,
+	value: string,
+): { key: string, prefix: string, close: string } | undefined {
+	const head = scope.slice(Math.max(0, idx - MAX_KEY_LOOKBACK), idx)
+	const m = KEY_BEFORE_VALUE.exec(head)
+	const key = m?.[1] ?? m?.[2] ?? m?.[3]
+	if(!m || !key || key.length < 2) {
+		return undefined
+	}
+
+	// A bare key before a bare value (`score: 5`) is prose as often as code;
+	// require a quoted key or a quoted value.
+	const quote = m[4] ?? ''
+	const after = scope[idx + value.length] ?? ''
+	const complete = quote ? after === quote : /^$|[\s,;}\]]/.test(after)
+	if(!complete || (!quote && m[3])) {
+		return undefined
+	}
+
+	return { key, prefix: m[0], close: quote }
+}
+
+/** Every value written under `key` in `scope` (JSON or JS-object syntax),
+ *  raw — quoted strings without their quotes, bare tokens as-is. */
+function valuesForKey(scope: string, key: string): string[] {
+	const re = new RegExp(
+		`(?<![\\w$])(["']?)${escapeRegex(key)}\\1\\s*:\\s*`, 'g',
+	)
+	const values: string[] = []
+	for(const m of scope.matchAll(re)) {
+		const start = m.index + m[0].length
+		const quote = scope[start]
+		if(quote === '"' || quote === '\'') {
+			let j = start + 1
+			while(j < scope.length && scope[j] !== quote) {
+				j += scope[j] === '\\' ? 2 : 1
+			}
+
+			values.push(scope.slice(start + 1, j))
+			continue
+		}
+
+		values.push(/^[^\s,;}\]]*/.exec(scope.slice(start))?.[0] ?? '')
+	}
+
+	return values
 }
 
 /**
@@ -743,14 +929,33 @@ function buildContentRegex(
 			const from = Math.max(0, start)
 			const before = escapeRegexFlexWs(body.slice(from, idx))
 			const re = `${before}(?<${paramName}>${valueClass})${right}`
+			let found = false
 			try {
-				if(new RegExp(re).exec(body)?.groups?.[paramName] === cand) {
-					return re
-				}
+				found = new RegExp(re).exec(body)?.groups?.[paramName] === cand
 			} catch{
 				break
 			}
 
+			if(found) {
+				const problem = contextProblem(
+					body.slice(from, idx), right ? rawRight : '',
+				)
+				assert(
+					!problem,
+					badRequest(
+						`target value "${target}" has no key name or element next to `
+						+ `it; the shortest window that pins it, "${body.slice(from, idx)}`
+						+ `{{${paramName}}}${right ? rawRight : ''}", ${problem}. That `
+						+ 'would pin the provider to this user or match the wrong value. '
+						+ 'This happens with positional arrays such as SvelteKit '
+						+ '`__data.json`. Use a request where the value sits under its '
+						+ 'own key (the HTML page or another JSON API), or stop and '
+						+ 'report.',
+					),
+				)
+				return re
+			}
+
 			if(from === 0) {
 				break
 			}
@@ -760,6 +965,34 @@ function buildContentRegex(
 	return undefined
 }
 
+/** A standalone number token (`33` in `:33},`) — a points balance, count, or
+ *  id that differs per user; not a digit inside a word or class (`mb-2`). */
+const VOLATILE_NUMBER = /(?<![\w.-])\d+(?:\.\d+)?(?![\w.])/
+
+/** The closing quote of a string followed by a separator — the end of a
+ *  neighbouring VALUE (keys are followed by `:`), usually user-specific. */
+const NEIGHBOUR_STRING = /"\s*[,\]}]/
+
+/** Why a content window around a value isn't portable, or undefined when it
+ *  is: only punctuation (pins nothing), a number next to the value, or a
+ *  neighbouring string value. */
+function contextProblem(left: string, right: string): string | undefined {
+	if(VOLATILE_NUMBER.test(left) || VOLATILE_NUMBER.test(right)) {
+		return 'contains a number next to the value, which is likely '
+			+ 'user-specific (for example, a points balance)'
+	}
+
+	if(NEIGHBOUR_STRING.test(left)) {
+		return 'contains a neighbouring value, which is likely user-specific'
+	}
+
+	if(!/[A-Za-z]/.test(left + right)) {
+		return 'is only punctuation and matches almost any value'
+	}
+
+	return undefined
+}
+
 /**
  * The literal `contains` template: surrounding structural context with the
  * value replaced by `{{paramName}}`. Whitespace-adjacent context is dropped (a