@reclaimprotocol/agent 0.1.21 → 0.1.23

package/src/old/tools/publish.ts

@@ -99,12 +99,15 @@ export function createVersionFromCaptureTool(
 		stepsToFollow?: string
 		useIncognitoWebview?: boolean
 		extensionConfig?: unknown
-		disableRequestReplay?: boolean
+		injectionType?: 'NONE' | 'MSWJS' | 'XHOOK' | 'CDP' | 'HAWKEYE'
 	}>(
 		{
 			name: 'create_provider_version_from_capture',
 			description:
 				'Publish drafted ReclaimProvider(s) to the devtools backend. ' +
+				'Requires its OWN old-devtools login via reclaim_authenticate — ' +
+				'separate from authenticate_builder/Builder auth, which does not ' +
+				'authorize this call. Check status with get_devtools_mode. \n\n' +
 				'Pass the `provider` object from propose_provider VERBATIM and the ' +
 				'`initialUrl` the user landed on (the login/landing page, NOT the ' +
 				'API url). \n\n' +
@@ -122,8 +125,9 @@ export function createVersionFromCaptureTool(
 				"PUBLIC provider you don't own, omit `providerId` to register your " +
 				'own copy (you cannot edit someone else\'s provider here). \n\n' +
 				'For data gated behind login/navigation, pass `customInjection` — a ' +
-				'page-injection script (the devtools name for the builder\'s ' +
-				'jsUserScripts) that drives the page to where the request fires. \n\n' +
+				'USER SCRIPT (same concept as the builder\'s `jsUserScripts` — ' +
+				'devtools just calls the field `customInjection`) that drives the ' +
+				'page to where the request fires. \n\n' +
 				'BY DESIGN, this backend only ever CREATES A NEW VERSION (POST) — ' +
 				'there is no partial-patch/edit endpoint, and this tool does NOT ' +
 				'read or carry anything forward from the previous version. Every ' +
@@ -139,15 +143,20 @@ export function createVersionFromCaptureTool(
 				'values plus your change) back into this call. ' +
 				'`allowedInjectedRequestData`/`useIncognitoWebview`/' +
 				'`extensionConfig` only take effect on add-version — the register ' +
-				'endpoint (no `providerId`) ignores them. `disableRequestReplay` ' +
-				'is also add-version-only, but when it is true the tool follows ' +
-				'a register up with an add-version call automatically so it takes ' +
-				'effect on new providers too. Per-request fields you may set on ' +
+				'endpoint (no `providerId`) ignores them, so on a fresh register ' +
+				'this tool automatically follows up with an add-version call that ' +
+				'applies them (also where document replay gets force-disabled — ' +
+				'see below). Per-request fields you may set on ' +
 				'`provider`/`providers`: `credentials` (omit/same-origin/include, ' +
 				'default include), `urlType` (REGEX/CONSTANT/TEMPLATE, default ' +
 				'derived from {{placeholders}}), and `isOptional` on each ' +
 				'responseMatch (default false). \n\n' +
 				'Captured requests must be GET/POST. OPRF redactions are supported. ' +
+					'Document replay — the HAWKEYE-level flag controlling whether the ' +
+					'page\'s own document/navigation request gets genuinely re-sent ' +
+					'through the interceptor at verification time — is ALWAYS ' +
+					'published disabled (`disableRequestReplay: true`); there is no ' +
+					'way to enable it here. ' +
 					'On success, if a browser is still attached, the response\'s ' +
 					'`_notes` reminds you to confirm with the developer before ' +
 					'calling `dispose_browser` — never close it unprompted, they ' +
@@ -458,14 +467,25 @@ export function createVersionFromCaptureTool(
 							'Free-form JSON passed through to the client. Add-version ' +
 							'ONLY.',
 					},
-					disableRequestReplay: {
-						type: 'boolean',
+					injectionType: {
+						type: 'string',
+						enum: ['NONE', 'MSWJS', 'XHOOK', 'CDP', 'HAWKEYE'],
 						description:
-							'Skip replaying the main document request (HAWKEYE ' +
-							'provider-level flag). Honored on add-version; for a ' +
-							'provider (no `providerId`) the register endpoint ignores ' +
-							'it, so when true the tool automatically follows the ' +
-							'register up with an add-version call that applies it.',
+							'Interception mechanism for this version. Omit to leave it ' +
+							'unset — the devtools backend\'s own column default ' +
+							'(`HAWKEYE`) applies, which is inert and harmless for the ' +
+							'standard capture→replay flow with no interceptor script ' +
+							'installed. Only set this to `HAWKEYE`/`MSWJS`/`XHOOK` ' +
+							'when you are ALSO supplying a `customInjection` script ' +
+							'that calls ' +
+							'`window.reclaimInterceptor.addResponseMiddleware(...)` for ' +
+							'genuine live interception, or set `NONE` explicitly if you ' +
+							'want to be certain no interception is configured — see ' +
+							'injection-api.md\'s "Recognizing this on a provider you ' +
+							'didn\'t author" section for what these values actually do. ' +
+							'Builder calls the equivalent concept `interceptorType` (no ' +
+							'`NONE` there — omitting the whole `interceptorOptions` ' +
+							'object is its no-interception signal instead).',
 					},
 				},
 				required: ['initialUrl'],
@@ -505,7 +525,13 @@ export function createVersionFromCaptureTool(
 				stepsToFollow: args.stepsToFollow,
 				useIncognitoWebview: args.useIncognitoWebview,
 				extensionConfig: args.extensionConfig,
-				disableRequestReplay: args.disableRequestReplay,
+				// Author-settable, no forced default — the backend's own HAWKEYE
+				// column default applies when omitted, same as before this field
+				// existed at all.
+				injectionType: args.injectionType,
+				// Always disabled — see the tool description above; there is no
+				// author-facing way to enable document replay.
+				disableRequestReplay: true,
 				allowedInjectedRequestData: args.allowedInjectedRequestData
 					? injectedRequestDataFrom(
 						args.allowedInjectedRequestData,
@@ -558,12 +584,12 @@ export function createVersionFromCaptureTool(
 					)
 				}
 
-				// disableRequestReplay is honored ONLY on add-version, so a
-				// register can't set it — when true we immediately follow the
-				// register with an add-version call (which also lands any other
-				// register-ignored fields).
-				const followUp = args.disableRequestReplay === true
-
+				// disableRequestReplay is honored ONLY on add-version, and it's now
+				// always forced true — so every register is immediately followed by
+				// an add-version call that actually applies it (which also lands
+				// an explicit injectionType, when passed — unverified whether
+				// register persists it — and any other register-ignored fields the
+				// caller passed).
 				const REGISTER_IGNORED_KEYS = [
 					'allowedInjectedRequestData',
 					'useIncognitoWebview',
@@ -576,46 +602,66 @@ export function createVersionFromCaptureTool(
 						`${ignoredOnRegister.join(', ')} ` +
 						`${ignoredOnRegister.length > 1 ? 'are' : 'is'} ` +
 						'ignored on register (no providerId) — the devtools backend only ' +
-						'reads them when adding a version. ' +
-						(followUp
-							? 'They were applied by the follow-up version the tool ' +
-							'created to set disableRequestReplay.'
-							: 'Publish once to create the provider, then call this ' +
-							'again WITH the returned providerId to set them.'),
+						'reads them when adding a version. They were applied by the ' +
+						'automatic follow-up version this tool always creates to ' +
+						'force-disable document replay.',
 					)
 				}
 
 				const body = providerToRegisterBody(providers, opts)
 				const registered = await client.registerProvider(body)
 
-				if(followUp) {
-					const newProviderId = extractProviderId(registered)
-					if(newProviderId) {
+				const newProviderId = extractProviderId(registered)
+				if(newProviderId) {
+					// The provider already exists as of the register call above —
+					// a failure past this point must NOT reject the tool and
+					// discard `registered`/`newProviderId`, or a retry without
+					// `providerId` would register a DUPLICATE provider. Catch and
+					// warn instead, telling the caller to resume WITH this
+					// providerId.
+					try {
 						const versionsResp = await client.listVersions(newProviderId)
 						const followVersion = bumpPatchString(
 							extractVersions(versionsResp),
 						)
 						const configBody = providerToConfigBody(
 							providers,
-							{ ...opts, disableRequestReplay: true },
+							opts,
 							followVersion,
 							DEFAULT_VERSION_INFO,
 						)
 						await client.updateProviderConfig(newProviderId, configBody)
 						notes.push(
-							'disableRequestReplay is ignored by the register endpoint — ' +
-							`applied it via a follow-up version (${followVersion}) on ` +
-							'the new provider, which also lands any register-ignored ' +
-							'fields you passed.',
+							`Created an automatic follow-up version (${followVersion}) to ` +
+							'force-disable document replay (disableRequestReplay)' +
+							(opts.injectionType !== undefined
+								? ` and set injectionType (${opts.injectionType})`
+								: '') +
+							' — the register endpoint ignores/may ignore those fields, so ' +
+							'every fresh register gets one. It also landed any other ' +
+							'register-ignored fields you passed.',
 						)
-					} else {
+					} catch(err) {
+						const detail = isBoom(err) ? err.message : String(err)
 						notes.push(
-							'Could not apply disableRequestReplay: the register ' +
-							'response did not include the new providerId. Call this ' +
-							'tool again WITH `providerId` and `disableRequestReplay: ' +
-							'true` to set it.',
+							`Provider ${newProviderId} WAS created, but the automatic ` +
+							`follow-up version (to force-disable document replay${
+								opts.injectionType !== undefined
+									? ' and set injectionType'
+									: ''
+							}) failed: ${detail}. Do NOT retry without providerId — that ` +
+							'would register a duplicate. Instead, re-run ' +
+							'create_provider_version_from_capture WITH ' +
+							`providerId: "${newProviderId}" to add the follow-up version.`,
 						)
 					}
+				} else {
+					notes.push(
+						'Could not force-disable document replay: the ' +
+						'register response did not include the new providerId. Call ' +
+						'this tool again WITH `providerId` to create a version that ' +
+						'applies them.',
+					)
 				}
 
 				return withNotes(registered)

package/src/proof/page-replay.ts

@@ -0,0 +1,131 @@
+/**
+ * Replay a drafted provider's AUTHENTICATED request FROM INSIDE the live
+ * attached browser tab, over CDP `Runtime.evaluate` — so cookies/session are
+ * always the real browser's, for every browser source the MCP can attach to
+ * (local Chrome, remote Popcorn, or a caller-supplied custom CDP endpoint all
+ * funnel through one `AuthoringBackend.sendCdp`). Ports the pattern already
+ * proven in the cloud/Popcorn authoring backend
+ * (`packages/app/src/author/backend.ts` `pageFetch`/`authenticatedFetch`),
+ * adapted to the MCP's single-page `sendCdp(method, params)` (no separate
+ * `pageSession` argument). The ANONYMOUS auth-bound check does NOT go through
+ * the page — see `replayProviderInPage`'s doc comment below for why.
+ */
+import type { ReclaimProvider } from '../provider/schema.ts'
+import {
+	buildReplayRequest,
+	evaluateReplay,
+	replayProvider,
+	type ReplayResult
+} from './replay.ts'
+
+/** Everything this module needs from an `AuthoringBackend` — just the raw CDP
+ *  seam, so callers can pass a minimal object instead of a full backend. */
+export interface PageReplayTarget {
+	sendCdp(method: string, params?: Record<string, unknown>): Promise<unknown>
+}
+
+interface PageFetchResult {
+	status?: number
+	body?: string
+	error?: string
+}
+
+/** Run `fetch(...)` for `provider`'s (already-templated) request inside the
+ *  attached tab, via `Runtime.evaluate`. `credentials` selects the visitor:
+ *  `'omit'` = anonymous (the auth-bound check), `'include'` = the logged-in
+ *  user (the reproducibility gate). Same-origin only — the tab is already on
+ *  the target site; a cross-origin/CORS failure comes back as `error`. */
+async function pageFetch(
+	backend: PageReplayTarget,
+	provider: ReclaimProvider,
+	credentials: 'omit' | 'include',
+	extraSecrets: Record<string, string>,
+): Promise<PageFetchResult> {
+	const req = buildReplayRequest(provider, extraSecrets)
+	const expression = `(async () => {
+		try {
+			const r = await fetch(${JSON.stringify(req.url)}, {
+				method: ${JSON.stringify(req.method)},
+				headers: ${JSON.stringify(req.headers)},
+				credentials: ${JSON.stringify(credentials)},${req.body !== undefined
+					? `\n\t\t\t\tbody: ${JSON.stringify(req.body)},`
+					: ''}
+			})
+			return { status: r.status, body: await r.text() }
+		} catch(e) { return { error: String((e && e.message) || e) } }
+	})()`
+	try {
+		const res = (await backend.sendCdp('Runtime.evaluate', {
+			expression,
+			awaitPromise: true,
+			returnByValue: true,
+		})) as {
+			result?: { value?: PageFetchResult }
+			exceptionDetails?: { exception?: { description?: string }, text?: string }
+		}
+		if(res.exceptionDetails) {
+			return {
+				error: res.exceptionDetails.exception?.description
+					?? res.exceptionDetails.text
+					?? 'page fetch evaluation error',
+			}
+		}
+
+		return res.result?.value ?? { error: 'page fetch returned no value' }
+	} catch(err) {
+		return { error: err instanceof Error ? err.message : String(err) }
+	}
+}
+
+/** Authenticated re-issue for the reproducibility gate. Cookies ride along
+ *  via `credentials: 'include'` (a `Cookie` header can't be set from
+ *  `fetch()` — it's a forbidden header name), so only NON-cookie secrets
+ *  (Authorization, X-* tokens) are injected explicitly. */
+function authenticatedPageFetch(
+	backend: PageReplayTarget,
+	provider: ReclaimProvider,
+	secrets: Record<string, string>,
+): Promise<PageFetchResult> {
+	const extra: Record<string, string> = {}
+	for(const [k, v] of Object.entries(secrets)) {
+		if(k.toLowerCase() !== 'cookie') {
+			extra[k] = v
+		}
+	}
+
+	return pageFetch(backend, provider, 'include', extra)
+}
+
+/**
+ * Replay `provider` and judge the response. Without `withoutSecrets`, replays
+ * as the logged-in session — run FROM INSIDE the live tab
+ * (`authenticatedPageFetch`) so cookies are the real browser's, which a
+ * detached call could never reproduce without manually re-deriving them.
+ * WITH `withoutSecrets` (the auth-bound check — "does this still work with
+ * no credentials, i.e. is it actually public"), replay instead through the
+ * detached Node-side `replayProvider`: the whole point of that check is to
+ * prove NO secret rides along, and the live page's own global `fetch` can't
+ * give that guarantee — a site (or a script installed via test_user_script)
+ * can wrap `fetch` and re-attach a token from localStorage/a service worker
+ * even under `credentials:'omit'`. A fresh, page-uninvolved fetch has no
+ * such surface.
+ */
+export async function replayProviderInPage(
+	backend: PageReplayTarget,
+	provider: ReclaimProvider,
+	secrets: Record<string, string>,
+	opts?: { withoutSecrets?: boolean },
+): Promise<ReplayResult> {
+	if(opts?.withoutSecrets) {
+		return replayProvider(provider, {})
+	}
+
+	const result = await authenticatedPageFetch(backend, provider, secrets)
+	if(result.error !== undefined || result.status === undefined) {
+		throw new Error(
+			`in-browser replay failed: ${result.error ?? 'no response'}`,
+		)
+	}
+
+	return evaluateReplay(result.status, result.body ?? '', provider)
+}

package/src/proof/replay.ts

@@ -9,7 +9,7 @@ export interface ReplayResult {
 
 /** Expand `{{name}}` placeholders from `paramValues`. Matches the attestor
  *  SDK's behavior so replay and proof both hit the same final URL. */
-function expandTemplate(
+export function expandTemplate(
 	s: string,
 	params: Record<string, string> | undefined,
 ): string {
@@ -23,10 +23,22 @@ function expandTemplate(
 	})
 }
 
-export async function replayProvider(
+/** Build the request the wire replay actually sends: `{{name}}` placeholders
+ *  in the URL/headers/body expanded from `provider.paramValues`, secrets
+ *  layered onto the public headers. Shared by every replay transport (Node
+ *  `fetch` here, or a `fetch` run inside the live browser tab) so they all
+ *  send the identical request. */
+export interface ReplayRequest {
+	url: string
+	method: string
+	headers: Record<string, string>
+	body?: string
+}
+
+export function buildReplayRequest(
 	provider: ReclaimProvider,
 	secrets: Record<string, string>,
-): Promise<ReplayResult> {
+): ReplayRequest {
 	const params = provider.paramValues
 	const url = expandTemplate(provider.url, params)
 	const expandedHeaders: Record<string, string> = {}
@@ -35,23 +47,35 @@ export async function replayProvider(
 	}
 
 	const headers = { ...expandedHeaders, ...secrets }
-	const fetchInit: RequestInit = {
+	const body = provider.body !== undefined
+		? expandTemplate(provider.body, params)
+		: undefined
+	return {
+		url,
 		method: provider.method,
 		headers,
-		redirect: 'manual',
-	}
-	if(provider.body !== undefined) {
-		fetchInit.body = expandTemplate(provider.body, params)
+		...(body !== undefined ? { body } : {}),
 	}
+}
 
-	const res = await fetch(url, fetchInit)
-	const body = await res.text()
+/**
+ * Judge a replayed response against `provider`'s matchers — the part of
+ * replay that's identical regardless of HOW the request was sent (detached
+ * Node `fetch` or a live in-browser `fetch`). Separated from the actual
+ * network call so both transports share one evaluation.
+ */
+export function evaluateReplay(
+	status: number,
+	body: string,
+	provider: ReclaimProvider,
+): ReplayResult {
+	const params = provider.paramValues
 	const hints: string[] = []
-	if(res.status === 401 || res.status === 403) {
+	if(status === 401 || status === 403) {
 		hints.push('auth/permission failed')
 	}
 
-	if(res.status >= 500) {
+	if(status >= 500) {
 		hints.push('upstream server error')
 	}
 
@@ -66,18 +90,14 @@ export async function replayProvider(
 			value: expandTemplate(m.value, params),
 		}
 		if(!matchesBody(body, expanded)) {
-			if(res.status >= 200 && res.status < 300) {
+			if(status >= 200 && status < 300) {
 				hints.push(
 					`matcher ${expanded.type} did not match — ` +
 					'value may have changed or matcher is too tight',
 				)
 			}
 
-			const result: ReplayResult = {
-				status: res.status,
-				body,
-				matchesOriginal: false,
-			}
+			const result: ReplayResult = { status, body, matchesOriginal: false }
 			if(hints.length > 0) {
 				result.hints = hints
 			}
@@ -86,7 +106,28 @@ export async function replayProvider(
 		}
 	}
 
-	return { status: res.status, body, matchesOriginal: res.status < 400 }
+	return { status, body, matchesOriginal: status < 400 }
+}
+
+/** Detached Node-side replay: sends the request directly from this process,
+ *  NOT from any attached browser tab — its cookies/secrets are exactly the
+ *  `secrets` passed in, never a real browser session. Used by tests and by
+ *  any caller with no live browser to replay through (the MCP's authoring
+ *  backend instead runs the SAME request inside the attached tab — see
+ *  `proof/page-replay.ts` — so replay reflects real session cookies). */
+export async function replayProvider(
+	provider: ReclaimProvider,
+	secrets: Record<string, string>,
+): Promise<ReplayResult> {
+	const req = buildReplayRequest(provider, secrets)
+	const res = await fetch(req.url, {
+		method: req.method,
+		headers: req.headers,
+		redirect: 'manual',
+		...(req.body !== undefined ? { body: req.body } : {}),
+	})
+	const body = await res.text()
+	return evaluateReplay(res.status, body, provider)
 }
 
 function matchesBody(body: string, m: ResponseMatch): boolean {

package/src/provider/draft.ts

@@ -275,12 +275,45 @@ function deriveParamName(providerName: string): string {
 	return last.toLowerCase()
 }
 
+/** A `paramName` doubles as a regex NAMED CAPTURE GROUP (`(?<name>...)`) and a
+ *  `paramValues` object key — both require a plain identifier. Nothing
+ *  downstream validates this: an invalid name builds a syntactically broken
+ *  regex that silently fails to extract at verification time instead of
+ *  erroring here, at draft time, where it's actually fixable. */
+const VALID_PARAM_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/
+
+/** Resolve a target's `paramName`: shortened (the default — last alphanumeric
+ *  token, lowercased) or, with `shorten: false`, the name verbatim — the
+ *  caller's choice, validated so it's still safe as a capture-group
+ *  identifier. */
+function resolveParamName(targetName: string, shorten: boolean): string {
+	if(!shorten) {
+		if(!VALID_PARAM_NAME.test(targetName)) {
+			throw new Error(
+				`target name "${targetName}" isn't safe as a param name with `
+				+ 'shortenNames:false — it becomes a regex capture-group name AND '
+				+ 'a paramValues key, so it must start with a letter/underscore and '
+				+ 'contain only letters, digits, and underscores (we recommend '
+				+ 'lower_snake_case, e.g. "employer_name"). Rename it, or drop '
+				+ '`shortenNames: false` to auto-derive a safe one.',
+			)
+		}
+
+		return targetName
+	}
+
+	return deriveParamName(targetName)
+}
+
 export function draftProvider(
 	req: CapturedRequest,
 	targets: DraftTarget[],
 	name: string,
 	hash?: OprfMode,
+	opts?: { shortenNames?: boolean },
 ): DraftResult {
+	const shortenNames = opts?.shortenNames ?? true
+
 	if(!req.responseBody) {
 		throw new Error(`captured request ${req.requestId} has no response body`)
 	}
@@ -297,17 +330,24 @@ export function draftProvider(
 	const extractions: Extraction[] = []
 	const effectiveHashes: (OprfMode | undefined)[] = []
 	let paramValues: Record<string, string> | undefined
-	const seenParams = new Set<string>()
+	const seenParams = new Map<string, string>()
 	for(const t of targets) {
-		const paramName = deriveParamName(t.name)
-		if(seenParams.has(paramName)) {
+		const paramName = resolveParamName(t.name, shortenNames)
+		const priorName = seenParams.get(paramName)
+		if(priorName !== undefined) {
 			throw new Error(
-				`two targets resolve to the same param "${paramName}"; `
-				+ 'give them distinct names',
+				`targets "${priorName}" and "${t.name}" both resolve to the same `
+				+ `param name "${paramName}"`
+				+ (shortenNames
+					? ' (names are shortened to their last alphanumeric token) — '
+						+ 'give them more distinct names, e.g. "firstName"/"surname" '
+						+ 'instead of "first_name"/"last_name"'
+					: ' — give them distinct names, or drop `shortenNames: false` '
+						+ 'to auto-derive distinct ones'),
 			)
 		}
 
-		seenParams.add(paramName)
+		seenParams.set(paramName, t.name)
 		// Precedence: explicit per-target hash → explicit request-wide hash →
 		// the anti-leak floor (force-OPRF values whose SHAPE is unambiguous PII).
 		// The model makes the contextual calls (which named fields are sensitive)