imprint-mcp 0.6.6 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/README.md +278 -16
  3. package/package.json +2 -2
  4. package/prompts/audit-agent.md +25 -4
  5. package/prompts/auth-compile-agent.md +28 -12
  6. package/prompts/build-planning.md +3 -3
  7. package/prompts/compile-agent.md +895 -180
  8. package/prompts/live-verifier-agent.md +10 -3
  9. package/prompts/master-teach-api-researcher.md +813 -0
  10. package/prompts/master-teach-baseline-mvp-review.md +217 -0
  11. package/prompts/master-teach-completion-review.md +188 -0
  12. package/prompts/master-teach-decision.md +885 -0
  13. package/prompts/master-teach-focused-planner.md +764 -0
  14. package/prompts/master-teach-parameter-advisor.md +58 -0
  15. package/prompts/master-teach-recording-evidence.md +19 -0
  16. package/prompts/master-teach-refine-plan.md +7 -0
  17. package/prompts/master-teach-tool-advisor.md +130 -0
  18. package/prompts/playbook-compilation.md +45 -54
  19. package/prompts/prereq-builder.md +28 -6
  20. package/prompts/prereq-planner.md +2 -2
  21. package/prompts/tool-candidate-detection.md +70 -39
  22. package/prompts/tool-planning.md +3 -3
  23. package/src/cli.ts +190 -150
  24. package/src/imprint/agent.ts +47 -33
  25. package/src/imprint/api-research-agent.ts +933 -0
  26. package/src/imprint/audit.ts +49 -2
  27. package/src/imprint/auth-compile-agent.ts +125 -55
  28. package/src/imprint/auth-compile-tools.ts +20 -48
  29. package/src/imprint/auth-verifier.ts +8 -19
  30. package/src/imprint/backend-ladder.ts +499 -367
  31. package/src/imprint/body-structure.ts +656 -0
  32. package/src/imprint/build-plan.ts +33 -323
  33. package/src/imprint/cdp-browser-fetch.ts +734 -29
  34. package/src/imprint/cdp-jar-cache.ts +7 -12
  35. package/src/imprint/claude-cli-compile.ts +276 -320
  36. package/src/imprint/codex-cli-compile.ts +476 -350
  37. package/src/imprint/codex-sdk-turn.ts +102 -0
  38. package/src/imprint/compile-agent-types.ts +49 -10
  39. package/src/imprint/compile-agent.ts +189 -274
  40. package/src/imprint/compile-done-sentinel.ts +211 -0
  41. package/src/imprint/compile-provider-control.ts +401 -0
  42. package/src/imprint/compile-provider-recovery.ts +142 -0
  43. package/src/imprint/compile-strategy.ts +5 -0
  44. package/src/imprint/compile-tools.ts +1695 -3097
  45. package/src/imprint/compile-verification.ts +9 -1
  46. package/src/imprint/compile.ts +107 -29
  47. package/src/imprint/compiler-log.ts +12 -0
  48. package/src/imprint/compiler-process.ts +440 -0
  49. package/src/imprint/concurrency.ts +43 -58
  50. package/src/imprint/credential-extract.ts +1 -88
  51. package/src/imprint/emit.ts +18 -2
  52. package/src/imprint/evidence-inspection.ts +55 -0
  53. package/src/imprint/freeform-redact.ts +34 -234
  54. package/src/imprint/import-module-fresh.ts +30 -0
  55. package/src/imprint/live-finesse-runner.ts +299 -0
  56. package/src/imprint/live-verifier.ts +299 -358
  57. package/src/imprint/llm.ts +466 -422
  58. package/src/imprint/log.ts +1 -9
  59. package/src/imprint/master-teach-agent-contracts.ts +875 -0
  60. package/src/imprint/master-teach-agents.ts +2850 -0
  61. package/src/imprint/master-teach-checks.ts +341 -0
  62. package/src/imprint/master-teach-controller.ts +7159 -0
  63. package/src/imprint/master-teach-plan.ts +1410 -0
  64. package/src/imprint/master-teach-prompt-projections.ts +347 -0
  65. package/src/imprint/master-teach-store.ts +961 -0
  66. package/src/imprint/mcp-cdp-sessions.ts +60 -0
  67. package/src/imprint/mcp-compile-server.ts +519 -668
  68. package/src/imprint/mcp-live-verifier-server.ts +246 -190
  69. package/src/imprint/mcp-server.ts +46 -57
  70. package/src/imprint/native-agent-pass.ts +260 -0
  71. package/src/imprint/native-agent-usage.ts +141 -0
  72. package/src/imprint/native-teach-agents.ts +495 -0
  73. package/src/imprint/param-grounding.ts +488 -347
  74. package/src/imprint/playbook-runner.ts +254 -63
  75. package/src/imprint/prereq-builder.ts +55 -543
  76. package/src/imprint/private-file.ts +18 -0
  77. package/src/imprint/probe-backends.ts +129 -33
  78. package/src/imprint/provider-retry.ts +501 -0
  79. package/src/imprint/provider-terminal.ts +214 -0
  80. package/src/imprint/recording-request.ts +290 -1
  81. package/src/imprint/recording-verification.ts +444 -0
  82. package/src/imprint/redact.ts +33 -749
  83. package/src/imprint/refine-worker.ts +326 -0
  84. package/src/imprint/refine.ts +241 -0
  85. package/src/imprint/replay-capture.ts +41 -12
  86. package/src/imprint/replay-evidence.ts +1007 -0
  87. package/src/imprint/research-response-evidence.ts +45 -0
  88. package/src/imprint/runtime.ts +514 -106
  89. package/src/imprint/sensitive-keys.ts +0 -6
  90. package/src/imprint/session-merge.ts +103 -7
  91. package/src/imprint/stealth-fetch.ts +25 -8
  92. package/src/imprint/stealth-token-cache.ts +16 -16
  93. package/src/imprint/teach-plan.ts +0 -242
  94. package/src/imprint/teach-research-memory.ts +229 -0
  95. package/src/imprint/teach-state.ts +47 -625
  96. package/src/imprint/teach.ts +12 -3699
  97. package/src/imprint/tool-candidates.ts +58 -210
  98. package/src/imprint/tool-loader.ts +3 -0
  99. package/src/imprint/tool-plan.ts +1 -134
  100. package/src/imprint/tracing.ts +24 -7
  101. package/src/imprint/types.ts +68 -4
  102. package/src/imprint/endpoint-key.ts +0 -23
@@ -1,376 +1,517 @@
1
- /**
2
- * Event-correlated differential param grounding.
3
- *
4
- * The candidate detector reliably identifies WHICH inputs the user controlled
5
- * (`likelyParams`) and WHICH events toggled them (`eventSeqs`) — but the compile
6
- * agent historically grounded a param by eyeballing a single request, and when
7
- * the value wasn't obviously present it gave up and shipped the param
8
- * `verified:false`, inert. Yet the encoding is almost always right there: the
9
- * request a filter-toggle event triggers differs from the prior equivalent
10
- * request at exactly the position that param controls.
11
- *
12
- * This module makes that differential deterministic: for each UI event, find the
13
- * request it triggered, diff it against the most recent comparable request (same
14
- * endpoint), and report the changed paths. The compile agent (and the precomputed
15
- * hint surfaced to it) then maps each diff to a `likelyParam` — the semantic step
16
- * the model is good at — instead of guessing at an encoding. Decoding uses a
17
- * pluggable envelope registry (batchexecute is one registered handler), raw JSON
18
- * bodies, or plain form fields.
19
- */
20
-
21
- import { getEndpointKey } from './endpoint-key.ts';
1
+ import {
2
+ type BodyComparison,
3
+ type BodyScalarType,
4
+ type BodyStructure,
5
+ type StructureDifference,
6
+ compareBodyStructures,
7
+ decodeBodyStructure,
8
+ describeBodyPaths,
9
+ parseBodyFormat,
10
+ prepareBodyScalarEquality,
11
+ } from './body-structure.ts';
22
12
  import type { CapturedRequest, Session } from './types.ts';
23
13
 
24
- interface GroundingChange {
25
- /** JSON path into the decoded request body, e.g. "[1][4][3]". */
26
- path: string;
27
- before: string;
28
- after: string;
14
+ type GroundingState = 'compared' | 'not_checked' | 'invalid' | 'not_found';
15
+ type GroundingTruncation =
16
+ | 'request_index'
17
+ | 'event_index'
18
+ | 'before_alternatives'
19
+ | 'after_alternatives';
20
+ interface GroundingLimits {
21
+ requestIndex: { total: number; indexed: number; cap: number; truncated: boolean };
22
+ eventIndex: { total: number; indexed: number; cap: number; truncated: boolean };
23
+ alternativesPerSide: number;
24
+ }
25
+ interface RequestAlternative {
26
+ seq: number;
27
+ method: string;
28
+ resourceType: string;
29
+ hasBody: boolean;
30
+ responseStatus?: number;
29
31
  }
30
-
31
32
  interface EventGrounding {
32
33
  eventSeq: number;
33
- /** Human label from the event detail (button text / aria-label / id). */
34
- label: string;
35
- /** The request the event triggered (first comparable request after it). */
36
- triggeredSeq?: number;
37
- /** The prior request of the same endpoint that the diff is taken against. */
38
- priorSeq?: number;
39
- endpoint?: string;
40
- changes: GroundingChange[];
34
+ state: GroundingState;
35
+ reasonCode?: string;
36
+ outcome?: 'changed' | 'no_change' | 'inconclusive';
37
+ changes: StructureDifference[];
38
+ comparison?: Omit<BodyComparison, 'differences'>;
39
+ selectedPair?: { beforeSeq: number; afterSeq: number };
40
+ alternatives: { before: RequestAlternative[]; after: RequestAlternative[] };
41
+ association: { mode: 'unselected' | 'exact_pair' };
42
+ bodyEncoding?: {
43
+ beforeFormat: BodyStructure['format'];
44
+ afterFormat: BodyStructure['format'];
45
+ beforeJsonStringBoundaries: Array<{ depth: number; path?: string }>;
46
+ afterJsonStringBoundaries: Array<{ depth: number; path?: string }>;
47
+ beforePathsTruncated?: true;
48
+ afterPathsTruncated?: true;
49
+ };
50
+ bodyChecks?: Array<{
51
+ seq: number;
52
+ status: 'decoded' | 'skipped' | 'not_checked';
53
+ reasonCode?: string;
54
+ }>;
55
+ work: {
56
+ beforeRequestsAvailable: number;
57
+ afterRequestsAvailable: number;
58
+ alternativesReturned: number;
59
+ bodyDecodesAttempted: number;
60
+ bodyDecodesSucceeded: number;
61
+ bodyDecodesSkipped: number;
62
+ bodyDecodesTruncated: number;
63
+ };
64
+ limits: GroundingLimits;
65
+ truncated?: GroundingTruncation[];
41
66
  }
42
-
43
- /** First request after `eventSeq`, within a window, that has a decodable body. */
44
- const TRIGGER_WINDOW = 12;
45
-
46
- /** Envelope handlers for unwrapping nested request bodies. Each handler provides
47
- * a detection predicate and an unwrapping function. Handlers are tried in order;
48
- * first handler whose `detect` returns true and whose `unwrap` returns non-
49
- * undefined wins. */
50
- interface EnvelopeHandler {
51
- detect(formParams: URLSearchParams): boolean;
52
- unwrap(formParams: URLSearchParams): unknown;
67
+ const MAX_GROUNDING_REQUESTS = 10_000;
68
+ const MAX_GROUNDING_INDEXED_EVENTS = 10_000;
69
+ const MAX_EVENT_ALTERNATIVES = 4;
70
+ const ENCODING_CONTEXT_LIMIT = 4;
71
+ const groundingEncoder = new TextEncoder();
72
+ function bodyOf(request: CapturedRequest): string | undefined {
73
+ return (
74
+ (request as unknown as { body?: string }).body ??
75
+ (request as unknown as { requestBody?: string }).requestBody
76
+ );
53
77
  }
54
-
55
- const ENVELOPE_HANDLERS: EnvelopeHandler[] = [
56
- {
57
- // Google batchexecute envelope: f.req=[[["rpcid","<inner-json-string>",…]]]
58
- detect: (formParams) => formParams.get('f.req') != null,
59
- unwrap: (formParams) => {
60
- const freq = formParams.get('f.req');
61
- if (!freq) return undefined;
62
- try {
63
- const env = JSON.parse(freq);
64
- // Unwrap [[["rpcid","<inner json string>", …]]] to the inner payload.
65
- const innerStr = env?.[0]?.[0]?.[1];
66
- if (typeof innerStr === 'string') {
67
- try {
68
- return JSON.parse(innerStr);
69
- } catch {
70
- // Fall through to return the outer envelope if inner parse fails.
71
- // This preserves the f.req-wraps-a-flat-object sub-shape that
72
- // google-hotels/search_hotels relies on.
73
- return env;
74
- }
75
- }
76
- return env;
77
- } catch {
78
- return undefined;
79
- }
80
- },
81
- },
82
- ];
83
-
84
- /** Decode a request body into a comparable structure. Handles, in order:
85
- * a raw JSON body; registered envelope handlers (batchexecute is one example);
86
- * otherwise a flat form-field map; else the raw string. Never throws. */
87
- export function decodeBodyForDiff(body: string | undefined): unknown {
88
- if (!body) return undefined;
89
- const trimmed = body.trim();
90
- if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
91
- try {
92
- return JSON.parse(trimmed);
93
- } catch {
94
- /* not JSON */
95
- }
96
- }
97
- // form-encoded?
98
- if (/(^|&)[\w.]+=/.test(trimmed)) {
99
- const params = new URLSearchParams(trimmed);
100
- // Try registered envelope handlers.
101
- for (const handler of ENVELOPE_HANDLERS) {
102
- if (handler.detect(params)) {
103
- const unwrapped = handler.unwrap(params);
104
- if (unwrapped !== undefined) return unwrapped;
105
- }
106
- }
107
- // Fall through to flat form field map.
108
- const out: Record<string, string> = {};
109
- for (const [k, v] of params) out[k] = v;
110
- return out;
111
- }
112
- return trimmed;
78
+ function bodyStructureNotCheckedReason(structure: BodyStructure): string | undefined {
79
+ if (structure.truncated) return structure.truncated;
80
+ if ((structure.nestedJsonExpansion?.candidateNotChecked ?? 0) > 0)
81
+ return 'nested_json_candidate_not_checked';
82
+ return undefined;
113
83
  }
114
-
115
- /** Deep structural diff → changed leaf paths (a→b). Identical subtrees are
116
- * skipped via a cheap stringify equality check. */
117
- export function structuralDiff(
118
- a: unknown,
119
- b: unknown,
120
- path = '',
121
- out: GroundingChange[] = [],
122
- ): GroundingChange[] {
123
- if (JSON.stringify(a) === JSON.stringify(b)) return out;
124
- if (Array.isArray(a) && Array.isArray(b)) {
125
- const n = Math.max(a.length, b.length);
126
- for (let i = 0; i < n; i++) structuralDiff(a[i], b[i], `${path}[${i}]`, out);
127
- return out;
128
- }
129
- if (a && b && typeof a === 'object' && typeof b === 'object') {
130
- const keys = new Set([...Object.keys(a as object), ...Object.keys(b as object)]);
131
- for (const k of keys) {
132
- structuralDiff(
133
- (a as Record<string, unknown>)[k],
134
- (b as Record<string, unknown>)[k],
135
- path ? `${path}.${k}` : k,
136
- out,
137
- );
138
- }
139
- return out;
84
+ function lowerBound(requests: CapturedRequest[], seq: number): number {
85
+ let low = 0;
86
+ let high = requests.length;
87
+ while (low < high) {
88
+ const middle = low + Math.floor((high - low) / 2);
89
+ if ((requests[middle]?.seq ?? Number.POSITIVE_INFINITY) < seq) low = middle + 1;
90
+ else high = middle;
140
91
  }
141
- const cap = (v: unknown) => {
142
- const s = v === undefined ? 'undefined' : JSON.stringify(v);
143
- return s.length > 48 ? `${s.slice(0, 48)}…` : s;
92
+ return low;
93
+ }
94
+ interface GroundingIndex {
95
+ requests: CapturedRequest[];
96
+ events: Set<number>;
97
+ limits: GroundingLimits;
98
+ }
99
+ function buildGroundingIndex(session: Session): GroundingIndex {
100
+ const requests = session.requests
101
+ .slice(0, MAX_GROUNDING_REQUESTS)
102
+ .sort((left, right) => left.seq - right.seq);
103
+ const indexedEvents = session.events.slice(0, MAX_GROUNDING_INDEXED_EVENTS);
104
+ const events = new Set(indexedEvents.map((event) => event.seq));
105
+ return {
106
+ requests,
107
+ events,
108
+ limits: {
109
+ requestIndex: {
110
+ total: session.requests.length,
111
+ indexed: requests.length,
112
+ cap: MAX_GROUNDING_REQUESTS,
113
+ truncated: session.requests.length > requests.length,
114
+ },
115
+ eventIndex: {
116
+ total: session.events.length,
117
+ indexed: indexedEvents.length,
118
+ cap: MAX_GROUNDING_INDEXED_EVENTS,
119
+ truncated: session.events.length > indexedEvents.length,
120
+ },
121
+ alternativesPerSide: MAX_EVENT_ALTERNATIVES,
122
+ },
144
123
  };
145
- out.push({ path: path || '(root)', before: cap(a), after: cap(b) });
146
- return out;
147
124
  }
148
-
149
- function bodyOf(req: CapturedRequest): string | undefined {
150
- // CapturedRequest stores the request body on `.body`; tolerate alt shapes.
151
- return (
152
- (req as unknown as { body?: string }).body ??
153
- (req as unknown as { requestBody?: string }).requestBody ??
154
- undefined
155
- );
125
+ function boundedEncodingPaths(paths: string[], includePaths: boolean) {
126
+ const described = describeBodyPaths(paths, includePaths);
127
+ return {
128
+ facts: described.facts.slice(0, ENCODING_CONTEXT_LIMIT),
129
+ ...(described.truncated || described.facts.length > ENCODING_CONTEXT_LIMIT
130
+ ? { truncated: true as const }
131
+ : {}),
132
+ };
156
133
  }
157
-
158
- function eventLabel(detail: string): string {
159
- let d: Record<string, unknown> = {};
160
- try {
161
- d = JSON.parse(detail);
162
- } catch {
163
- return detail.slice(0, 48);
164
- }
165
- const txt = (d.text ?? d.ariaLabel ?? d.name ?? d.id ?? '') as string;
166
- return String(txt).replace(/\s+/g, ' ').trim().slice(0, 48);
134
+ function alternative(request: CapturedRequest): RequestAlternative {
135
+ return {
136
+ seq: request.seq,
137
+ method: String(request.method ?? 'GET'),
138
+ resourceType: request.resourceType,
139
+ hasBody: bodyOf(request) !== undefined,
140
+ ...(request.response ? { responseStatus: request.response.status } : {}),
141
+ };
167
142
  }
168
-
169
- /** Telemetry/beacon endpoints that fire constantly and are never the tool's
170
- * load-bearing request — excluded when we can't scope to the candidate's own
171
- * endpoints. */
172
- const TELEMETRY = /\/(log|gen_204|jserror|ping|beacon|csi|_\/bscframe|metrics|stats)\b/i;
173
-
174
- /** A decoded body worth diffing: a structured array/object, not a raw (often
175
- * gzipped/opaque) string. */
176
- function isStructured(v: unknown): boolean {
177
- return v != null && typeof v === 'object';
143
+ interface GroundEventOptions {
144
+ includePaths?: boolean;
145
+ compare?: {
146
+ beforeSeq: number;
147
+ afterSeq: number;
148
+ beforeFormat?: unknown;
149
+ afterFormat?: unknown;
150
+ };
178
151
  }
179
-
180
- /** Ground a single event: find the request it triggered and diff against the
181
- * most recent prior request of the same endpoint.
182
- *
183
- * `relevantEndpoints` (the candidate's own request endpoints, via endpointKey)
184
- * scopes the search to the tool's load-bearing requests — without it a burst of
185
- * telemetry POSTs between the click and the real request would be mistaken for
186
- * the trigger. */
187
- export function groundEvent(
188
- session: Session,
152
+ function groundEventWithIndex(
153
+ index: GroundingIndex,
189
154
  eventSeq: number,
190
- relevantEndpoints?: Set<string>,
155
+ options: GroundEventOptions = {},
191
156
  ): EventGrounding {
192
- const reqs = [...session.requests].sort((a, b) => a.seq - b.seq);
193
- const ev = session.events.find((e) => e.seq === eventSeq);
194
- const label = ev ? eventLabel(ev.detail) : '';
195
-
196
- const triggered = reqs.find((r) => {
197
- if (r.seq <= eventSeq || r.seq > eventSeq + windowEnd(reqs, eventSeq)) return false;
198
- const decoded = decodeBodyForDiff(bodyOf(r));
199
- if (decoded === undefined) return false;
200
- if (relevantEndpoints && relevantEndpoints.size > 0)
201
- return relevantEndpoints.has(getEndpointKey(r));
202
- // Fallback: structured body + not an obvious telemetry endpoint.
203
- return isStructured(decoded) && !TELEMETRY.test(r.url ?? '');
204
- });
205
- if (!triggered) return { eventSeq, label, changes: [] };
206
-
207
- const key = getEndpointKey(triggered);
208
- const prior = [...reqs]
157
+ const split = Number.isInteger(eventSeq) ? lowerBound(index.requests, eventSeq + 1) : 0;
158
+ const beforeAvailable = split;
159
+ const afterAvailable = index.requests.length - split;
160
+ const before = index.requests
161
+ .slice(Math.max(0, split - MAX_EVENT_ALTERNATIVES), split)
209
162
  .reverse()
210
- .find(
211
- (r) =>
212
- r.seq < triggered.seq &&
213
- getEndpointKey(r) === key &&
214
- decodeBodyForDiff(bodyOf(r)) !== undefined,
215
- );
216
-
217
- const changes = prior
218
- ? structuralDiff(decodeBodyForDiff(bodyOf(prior)), decodeBodyForDiff(bodyOf(triggered)))
219
- : [];
220
- return {
163
+ .map(alternative);
164
+ const after = index.requests.slice(split, split + MAX_EVENT_ALTERNATIVES).map(alternative);
165
+ const truncated: GroundingTruncation[] = [];
166
+ if (index.limits.requestIndex.truncated) truncated.push('request_index');
167
+ if (index.limits.eventIndex.truncated) truncated.push('event_index');
168
+ if (beforeAvailable > before.length) truncated.push('before_alternatives');
169
+ if (afterAvailable > after.length) truncated.push('after_alternatives');
170
+ const base = {
221
171
  eventSeq,
222
- label,
223
- triggeredSeq: triggered.seq,
224
- priorSeq: prior?.seq,
225
- endpoint: key,
226
- changes,
172
+ state: 'not_checked' as GroundingState,
173
+ changes: [] as StructureDifference[],
174
+ alternatives: { before, after },
175
+ association: { mode: options.compare ? ('exact_pair' as const) : ('unselected' as const) },
176
+ work: {
177
+ beforeRequestsAvailable: beforeAvailable,
178
+ afterRequestsAvailable: afterAvailable,
179
+ alternativesReturned: before.length + after.length,
180
+ bodyDecodesAttempted: 0,
181
+ bodyDecodesSucceeded: 0,
182
+ bodyDecodesSkipped: 0,
183
+ bodyDecodesTruncated: 0,
184
+ },
185
+ limits: index.limits,
186
+ ...(truncated.length ? { truncated } : {}),
187
+ };
188
+ if (!Number.isInteger(eventSeq))
189
+ return { ...base, state: 'invalid', reasonCode: 'invalid_event_seq' };
190
+ if (!index.events.has(eventSeq))
191
+ return index.limits.eventIndex.truncated
192
+ ? { ...base, reasonCode: 'event_not_checked_due_to_index_cap' }
193
+ : { ...base, state: 'not_found', reasonCode: 'event_not_found' };
194
+ if (!options.compare) return { ...base, reasonCode: 'agent_pair_required' };
195
+ const { beforeSeq, afterSeq } = options.compare;
196
+ if (!Number.isInteger(beforeSeq) || !Number.isInteger(afterSeq))
197
+ return { ...base, state: 'invalid', reasonCode: 'invalid_request_pair' };
198
+ const beforeRequest = index.requests.find((request) => request.seq === beforeSeq);
199
+ const afterRequest = index.requests.find((request) => request.seq === afterSeq);
200
+ if (!beforeRequest || !afterRequest)
201
+ return {
202
+ ...base,
203
+ state: index.limits.requestIndex.truncated ? 'not_checked' : 'not_found',
204
+ reasonCode: index.limits.requestIndex.truncated
205
+ ? 'selected_request_not_checked_due_to_index_cap'
206
+ : 'selected_request_not_found',
207
+ selectedPair: { beforeSeq, afterSeq },
208
+ };
209
+ const beforeFormat = parseBodyFormat(options.compare.beforeFormat);
210
+ if (!beforeFormat) return { ...base, state: 'invalid', reasonCode: 'invalid_before_format' };
211
+ const afterFormat = parseBodyFormat(options.compare.afterFormat);
212
+ if (!afterFormat) return { ...base, state: 'invalid', reasonCode: 'invalid_after_format' };
213
+ const beforeDecoded = decodeBodyStructure(bodyOf(beforeRequest), beforeFormat);
214
+ const afterDecoded = decodeBodyStructure(bodyOf(afterRequest), afterFormat);
215
+ const beforeTruncation = beforeDecoded.ok
216
+ ? bodyStructureNotCheckedReason(beforeDecoded.structure)
217
+ : undefined;
218
+ const afterTruncation = afterDecoded.ok
219
+ ? bodyStructureNotCheckedReason(afterDecoded.structure)
220
+ : undefined;
221
+ base.work.bodyDecodesAttempted = 2;
222
+ base.work.bodyDecodesSucceeded =
223
+ Number(beforeDecoded.ok && !beforeTruncation) + Number(afterDecoded.ok && !afterTruncation);
224
+ base.work.bodyDecodesSkipped = Number(!beforeDecoded.ok) + Number(!afterDecoded.ok);
225
+ base.work.bodyDecodesTruncated =
226
+ Number(Boolean(beforeTruncation)) + Number(Boolean(afterTruncation));
227
+ const bodyChecks = [
228
+ {
229
+ seq: beforeSeq,
230
+ status: !beforeDecoded.ok
231
+ ? ('skipped' as const)
232
+ : beforeTruncation
233
+ ? ('not_checked' as const)
234
+ : ('decoded' as const),
235
+ ...(!beforeDecoded.ok
236
+ ? { reasonCode: beforeDecoded.code }
237
+ : beforeTruncation
238
+ ? { reasonCode: beforeTruncation }
239
+ : {}),
240
+ },
241
+ {
242
+ seq: afterSeq,
243
+ status: !afterDecoded.ok
244
+ ? ('skipped' as const)
245
+ : afterTruncation
246
+ ? ('not_checked' as const)
247
+ : ('decoded' as const),
248
+ ...(!afterDecoded.ok
249
+ ? { reasonCode: afterDecoded.code }
250
+ : afterTruncation
251
+ ? { reasonCode: afterTruncation }
252
+ : {}),
253
+ },
254
+ ];
255
+ if (!beforeDecoded.ok || !afterDecoded.ok)
256
+ return {
257
+ ...base,
258
+ reasonCode: 'selected_body_not_decoded',
259
+ selectedPair: { beforeSeq, afterSeq },
260
+ bodyChecks,
261
+ };
262
+ if (beforeTruncation || afterTruncation)
263
+ return {
264
+ ...base,
265
+ reasonCode: 'selected_body_decode_truncated',
266
+ selectedPair: { beforeSeq, afterSeq },
267
+ bodyChecks,
268
+ };
269
+ const comparison = compareBodyStructures(beforeDecoded.structure, afterDecoded.structure, {
270
+ includePaths: options.includePaths === true,
271
+ });
272
+ const beforeEncoding = boundedEncodingPaths(
273
+ beforeDecoded.structure.jsonEncodedStringPaths,
274
+ options.includePaths === true,
275
+ );
276
+ const afterEncoding = boundedEncodingPaths(
277
+ afterDecoded.structure.jsonEncodedStringPaths,
278
+ options.includePaths === true,
279
+ );
280
+ return {
281
+ ...base,
282
+ state: 'compared',
283
+ outcome:
284
+ comparison.differences.length > 0
285
+ ? 'changed'
286
+ : comparison.truncated
287
+ ? 'inconclusive'
288
+ : 'no_change',
289
+ selectedPair: { beforeSeq, afterSeq },
290
+ changes: comparison.differences,
291
+ comparison: {
292
+ visitedNodes: comparison.visitedNodes,
293
+ wireEvidence: comparison.wireEvidence,
294
+ ...(comparison.truncated ? { truncated: comparison.truncated } : {}),
295
+ },
296
+ bodyEncoding: {
297
+ beforeFormat: beforeDecoded.structure.format,
298
+ afterFormat: afterDecoded.structure.format,
299
+ beforeJsonStringBoundaries: beforeEncoding.facts,
300
+ afterJsonStringBoundaries: afterEncoding.facts,
301
+ ...(beforeEncoding.truncated ? { beforePathsTruncated: true } : {}),
302
+ ...(afterEncoding.truncated ? { afterPathsTruncated: true } : {}),
303
+ },
304
+ bodyChecks,
227
305
  };
228
306
  }
229
-
230
- /** Window end: don't scan unboundedly — cap at TRIGGER_WINDOW requests past the
231
- * event (by seq distance to the Nth following request). */
232
- function windowEnd(reqs: CapturedRequest[], eventSeq: number): number {
233
- const after = reqs.filter((r) => r.seq > eventSeq).slice(0, TRIGGER_WINDOW);
234
- const last = after.at(-1);
235
- return last ? last.seq - eventSeq : TRIGGER_WINDOW;
236
- }
237
-
238
- /** Precompute grounding diffs for a candidate's filter-toggle events, dropping
239
- * events that triggered nothing or changed nothing.
240
- *
241
- * Pass `relevantEndpoints` = endpointKey() of the candidate's own request seqs
242
- * so the diff is taken against the tool's load-bearing request, not telemetry. */
243
- export function groundingForEvents(
307
+ export function groundEvent(
244
308
  session: Session,
245
- eventSeqs: number[],
246
- relevantEndpoints?: Set<string>,
247
- ): EventGrounding[] {
248
- const all = eventSeqs
249
- .map((seq) => groundEvent(session, seq, relevantEndpoints))
250
- .filter((g) => g.changes.length > 0);
251
-
252
- // Drop session-churn paths — positions that change across MOST events are
253
- // per-call session state (rotating tokens, pagination flags, a display-mode
254
- // value), not the param the event toggled. A param's encoding shows up only
255
- // in the diff(s) of the event(s) that control it, so frequency cleanly
256
- // separates signal from churn.
257
- const pathFreq = new Map<string, number>();
258
- for (const g of all) {
259
- for (const p of new Set(g.changes.map((c) => c.path)))
260
- pathFreq.set(p, (pathFreq.get(p) ?? 0) + 1);
261
- }
262
- const churnAt = Math.max(3, Math.ceil(all.length / 2));
263
- for (const g of all) g.changes = g.changes.filter((c) => (pathFreq.get(c.path) ?? 0) < churnAt);
264
- return all.filter((g) => g.changes.length > 0);
265
- }
266
-
267
- /** Derive the relevant-endpoint set from a candidate's request seqs. */
268
- export function endpointsForSeqs(session: Session, seqs: number[]): Set<string> {
269
- const set = new Set<string>();
270
- for (const seq of seqs) {
271
- const r = session.requests.find((x) => x.seq === seq);
272
- if (r) set.add(getEndpointKey(r));
273
- }
274
- return set;
275
- }
276
-
277
- // ─── Input-value provenance ──────────────────────────────────────────────────
278
- //
279
- // The grounding above covers params the user *toggled* (filters/sort). It does
280
- // not cover a primary param whose value is an opaque id the request can't carry
281
- // as plain text — e.g. an entity/object handle, an account id, a place/geo id, a
282
- // category token. The compile agent historically shipped these as the raw param
283
- // text, which the backend silently ignores and falls back to a default (an
284
- // unfiltered/global result set, or a server-chosen default scope). The id was
285
- // never the user's text; it was *minted by an earlier response* and chained into
286
- // the request. That cross-request data-flow is the signal this detects — keyed
287
- // on structure, not any vendor's id format.
288
-
289
- interface InputProvenance {
290
- /** JSON path into the decoded request body where the minted value sits. */
291
- path: string;
292
- /** Example resolved value (truncated). Varies per call — the PATH is the signal. */
293
- valueSample: string;
294
- /** The candidate request that consumes the value. */
295
- requestSeq: number;
296
- /** Earliest earlier request whose RESPONSE first carried this value. */
297
- sourceSeq: number;
298
- sourceEndpoint: string;
299
- /** True when the source is the tool's own endpoint (resolve-then-refine: an
300
- * initial text request whose response yields the id, re-sent as a refined
301
- * request carrying that id). */
302
- selfChain: boolean;
303
- }
304
-
305
- /** An opaque, machine-minted identifier — not human-typed text. Vendor-agnostic:
306
- * keyed on structure (no whitespace, long enough, mixes character classes or is
307
- * a delimited handle), not on any specific id format. Excludes free text
308
- * (multi-word phrases, single dictionary words), ISO dates, and bare counts so
309
- * they never trip it, while still catching namespaced handles ("ns/abc123"),
310
- * hex ids, UUIDs, and base64-ish session handles. */
311
- function isIdLike(v: string): boolean {
312
- if (/\s/.test(v)) return false; // free text has spaces
313
- if (v.length < 6) return false; // too short to be an opaque handle
314
- if (/^\d{4}-\d{2}-\d{2}([T ]|$)/.test(v)) return false; // ISO date / datetime
315
- const hasLetter = /[A-Za-z]/.test(v);
316
- const hasDigit = /\d/.test(v);
317
- const hasIdPunct = /[/:_.+=~-]/.test(v); // namespaced / delimited handle
318
- // Opaque if it mixes letters+digits (a token), or is a delimited handle that
319
- // still carries an alphanumeric payload. A bare word or a pure number is not.
320
- return (hasLetter && hasDigit) || (hasIdPunct && (hasLetter || hasDigit));
309
+ eventSeq: number,
310
+ options: GroundEventOptions = {},
311
+ ): EventGrounding {
312
+ return groundEventWithIndex(buildGroundingIndex(session), eventSeq, options);
321
313
  }
322
-
323
- function responseBodyOf(req: CapturedRequest): string | undefined {
324
- const b = (req as unknown as { response?: { body?: string } }).response?.body;
325
- return typeof b === 'string' ? b : undefined;
314
+ type Scalar = string | number | boolean | null;
315
+ type ScalarType = BodyScalarType;
316
+ const MAX_EQUALITY_HISTORY_REQUESTS = 4_096;
317
+ const MAX_EQUALITY_RESPONSES = 12;
318
+ const MAX_EQUALITY_RESPONSE_BYTES = 64 * 1024;
319
+ const MAX_EQUALITY_TOTAL_BYTES = 256 * 1024;
320
+ const MAX_EQUALITY_NODES = 512;
321
+ const MAX_EQUALITY_LEAVES = 128;
322
+ const MAX_EQUALITY_MATCHES = 8;
323
+ function scalarType(value: unknown): ScalarType | undefined {
324
+ if (value === null) return 'null';
325
+ const type = typeof value;
326
+ if (type === 'number' && !Number.isFinite(value)) return undefined;
327
+ return type === 'string' || type === 'number' || type === 'boolean' ? type : undefined;
326
328
  }
327
-
328
- function leafStrings(
329
- v: unknown,
330
- path = '',
331
- out: { path: string; val: string }[] = [],
332
- ): { path: string; val: string }[] {
333
- if (Array.isArray(v)) {
334
- v.forEach((x, i) => leafStrings(x, `${path}[${i}]`, out));
335
- } else if (v && typeof v === 'object') {
336
- for (const k of Object.keys(v as object))
337
- leafStrings((v as Record<string, unknown>)[k], path ? `${path}.${k}` : k, out);
338
- } else if (typeof v === 'string' && v.length >= 4) {
339
- out.push({ path, val: v });
329
+ function collectResponseScalars(value: unknown) {
330
+ const leaves: Array<{ path: string; depth: number; type: ScalarType; value: Scalar }> = [];
331
+ const stack: Array<{ value: unknown; path: string; depth: number }> = [
332
+ { value, path: '', depth: 0 },
333
+ ];
334
+ let nodes = 0;
335
+ let nodesTruncated = false;
336
+ let leavesTruncated = false;
337
+ while (stack.length) {
338
+ if (nodes >= MAX_EQUALITY_NODES) {
339
+ nodesTruncated = true;
340
+ break;
341
+ }
342
+ const current = stack.pop();
343
+ if (!current) break;
344
+ nodes++;
345
+ const type = scalarType(current.value);
346
+ if (type) {
347
+ if (leaves.length >= MAX_EQUALITY_LEAVES) {
348
+ leavesTruncated = true;
349
+ break;
350
+ }
351
+ leaves.push({ ...current, type, value: current.value as Scalar });
352
+ continue;
353
+ }
354
+ const entries = Array.isArray(current.value)
355
+ ? current.value.map((item, index) => [String(index), item] as const)
356
+ : current.value && typeof current.value === 'object'
357
+ ? Object.entries(current.value)
358
+ : [];
359
+ const room = Math.max(0, MAX_EQUALITY_NODES - nodes - stack.length);
360
+ if (entries.length > room) nodesTruncated = true;
361
+ for (let index = Math.min(entries.length, room) - 1; index >= 0; index--) {
362
+ const entry = entries[index];
363
+ if (!entry) continue;
364
+ stack.push({
365
+ value: entry[1],
366
+ path: `${current.path}/${entry[0].replaceAll('~', '~0').replaceAll('/', '~1')}`,
367
+ depth: current.depth + 1,
368
+ });
369
+ }
340
370
  }
341
- return out;
371
+ return {
372
+ leaves,
373
+ nodes,
374
+ nodesTruncated,
375
+ leavesTruncated,
376
+ };
342
377
  }
343
-
344
- /** For each candidate request, find body positions holding an id-like value that
345
- * first appears in an EARLIER response — i.e. a value the request did not get
346
- * from the user's text but chained in from upstream. Deduped by endpoint+path
347
- * (the value varies per call; the position is the durable signal). */
348
- export function inputProvenance(session: Session, candidateSeqs: number[]): InputProvenance[] {
349
- const reqs = [...session.requests].sort((a, b) => a.seq - b.seq);
350
- const seen = new Set<string>();
351
- const out: InputProvenance[] = [];
352
- for (const seq of [...candidateSeqs].sort((a, b) => a - b)) {
353
- const r = reqs.find((x) => x.seq === seq);
354
- if (!r) continue;
355
- const decoded = decodeBodyForDiff(bodyOf(r));
356
- if (decoded == null || typeof decoded !== 'object') continue;
357
- const ep = getEndpointKey(r);
358
- for (const { path, val } of leafStrings(decoded)) {
359
- if (!isIdLike(val)) continue;
360
- const key = `${ep}|${path}`;
361
- if (seen.has(key)) continue;
362
- const src = reqs.find((x) => x.seq < seq && (responseBodyOf(x)?.includes(val) ?? false));
363
- if (!src) continue; // not minted upstream → it IS the param's own text / a constant
364
- seen.add(key);
365
- out.push({
366
- path,
367
- valueSample: val.length > 40 ? `${val.slice(0, 40)}…` : val,
368
- requestSeq: seq,
369
- sourceSeq: src.seq,
370
- sourceEndpoint: getEndpointKey(src),
371
- selfChain: getEndpointKey(src) === ep,
378
+ export function findEarlierResponseEqualities(
379
+ session: Session,
380
+ requestSeq: number,
381
+ requestPointer: string,
382
+ selectedStructure: BodyStructure,
383
+ options: { includePaths?: boolean; responseFormat?: unknown } = {},
384
+ ) {
385
+ const responseFormat = parseBodyFormat(options.responseFormat);
386
+ const facts: Array<{
387
+ responseSeq: number;
388
+ responsePath?: string;
389
+ responseDepth: number;
390
+ matchKind: 'exact_scalar_equality_in_supplied_host_redaction_representation';
391
+ scalarType: ScalarType;
392
+ }> = [];
393
+ const work = {
394
+ earlierHistoryAvailable: 0,
395
+ earlierHistoryInWindow: 0,
396
+ earlierHistoryWindowLimit: MAX_EQUALITY_HISTORY_REQUESTS,
397
+ earlierRequestsScanned: 0,
398
+ responseBodiesFound: 0,
399
+ responsesDecoded: 0,
400
+ responsesSkipped: 0,
401
+ responseNodes: 0,
402
+ responseLeaves: 0,
403
+ comparisons: 0,
404
+ };
405
+ const skippedByReason: Record<string, number> = {};
406
+ const result = {
407
+ requestSeq,
408
+ requestPointer,
409
+ equalityScope: 'supplied_host_redaction_representation_only' as const,
410
+ responseFormat: responseFormat ?? 'invalid',
411
+ facts,
412
+ work,
413
+ skippedByReason,
414
+ };
415
+ if (!responseFormat) return { ...result, reasonCode: 'invalid_response_format' as const };
416
+ const requestIndex = session.requests.findIndex((item) => item.seq === requestSeq);
417
+ if (requestIndex < 0) return { ...result, reasonCode: 'request_not_found' as const };
418
+ if (!session.requests[requestIndex])
419
+ return { ...result, reasonCode: 'request_not_found' as const };
420
+ result.work.earlierHistoryAvailable = requestIndex;
421
+ result.work.earlierHistoryInWindow = Math.min(requestIndex, MAX_EQUALITY_HISTORY_REQUESTS);
422
+ const requests = session.requests
423
+ .slice(Math.max(0, requestIndex - MAX_EQUALITY_HISTORY_REQUESTS), requestIndex + 1)
424
+ .sort((left, right) => left.seq - right.seq);
425
+ const selected = prepareBodyScalarEquality(selectedStructure, requestPointer);
426
+ if (!selected.ok) return { ...result, reasonCode: `selected_pointer_${selected.error}` as const };
427
+ const requestType = selected.scalarType;
428
+ const truncated = new Set<
429
+ 'earlier_history' | 'responses' | 'bytes' | 'response_decode' | 'nodes' | 'leaves' | 'matches'
430
+ >();
431
+ if (requestIndex > MAX_EQUALITY_HISTORY_REQUESTS) truncated.add('earlier_history');
432
+ const before = lowerBound(requests, requestSeq);
433
+ let responsesSeen = 0;
434
+ let responseBytes = 0;
435
+ let partialResponseDecode = false;
436
+ const skip = (reason: string) => {
437
+ result.work.responsesSkipped++;
438
+ result.skippedByReason[reason] = (result.skippedByReason[reason] ?? 0) + 1;
439
+ };
440
+ outer: for (let position = before - 1; position >= 0; position--) {
441
+ result.work.earlierRequestsScanned++;
442
+ const earlier = requests[position];
443
+ const body = earlier?.response?.body;
444
+ if (!earlier || typeof body !== 'string') {
445
+ skip('missing_response_body');
446
+ continue;
447
+ }
448
+ if (responsesSeen++ >= MAX_EQUALITY_RESPONSES) {
449
+ truncated.add('responses');
450
+ skip('response_count_limit');
451
+ break;
452
+ }
453
+ result.work.responseBodiesFound++;
454
+ if (body.length > MAX_EQUALITY_RESPONSE_BYTES) {
455
+ truncated.add('bytes');
456
+ skip('response_size_limit');
457
+ continue;
458
+ }
459
+ const bytes = groundingEncoder.encode(body).length;
460
+ if (bytes > MAX_EQUALITY_RESPONSE_BYTES) {
461
+ truncated.add('bytes');
462
+ skip('response_size_limit');
463
+ continue;
464
+ }
465
+ if (bytes > MAX_EQUALITY_TOTAL_BYTES - responseBytes) {
466
+ truncated.add('bytes');
467
+ skip('total_response_budget');
468
+ continue;
469
+ }
470
+ responseBytes += bytes;
471
+ const decoded = decodeBodyStructure(body, responseFormat);
472
+ if (!decoded.ok) {
473
+ skip(`decode_${decoded.code}`);
474
+ continue;
475
+ }
476
+ const decodeNotCheckedReason = bodyStructureNotCheckedReason(decoded.structure);
477
+ if (decodeNotCheckedReason) {
478
+ partialResponseDecode = true;
479
+ truncated.add('response_decode');
480
+ skip(`decode_${decodeNotCheckedReason}`);
481
+ continue;
482
+ }
483
+ result.work.responsesDecoded++;
484
+ const collected = collectResponseScalars(decoded.structure.value);
485
+ result.work.responseNodes += collected.nodes;
486
+ result.work.responseLeaves += collected.leaves.length;
487
+ if (collected.nodesTruncated) truncated.add('nodes');
488
+ if (collected.leavesTruncated) truncated.add('leaves');
489
+ for (const leaf of collected.leaves) {
490
+ result.work.comparisons++;
491
+ if (leaf.type !== requestType || !selected.equals(leaf.value)) continue;
492
+ if (result.facts.length >= MAX_EQUALITY_MATCHES) {
493
+ truncated.add('matches');
494
+ break outer;
495
+ }
496
+ result.facts.push({
497
+ responseSeq: earlier.seq,
498
+ ...(options.includePaths ? { responsePath: leaf.path } : {}),
499
+ responseDepth: leaf.depth,
500
+ matchKind: 'exact_scalar_equality_in_supplied_host_redaction_representation',
501
+ scalarType: requestType,
372
502
  });
373
503
  }
374
504
  }
375
- return out;
505
+ return {
506
+ ...result,
507
+ reasonCode:
508
+ facts.length > 0
509
+ ? 'matches_found'
510
+ : partialResponseDecode
511
+ ? 'eligible_response_not_fully_decoded'
512
+ : work.responsesDecoded > 0
513
+ ? 'no_equal_scalar_in_supplied_representation'
514
+ : 'no_decoded_responses',
515
+ ...(truncated.size ? { truncated: [...truncated] } : {}),
516
+ };
376
517
  }