@enricai/barnacle 1.9.2 → 1.9.4

package/dist/scripts/recon-generate.js

@@ -26,6 +26,7 @@ exports.inferZodSchemaFromSamples = inferZodSchemaFromSamples;
 exports.selectPayloadAction = selectPayloadAction;
 exports.selectReturnAction = selectReturnAction;
 exports.selectEffectiveResponseBody = selectEffectiveResponseBody;
+exports.resolveManifestActionSequence = resolveManifestActionSequence;
 exports.extractActionSequence = extractActionSequence;
 exports.extractGraphQLActionSequence = extractGraphQLActionSequence;
 exports.detectFormSchemaFieldNames = detectFormSchemaFieldNames;
@@ -344,7 +345,7 @@ const IGNORE_REQUEST_HEADERS = new Set([
  * headers (a `X-CSRF-Token`, a `Job-Boards-API-Token`, an `API-ShortName`,
  * etc.) without the generator needing to know about any particular site.
  */
-function deriveRequestHeaders(captures, replays, baseUrl) {
+function deriveRequestHeaders(captures, replays, baseUrl, submitPatterns = null) {
     const successfulUrls = new Set(replays.filter((r) => r.success).map((r) => endpointKey(r.url)));
     // Prefer ACTION captures (non-GET 2xx to baseUrl host, non-telemetry) as
     // the authoritative header source. Replay-matched static-asset GETs lack
@@ -353,7 +354,7 @@ function deriveRequestHeaders(captures, replays, baseUrl) {
     // captures exist (multi-step submission flows), use them. For sites
     // where the flow is a single REST call (no detectable action sequence),
     // fall back to the replay-matched captures.
-    const actionCaptures = extractActionSequence(captures, baseUrl).map((a) => a.capture);
+    const actionCaptures = extractActionSequence(captures, baseUrl, submitPatterns).map((a) => a.capture);
     const replayMatchedCaptures = captures.filter((c) => successfulUrls.has(endpointKey(c.url)));
     const relevantCaptures = actionCaptures.length > 0 ? actionCaptures : replayMatchedCaptures;
     const counts = new Map();
@@ -409,16 +410,75 @@ function firstEndpointPath(captures) {
     }
     return "/api/search";
 }
+/**
+ * Builds the compiled submit-pattern predicate. A flow-declared regex that
+ * fails to compile is a broken flow (recon-browser validates it eagerly too),
+ * so we let the `RegExp` constructor throw rather than silently reverting to
+ * unfiltered selection, which would re-admit the page-chrome bloat this gate
+ * exists to remove.
+ */
+function compileSubmitMatcher(patterns) {
+    if (patterns === null || (patterns.endpoint === null && patterns.body === null)) {
+        return () => true;
+    }
+    const endpointRx = patterns.endpoint === null ? null : new RegExp(patterns.endpoint);
+    const bodyRx = patterns.body === null ? null : new RegExp(patterns.body);
+    return (capture) => {
+        if (endpointRx !== null && !endpointRx.test(capture.url))
+            return false;
+        if (bodyRx !== null && !bodyRx.test(capture.requestPostData ?? ""))
+            return false;
+        return true;
+    };
+}
+/**
+ * Reads `submit-manifest.json` (written by recon-browser) and resolves it to the
+ * authoritative submission action sequence. This is the deepest submit-selection
+ * signal: recon-browser matched these captures against the flow's declared submit
+ * patterns at run time, so generate emits exactly them instead of re-deriving the
+ * submission from raw traffic. Returns null when no manifest exists (older runs,
+ * `recon-http`-only) so the caller falls back to pattern/heuristic extraction.
+ *
+ * The manifest's `index` is the capture's sort-order position, and `captures`
+ * arrives already `.sort()`ed by `readJsonDir`, so `captures[index]` is the same
+ * capture recon-browser recorded — cross-checked on `url` as a guard against a
+ * capture set that drifted between runs.
+ */
+function resolveManifestActionSequence(runRoot, captures) {
+    const manifestPath = (0, node_path_1.join)(runRoot, "submit-manifest.json");
+    let entries;
+    try {
+        entries = JSON.parse((0, node_fs_1.readFileSync)(manifestPath, "utf8"));
+    }
+    catch {
+        return null;
+    }
+    if (!Array.isArray(entries) || entries.length === 0)
+        return null;
+    const resolved = [];
+    for (const entry of entries) {
+        const capture = captures[entry.index];
+        if (capture === undefined || capture.url !== entry.url)
+            return null;
+        resolved.push({ capture, index: entry.index });
+    }
+    return resolved;
+}
 /**
  * Extracts the ordered sequence of meaningful POSTs that represent the
  * transactional flow: same-host 2xx POSTs, minus telemetry and error-reporting
  * sinks. Assets need no filter of their own — they arrive as GETs.
  *
+ * When the flow declares submit patterns, only POSTs matching them survive —
+ * this isolates the submission from same-origin page chrome (bootstrap, chatbot,
+ * JWT refresh, reference-lookup) that a browser fires incidentally. Absent
+ * patterns preserve the host/noise heuristic exactly.
+ *
  * Exported for tests: this predicate decides what a generated plugin will POST
  * at a live site, and it is the only gate between a browser's incidental
  * chatter and the emitted hot path.
  */
-function extractActionSequence(captures, baseUrl) {
+function extractActionSequence(captures, baseUrl, submitPatterns = null) {
     let host;
     try {
         host = new URL(baseUrl).host;
@@ -426,6 +486,7 @@ function extractActionSequence(captures, baseUrl) {
     catch {
         host = "";
     }
+    const matchesSubmit = compileSubmitMatcher(submitPatterns);
     return captures
         .map((capture, index) => ({ capture, index }))
         .filter(({ capture }) => {
@@ -444,6 +505,8 @@ function extractActionSequence(captures, baseUrl) {
             return false;
         if ((0, capture_filters_1.isNoiseUrl)(capture.url))
             return false;
+        if (!matchesSubmit(capture))
+            return false;
         return true;
     });
 }
@@ -460,7 +523,7 @@ function extractActionSequence(captures, baseUrl) {
  * Exported for tests: this predicate decides what a generated GraphQL plugin
  * will send at a live site.
  */
-function extractGraphQLActionSequence(captures, baseUrl) {
+function extractGraphQLActionSequence(captures, baseUrl, submitPatterns = null) {
     let host;
     try {
         host = new URL(baseUrl).host;
@@ -468,6 +531,7 @@ function extractGraphQLActionSequence(captures, baseUrl) {
     catch {
         host = "";
     }
+    const matchesSubmit = compileSubmitMatcher(submitPatterns);
     return captures
         .map((capture, index) => ({ capture, index }))
         .filter(({ capture }) => {
@@ -486,6 +550,8 @@ function extractGraphQLActionSequence(captures, baseUrl) {
             return false;
         if ((0, capture_filters_1.isNoiseUrl)(capture.url))
             return false;
+        if (!matchesSubmit(capture))
+            return false;
         return capture.query !== null && /^\s*mutation\b/.test(capture.query);
     });
 }
@@ -533,6 +599,34 @@ function* walkStringLeaves(value, path = []) {
         }
     }
 }
+/**
+ * Parses a captured request body as JSON and returns its string leaf *values*
+ * (object keys are excluded — a key is never a JSON value). Returns null when
+ * the body is absent or not JSON (e.g. multipart raw bytes), so callers can
+ * fall back to whole-body substring matching. This lets the produces-filter
+ * match state values against JSON values only, so a response string that
+ * appears downstream solely in a JSON *key* position is not mistaken for a
+ * reused value — while a value legitimately embedded inside a longer value
+ * still matches via substring.
+ */
+function jsonBodyLeafValues(requestPostData) {
+    if (typeof requestPostData !== "string" || requestPostData.length === 0)
+        return null;
+    const parsed = (() => {
+        try {
+            return JSON.parse(requestPostData);
+        }
+        catch {
+            return undefined;
+        }
+    })();
+    if (parsed === undefined)
+        return null;
+    const values = [];
+    for (const { value } of walkStringLeaves(parsed))
+        values.push(value);
+    return values;
+}
 /**
  * Yields every primitive leaf (string, number, boolean, null) in the JSON
  * value with its path. Used by the body-literal substitution pass to find
@@ -1314,12 +1408,31 @@ function compileActionSteps(actions, stateIndex) {
     // Pre-scan: collect all state values referenced by ANY action's URL/headers/body
     // so we only "produce" the values that are actually consumed downstream.
     for (const { capture } of actions) {
-        const haystacks = [capture.url];
-        if (capture.requestPostData)
-            haystacks.push(capture.requestPostData);
+        const bodyLeafValues = jsonBodyLeafValues(capture.requestPostData);
         for (const sv of stateIndex.values()) {
-            if (haystacks.some((h) => h.includes(sv.value)))
+            if (capture.url.includes(sv.value)) {
+                usedValues.add(sv.value);
+                continue;
+            }
+            // Body consumption is matched against JSON *values* only, never keys: a
+            // state value is a real cross-step dependency when a later request
+            // re-sends it inside a JSON value — either standalone or embedded in a
+            // composite value (e.g. a jobId reused inside a longer jobSeqNo). A
+            // match that lands on a JSON *key* is not reuse: binding then splicing it
+            // would emit a variable into a key position and produce uncompilable
+            // `"${var}":…` / `${${var}}` (e.g. a response echoing field NAMES like
+            // `tokens:["firstName","lastName"]` — those strings appear downstream
+            // only as keys, never within any value). Object keys are never JSON
+            // string leaves, so testing against leaf values alone excludes them while
+            // preserving substring-in-value reuse. Non-JSON bodies (multipart raw
+            // bytes) keep whole-body substring matching.
+            if (bodyLeafValues === null) {
+                if (capture.requestPostData?.includes(sv.value))
+                    usedValues.add(sv.value);
+            }
+            else if (bodyLeafValues.some((leaf) => leaf.includes(sv.value))) {
                 usedValues.add(sv.value);
+            }
         }
         for (const [headerName, headerValue] of Object.entries(capture.requestHeaders)) {
             for (const sv of stateIndex.values()) {
@@ -2814,25 +2927,52 @@ async function main() {
             return [];
         }
     })();
-    const { flowSteps, frameSelector } = (() => {
+    const { flowSteps, frameSelector, submitEndpointPattern, submitBodyPattern } = (() => {
         const flowFile = `src/sites/${siteId}/recon-flow.json`;
         try {
             const raw = JSON.parse((0, node_fs_1.readFileSync)(flowFile, "utf8"));
             if (Array.isArray(raw))
-                return { flowSteps: raw, frameSelector: undefined };
+                return {
+                    flowSteps: raw,
+                    frameSelector: undefined,
+                    submitEndpointPattern: null,
+                    submitBodyPattern: null,
+                };
             if (raw !== null &&
                 typeof raw === "object" &&
                 "steps" in raw &&
                 Array.isArray(raw.steps)) {
                 const obj = raw;
-                return { flowSteps: obj.steps, frameSelector: obj.frameSelector };
+                return {
+                    flowSteps: obj.steps,
+                    frameSelector: obj.frameSelector,
+                    submitEndpointPattern: obj.submitEndpointPattern ?? null,
+                    submitBodyPattern: obj.submitBodyPattern ?? null,
+                };
             }
-            return { flowSteps: [], frameSelector: undefined };
+            return {
+                flowSteps: [],
+                frameSelector: undefined,
+                submitEndpointPattern: null,
+                submitBodyPattern: null,
+            };
         }
         catch {
-            return { flowSteps: [], frameSelector: undefined };
+            return {
+                flowSteps: [],
+                frameSelector: undefined,
+                submitEndpointPattern: null,
+                submitBodyPattern: null,
+            };
         }
     })();
+    // Flow-declared signals that isolate the submission POSTs from same-origin
+    // page chrome. Threaded into action-sequence extraction and header derivation
+    // so both draw from the real submission, not incidental widget/chatbot POSTs.
+    const submitPatterns = {
+        endpoint: submitEndpointPattern,
+        body: submitBodyPattern,
+    };
     // Resolved once and threaded down, never captured into a module const: a
     // module-level const would freeze at import time, which is the bug that makes
     // RECON_QUESTION_KEYWORDS silently inert for anyone setting it after load.
@@ -2842,7 +2982,7 @@ async function main() {
     const formSchema = await resolveFormSchema(formSchemaSpecifier);
     const pascal = toPascalCase(siteId);
     const baseUrl = deriveBaseUrl(captures);
-    const baseHeaders = deriveRequestHeaders(captures, replays, baseUrl);
+    const baseHeaders = deriveRequestHeaders(captures, replays, baseUrl, submitPatterns);
     const minTime = deriveMinTime(rateLimits);
     const safeRps = rateLimits.find((f) => f.safeRps !== null)?.safeRps ?? Math.floor(1000 / minTime);
     const responseBody = firstSuccessfulReplayBody(replays);
@@ -2852,9 +2992,19 @@ async function main() {
     // Detect a multi-step submission flow (transactional sites like apply forms,
     // checkout, etc.). When the action sequence has 2+ POSTs, switch the
     // contract template to emit a state-threaded executeHttp.
-    const rawActionCaptures = gql
-        ? extractGraphQLActionSequence(captures, baseUrl)
-        : collapseRedundantPatches(extractActionSequence(captures, baseUrl));
+    //
+    // Selection precedence: (A) the authoritative submit-manifest recon-browser
+    // wrote from the verified submission; else (B/C) pattern/heuristic extraction.
+    // The manifest is the only signal that separates a submission POST from a
+    // page-chrome POST sharing its URL, so it wins when present.
+    const manifestActionCaptures = resolveManifestActionSequence(runRoot, captures);
+    if (manifestActionCaptures !== null) {
+        logger.info(`submission selection: using submit-manifest.json (${manifestActionCaptures.length} authoritative capture(s))`);
+    }
+    const rawActionCaptures = manifestActionCaptures ??
+        (gql
+            ? extractGraphQLActionSequence(captures, baseUrl, submitPatterns)
+            : collapseRedundantPatches(extractActionSequence(captures, baseUrl, submitPatterns)));
     // Form-schema detection runs BEFORE state-indexing so the field-id/option-id
     // UUIDs can be shielded from indexing — those UUIDs are stable schema
     // anchors that T2/T3 substitution depends on remaining literal in body