@enricai/barnacle 1.4.1 → 1.4.2

package/dist/scripts/recon-generate.js

@@ -20,15 +20,19 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.resolveStepPayloadField = resolveStepPayloadField;
 exports.inferZodSchemaFromSamples = inferZodSchemaFromSamples;
 exports.selectPayloadAction = selectPayloadAction;
+exports.indexStateValues = indexStateValues;
+exports.compileActionSteps = compileActionSteps;
 exports.loadQuestionPromptKeywords = loadQuestionPromptKeywords;
 exports.emitMultiStepExecuteHttp = emitMultiStepExecuteHttp;
 exports.emitContractTs = emitContractTs;
 exports.emitConfigManifest = emitConfigManifest;
 exports.emitBrowserFlowTs = emitBrowserFlowTs;
+exports.emitIndexTs = emitIndexTs;
 const node_fs_1 = require("node:fs");
 const node_path_1 = require("node:path");
 const errors_1 = require("../lib/errors");
 const logging_1 = require("../lib/logging");
+const plugin_api_version_1 = require("../plugins/plugin-api-version");
 const plugin_manifest_envelope_1 = require("../plugins/plugin-manifest-envelope");
 const load_vocabulary_1 = require("../recon/load-vocabulary");
 const vocabulary_1 = require("../recon/vocabulary");
@@ -1057,12 +1061,35 @@ const MAX_STATE_VALUE_LENGTH = 256;
  * does not exist". Closed set, literal-string match — never expand to
  * pattern-based detection (would trip the no-regex-on-open-sets rule). */
 const PLACEHOLDER_STATE_VALUES = new Set(["00000000-0000-0000-0000-000000000000"]);
+/**
+ * Splits a raw `Set-Cookie` response-header string into `name`/`value` pairs.
+ * Captures store `responseHeaders` as a flat `Record<string, string>`
+ * (see recon-shared.ts's `Capture`), so multiple `Set-Cookie` headers from the
+ * same response — if the recon browser's CDP session folds them together —
+ * would already have lost their individual boundaries before reaching here;
+ * this only recovers name/value pairs from whatever single string survives.
+ */
+function* walkSetCookiePairs(rawSetCookie) {
+    const pair = rawSetCookie.split(";", 1)[0] ?? "";
+    const eq = pair.indexOf("=");
+    if (eq === -1)
+        return;
+    const name = pair.slice(0, eq).trim();
+    const value = pair.slice(eq + 1).trim();
+    if (name && value)
+        yield { name, value };
+}
 /**
  * Walks every capture's response (including GETs — formHistoryId-style values
  * may originate in a state-load GET, not a POST). Indexes every string leaf
  * whose length is in [MIN, MAX], recording the EARLIEST capture index that
  * produced it. Later occurrences of the same value reuse the earliest origin.
  *
+ * Also indexes response-header/cookie-origin values (e.g. a `Set-Cookie`
+ * auth token) the same way, tagged with `headerOrigin` instead of a body
+ * `path` — this is what lets a stateful API's token-mint response feed a
+ * later call's `Cookie` header via `compileActionSteps`.
+ *
  * The index is intentionally permissive — it doesn't try to shape-match
  * "what looks like a token" because token shapes are an open set across the
  * web. Authoritative filtering happens downstream in `compileActionSteps`,
@@ -1073,6 +1100,8 @@ const PLACEHOLDER_STATE_VALUES = new Set(["00000000-0000-0000-0000-000000000000"
  * the LATER non-placeholder occurrence at the same JSON path becomes the
  * canonical binding instead.
  */
+/** Exported for unit testing — lets tests exercise the produces[] walk (body
+ * AND header/cookie origins) directly against synthetic Capture sequences. */
 function indexStateValues(captures, shieldedUuids = new Set(), actionCaptureIndices = new Set()) {
     const index = new Map();
     // First pass: identify the earliest origin among ACTION captures for each
@@ -1085,10 +1114,32 @@ function indexStateValues(captures, shieldedUuids = new Set(), actionCaptureIndi
     const haveActionFilter = actionCaptureIndices.size > 0;
     for (let i = 0; i < captures.length; i++) {
         const c = captures[i];
-        if (c.responseBody === undefined || c.responseBody === null)
-            continue;
         if (haveActionFilter && !actionCaptureIndices.has(i))
             continue;
+        // Headers/cookies are indexed regardless of responseBody presence — a
+        // token-mint call like disneycruise's `authz/private` returns `{}` and
+        // carries its whole payload in `Set-Cookie`.
+        const rawSetCookie = Object.entries(c.responseHeaders).find(([k]) => k.toLowerCase() === "set-cookie")?.[1];
+        if (rawSetCookie !== undefined) {
+            for (const { name, value } of walkSetCookiePairs(rawSetCookie)) {
+                if (value.length < MIN_STATE_VALUE_LENGTH)
+                    continue;
+                if (value.length > MAX_STATE_VALUE_LENGTH)
+                    continue;
+                if (PLACEHOLDER_STATE_VALUES.has(value))
+                    continue;
+                if (!index.has(value)) {
+                    index.set(value, {
+                        value,
+                        originIndex: i,
+                        path: [],
+                        headerOrigin: { sourceHeader: "set-cookie", cookieName: name },
+                    });
+                }
+            }
+        }
+        if (c.responseBody === undefined || c.responseBody === null)
+            continue;
         // For GET captures, only index UUID-shaped strings. GET captures (today,
         // only the form-schema fetch inserted as an action step) surface stable
         // structural identifiers — UUIDs that downstream POSTs need to thread.
@@ -1125,9 +1176,18 @@ function indexStateValues(captures, shieldedUuids = new Set(), actionCaptureIndi
 function isValidJsIdentifier(s) {
     return /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(s);
 }
-/** Converts a path like ["Auth","Token"] to a JS access expression ".Auth.Token". */
+/**
+ * Converts a path like ["Auth","Token"] to a JS access expression ".Auth.Token".
+ * Bracket segments (numeric array indices, non-identifier keys) get a trailing
+ * `!` — under `noUncheckedIndexedAccess` an array/index-signature access types
+ * as `T | undefined`, and this accessor is only ever used against a real Zod-
+ * inferred array/object type (payload fields, captured response bodies), never
+ * against the object-literal assertion types `pathToAssertionType` builds (those
+ * use known string-literal keys, which `noUncheckedIndexedAccess` does not
+ * widen). Dot segments stay bare since object property access isn't affected.
+ */
 function pathToAccessor(path) {
-    return path.map((p) => (isValidJsIdentifier(p) ? `.${p}` : `[${JSON.stringify(p)}]`)).join("");
+    return path.map((p) => (isValidJsIdentifier(p) ? `.${p}` : `[${JSON.stringify(p)}]!`)).join("");
 }
 /**
  * Builds a nested TypeScript assertion type matching a JSON path. e.g.
@@ -1161,26 +1221,75 @@ function pathToVarName(path) {
  * var name, the state values its response produces (used by downstream steps),
  * and a multipart flag (request body bytes not captured).
  */
+/** Exported for unit testing — see `indexStateValues`. */
 function compileActionSteps(actions, stateIndex) {
     const usedValues = new Set();
+    // Maps a used state value to the request-header NAME that carries it, for
+    // values whose consuming reference is a request header (not the URL/body).
+    // A header-origin produce needs this as its `targetHeader` — the header the
+    // *next* httpClient call must send the bound value back on. Only the first
+    // consuming header name observed wins; a value used in more than one distinct
+    // header downstream isn't a shape this models (see http-client.ts's `bind`,
+    // which is single-target per binding).
+    const usedValueTargetHeader = new Map();
     // Pre-scan: collect all state values referenced by ANY action's URL/headers/body
     // so we only "produce" the values that are actually consumed downstream.
     for (const { capture } of actions) {
         const haystacks = [capture.url];
-        for (const v of Object.values(capture.requestHeaders))
-            haystacks.push(v);
         if (capture.requestPostData)
             haystacks.push(capture.requestPostData);
         for (const sv of stateIndex.values()) {
             if (haystacks.some((h) => h.includes(sv.value)))
                 usedValues.add(sv.value);
         }
+        for (const [headerName, headerValue] of Object.entries(capture.requestHeaders)) {
+            for (const sv of stateIndex.values()) {
+                if (!headerValue.includes(sv.value))
+                    continue;
+                usedValues.add(sv.value);
+                if (!usedValueTargetHeader.has(sv.value)) {
+                    usedValueTargetHeader.set(sv.value, headerName);
+                }
+            }
+        }
     }
     let lastHost = null;
     return actions.map(({ capture, index }, i) => {
         const varName = `r${i}`;
         const produces = [];
         const seenNames = new Set();
+        // Header/cookie-origin produces — walked first so a value that appears in
+        // BOTH a Set-Cookie and the JSON body (unlikely, but not ruled out) prefers
+        // the header binding, which is what the runtime actually threads.
+        const rawSetCookie = Object.entries(capture.responseHeaders).find(([k]) => k.toLowerCase() === "set-cookie")?.[1];
+        if (rawSetCookie !== undefined) {
+            for (const { name: cookieName, value } of walkSetCookiePairs(rawSetCookie)) {
+                if (!usedValues.has(value))
+                    continue;
+                const sv = stateIndex.get(value);
+                if (!sv || sv.originIndex !== index || !sv.headerOrigin)
+                    continue;
+                const targetHeader = usedValueTargetHeader.get(value);
+                if (!targetHeader)
+                    continue;
+                let name = `${cookieName.replace(/[^A-Za-z0-9]/g, "")}Cookie`;
+                if (!/^[A-Za-z_$]/.test(name))
+                    name = `_${name}`;
+                let suffix = 1;
+                while (seenNames.has(name)) {
+                    suffix++;
+                    name = `${cookieName.replace(/[^A-Za-z0-9]/g, "")}Cookie${suffix}`;
+                }
+                seenNames.add(name);
+                produces.push({
+                    kind: "header",
+                    name,
+                    sourceHeader: sv.headerOrigin.sourceHeader,
+                    cookieName: sv.headerOrigin.cookieName,
+                    targetHeader,
+                });
+            }
+        }
         if (capture.responseBody !== undefined && capture.responseBody !== null) {
             for (const { value, path } of walkStringLeaves(capture.responseBody)) {
                 if (!usedValues.has(value))
@@ -1196,7 +1305,7 @@ function compileActionSteps(actions, stateIndex) {
                     name = `${pathToVarName(path)}${suffix}`;
                 }
                 seenNames.add(name);
-                produces.push({ name, pathExpr: `${varName}${pathToAccessor(path)}`, path });
+                produces.push({ kind: "body", name, pathExpr: `${varName}${pathToAccessor(path)}`, path });
             }
         }
         const ct = Object.entries(capture.requestHeaders).find(([k]) => k.toLowerCase() === "content-type");
@@ -1213,6 +1322,27 @@ function compileActionSteps(actions, stateIndex) {
         return { capture, varName, produces, isMultipart, isCrossDomain };
     });
 }
+/**
+ * Collects every header/cookie-origin produce across an action sequence, in
+ * step order — this is what `emitContractTs` renders as `createHttpClient`'s
+ * `bind` option so the generated `executeHttp` actually forwards a value like
+ * disneycruise's `Set-Cookie: __pa=<jwt>` mint to the stateful call that 401s
+ * without it. Deduped by `targetHeader`: `HttpResponseBinding` (http-client.ts)
+ * is one binding per target header, so if two steps somehow produced the same
+ * target the earliest wins.
+ */
+function collectHeaderBindings(actionSteps) {
+    const byTarget = new Map();
+    for (const step of actionSteps) {
+        for (const p of step.produces) {
+            if (p.kind !== "header")
+                continue;
+            if (!byTarget.has(p.targetHeader))
+                byTarget.set(p.targetHeader, p);
+        }
+    }
+    return [...byTarget.values()];
+}
 /**
  * Replaces occurrences of state values in `template` with `${varName}`
  * interpolations. Returns a JS template-literal string fragment (no backticks).
@@ -1227,6 +1357,12 @@ function interpolateStateValues(template, priorSteps, payloadAccessorByValue = n
     const varNameByValue = new Map();
     for (const step of priorSteps) {
         for (const p of step.produces) {
+            // Header/cookie-origin produces have no body path — their value never
+            // appears as a literal in a URL/body template (http-client's `bind`
+            // forwards it directly as a request header), so there's nothing to
+            // interpolate here.
+            if (p.kind === "header")
+                continue;
             let cursor = step.capture.responseBody;
             for (const segment of p.path) {
                 if (cursor !== null &&
@@ -1820,6 +1956,11 @@ function emitMultiStepExecuteHttp(actions, inputBody, errorSignals, fieldNameMap
             }
         }
         for (const p of step.produces) {
+            // Header/cookie-origin produces never surface as a JS accessor —
+            // createHttpClient's `bind` option (rendered once, above the steps)
+            // captures and forwards the value internally.
+            if (p.kind === "header")
+                continue;
             if (declaredNames.has(p.name))
                 continue;
             if (!referencedNames.has(p.name))
@@ -1845,11 +1986,30 @@ function summariseResponseShape(value) {
     }
     return obj;
 }
+/**
+ * Renders `headerBindings` as a trailing `, bind: [...]` fragment for
+ * `createHttpClient`'s options object literal — empty string when there are
+ * none, so a plugin with no header/cookie-origin state keeps the exact output
+ * this emitter already produced. Structurally matches `HttpResponseBinding`
+ * (http-client.ts) without importing the type: the object literal typechecks
+ * against `HttpClientOptions.bind` on its own shape.
+ */
+function bindOptionLiteral(headerBindings) {
+    if (headerBindings.length === 0)
+        return "";
+    const entries = headerBindings
+        .map((b) => {
+        const cookieNameField = b.cookieName !== undefined ? ` cookieName: ${JSON.stringify(b.cookieName)},` : "";
+        return `{ sourceHeader: ${JSON.stringify(b.sourceHeader)},${cookieNameField} targetHeader: ${JSON.stringify(b.targetHeader)} }`;
+    })
+        .join(", ");
+    return `, bind: [${entries}]`;
+}
 // ── code emitters ─────────────────────────────────────────────────────────────
 /** Generates a complete contract.ts source string for a plugin — exported so
  * unit tests can drive the emitter directly without spawning the CLI. */
 function emitContractTs(opts) {
-    const { siteId, pascal, baseUrl, baseHeaders, minTime, safeRps, responseBody, gql, gqlQuery, endpointPath, auxFiles, multiStepBody, inputBody, hasMultipartStep = false, discoveredFormFields, fieldOptionsMap, discoveredOptionFields, discoveredRawOptionFields, discoveredAdditionalBodyKeys, payloadFieldNames, base64ContentHelper = "", } = opts;
+    const { siteId, pascal, baseUrl, baseHeaders, minTime, safeRps, responseBody, gql, gqlQuery, endpointPath, auxFiles, multiStepBody, inputBody, hasMultipartStep = false, discoveredFormFields, fieldOptionsMap, discoveredOptionFields, discoveredRawOptionFields, discoveredAdditionalBodyKeys, payloadFieldNames, base64ContentHelper = "", headerBindings = [], } = opts;
     // Multi-step plugins thread responses through many different shapes that a
     // single Zod schema can't cover — use z.unknown() so each per-step access
     // compiles cleanly. Single-endpoint plugins keep the inferred schema.
@@ -2007,7 +2167,7 @@ function getGql(baseUrl: string): GqlFn {
 }
 `
         : `
-const httpClient = createHttpClient({ schema: ${pascal}ResponseSchema, bottleneck: limiter, baseHeaders: BASE_HEADERS });
+const httpClient = createHttpClient({ schema: ${pascal}ResponseSchema, bottleneck: limiter, baseHeaders: BASE_HEADERS${bindOptionLiteral(headerBindings)} });
 `;
     const executeHttpBody = multiStepBody
         ? multiStepBody
@@ -2029,6 +2189,7 @@ const httpClient = createHttpClient({ schema: ${pascal}ResponseSchema, bottlenec
     const queryChecklistLine = gql
         ? `\n *   [ ] Trim UI-only fields from ${pascal.toUpperCase()}_QUERY (keep only fields you need)`
         : "";
+    const camel = siteId.replace(/-([a-z])/g, (_, c) => c.toUpperCase());
     return `/**
  * Generated by recon-generate.ts — review before shipping.
  *
@@ -2036,6 +2197,9 @@ const httpClient = createHttpClient({ schema: ${pascal}ResponseSchema, bottlenec
  *   [ ] Narrow ${pascal}ResponseSchema to match the real response shape
  *   [ ] Adjust ${pascal}PayloadSchema to your actual request parameters
  *   [ ] Verify BASE_HEADERS — remove any that aren't load-bearing
+ *   [ ] Out-of-tree: \`pnpm add bottleneck zod\` — this file imports both
+ *       directly, and a strict node_modules layout (pnpm) won't resolve
+ *       them as transitive deps of @enricai/barnacle alone
  */
 
 import Bottleneck from "bottleneck";
@@ -2067,13 +2231,14 @@ ${queryConst}${gqlCacheBlock}${fixtureComments}
  * Plugin for ${siteId}. Tries the direct-HTTP hot path first; falls back to
  * Stagehand automatically on schema drift or bot challenge.
  */
-export const ${siteId.replace(/-([a-z])/g, (_, c) => c.toUpperCase())}Plugin: SitePlugin<${pascal}Payload, ${pascal}Response> = {
+export const ${camel}Plugin: SitePlugin<${pascal}Payload, ${pascal}Response> = {
   meta: {
     siteId: ${JSON.stringify(siteId)},
     displayName: ${JSON.stringify(pascal.replace(/([A-Z])/g, " $1").trim())},
     bodySchema: ${pascal}PayloadSchema,
     responseSchema: ${pascal}ResponseSchema,
-    defaultBaseUrl: ${JSON.stringify(baseUrl)},${hasMultipartStep ? "\n    multipart: true," : ""}
+    defaultBaseUrl: ${JSON.stringify(baseUrl)},
+    apiVersion: ${JSON.stringify(plugin_api_version_1.PLUGIN_API_VERSION)},${hasMultipartStep ? "\n    multipart: true," : ""}
   },
 
   /** Hot path: direct HTTP — no browser, no LLM tokens. */
@@ -2094,6 +2259,11 @@ ${executeHttpBody}
     return { data: raw as ${pascal}Response };
   },
 };
+
+// Out-of-tree loader resolves \`m.plugin ?? m.default ?? m\` — this named
+// alias is what BARNACLE_PLUGINS finds; without it the loader would fall
+// through to \`m.default\` (the response schema above) and 404 at runtime.
+export { ${camel}Plugin as plugin };
 `;
 }
 /**
@@ -2332,19 +2502,23 @@ ${flowStepsBlock}
 `;
     return { code, payloadFieldNames };
 }
+/** Generates the site's index.ts barrel — exported so the out-of-tree e2e
+ * test can drive the emitter directly without spawning the CLI. */
 function emitIndexTs(opts) {
     const { siteId } = opts;
     const camel = siteId.replace(/-([a-z])/g, (_, c) => c.toUpperCase());
     return `/**
  * Generated by recon-generate.ts.
- * Register this built-in plugin by adding it to \`BUILTIN_SITE_PLUGINS\` in
- * src/plugins/discover.ts:
+ * Build this package, then point BARNACLE_PLUGINS at the compiled module —
+ * no core edits required:
+ *
+ *   BARNACLE_PLUGINS=./dist/sites/${siteId}/index.js pnpm start
  *
- *   import { ${camel}Plugin } from "@/sites/${siteId}";
- *   BUILTIN_SITE_PLUGINS.push(${camel}Plugin as SitePlugin<unknown, unknown>);
+ * The loader resolves \`m.plugin ?? m.default ?? m\` — the \`plugin\` alias
+ * below is what it finds.
  */
 
-export { ${camel}Plugin } from "@/sites/${siteId}/contract";
+export { ${camel}Plugin, ${camel}Plugin as plugin } from "@/sites/${siteId}/contract";
 `;
 }
 // ── main ──────────────────────────────────────────────────────────────────────
@@ -2613,6 +2787,7 @@ async function main() {
                     s.capture.url.includes("recruitingCEJobApplicationDrafts"));
                 if (draftPostStep && !draftPostStep.produces.some((p) => p.name === "draftId")) {
                     draftPostStep.produces.push({
+                        kind: "body",
                         name: "draftId",
                         pathExpr: `${draftPostStep.varName}.APPDraftId`,
                         path: ["APPDraftId"],
@@ -2621,6 +2796,7 @@ async function main() {
                 const attachPostStep = actionSteps.find((s) => s.capture.method === "POST" && s.capture.url.includes("/attachments"));
                 if (attachPostStep && !attachPostStep.produces.some((p) => p.name === "attachmentId")) {
                     attachPostStep.produces.push({
+                        kind: "body",
                         name: "attachmentId",
                         pathExpr: `${attachPostStep.varName}.Id`,
                         path: ["Id"],
@@ -2669,6 +2845,7 @@ async function main() {
         ? emitMultiStepExecuteHttp(actionSteps, inputBody, errorSignals, fieldNameMap, discoveredFormFields, fieldOptionsMap, discoveredOptionFields, discoveredRawOptionFields, discoveredAdditionalBodyKeys, baseUrl, baseUrlDerivedHeaders, tenantSubdomainHeaders, base64PatchOverride)
         : multiStepBody;
     const hasMultipartStep = actionSteps.some((s) => s.isMultipart);
+    const headerBindings = collectHeaderBindings(actionSteps);
     // For submission flows the final action's response body is the most useful
     // shape inference target (it's the terminal success signal). Fall back to
     // the replay body for single-endpoint sites.
@@ -2726,6 +2903,7 @@ async function main() {
         discoveredRawOptionFields,
         discoveredAdditionalBodyKeys,
         payloadFieldNames: browserFlow.payloadFieldNames,
+        headerBindings,
     }));
     logger.info(`wrote ${outDir}/contract.ts`);
     (0, node_fs_1.writeFileSync)(`${outDir}/flows/browser-flow.ts`, browserFlow.code);