@enricai/barnacle 1.12.7 → 1.12.8

package/dist/scripts/recon-generate.js

@@ -31,6 +31,7 @@ exports.selectReturnAction = selectReturnAction;
 exports.selectEffectiveResponseBody = selectEffectiveResponseBody;
 exports.extractEntryUrlParams = extractEntryUrlParams;
 exports.selectPrimaryGraphQLOperation = selectPrimaryGraphQLOperation;
+exports.firstEndpointPath = firstEndpointPath;
 exports.resolveManifestActionSequence = resolveManifestActionSequence;
 exports.extractActionSequence = extractActionSequence;
 exports.extractGraphQLActionSequence = extractGraphQLActionSequence;
@@ -631,10 +632,18 @@ function selectPrimaryGraphQLOperation(captures, flowSteps, vocabulary) {
     return { capture: winner.capture, endpointPath };
 }
 function firstEndpointPath(captures) {
+    const nonGetCaptures = captures.filter((c) => c.method !== "GET");
+    for (const c of nonGetCaptures) {
+        try {
+            return new URL(c.url).pathname;
+        }
+        catch {
+            // skip
+        }
+    }
     for (const c of captures) {
         try {
-            const u = new URL(c.url);
-            return u.pathname;
+            return new URL(c.url).pathname;
         }
         catch {
             // skip
@@ -3280,7 +3289,7 @@ function renderGqlVariablesExpr(variables, payloadFieldNames) {
     return entries.length > 0 ? `{ ${entries.join(", ")} }` : "{}";
 }
 function emitContractTs(opts) {
-    const { siteId, pascal, baseUrl, baseHeaders, minTime, safeRps, responseBody, gql, gqlQuery, endpointPath, gqlOperationName, gqlVariables, auxFiles, multiStepBody, inputBody, hasMultipartStep = false, discoveredFormFields, fieldOptionsMap, discoveredOptionFields, discoveredRawOptionFields, discoveredAdditionalBodyKeys, discoveredStructuredKeys, payloadFieldNames, headerBindings = [], } = opts;
+    const { siteId, pascal, baseUrl, baseHeaders, minTime, safeRps, responseBody, gql, gqlQuery, endpointPath, gqlOperationName, gqlVariables, auxFiles, multiStepBody, omitExecuteHttp = false, inputBody, hasMultipartStep = false, discoveredFormFields, fieldOptionsMap, discoveredOptionFields, discoveredRawOptionFields, discoveredAdditionalBodyKeys, discoveredStructuredKeys, payloadFieldNames, headerBindings = [], } = opts;
     // This is the CLIENT-level schema — createHttpClient's default, and the
     // plugin's caller-facing contract (what executeHttp's return value promises
     // its own caller). It does NOT validate any individual call in a multi-step
@@ -3300,7 +3309,9 @@ function emitContractTs(opts) {
     // without affecting per-call validation. Single-endpoint plugins keep the
     // inferred schema, since there both roles (client default and sole call)
     // coincide.
-    const responseSchemaExpr = multiStepBody ? `z.unknown()` : inferZodSchema(responseBody);
+    // Browser-flow-only plugins have no HTTP call to infer a shape from either
+    // — same z.unknown() treatment as the multi-step case, for the same reason.
+    const responseSchemaExpr = multiStepBody || omitExecuteHttp ? `z.unknown()` : inferZodSchema(responseBody);
     // Multi-step flows that include a multipart upload need the binary asset
     // on the payload. ApplicantContactSchema (via ApplicantResumeSchema) already
     // declares Resume/ResumeContentType/ResumeFilename, so submission flows
@@ -3485,14 +3496,22 @@ function emitContractTs(opts) {
         .map(([k, v]) => `  ${isValidJsIdentifier(k) ? k : JSON.stringify(k)}: ${JSON.stringify(v)}`)
         .join(",\n");
     const fixtureImport = auxFiles.length > 0 ? `// import { loadFixture } from "${ENGINE_PKG}/scraper/fixtures";\n` : "";
-    const clientImport = gql
-        ? `import { createGraphqlClient } from "${ENGINE_PKG}/scraper/graphql-client";`
-        : `import { createHttpClient } from "${ENGINE_PKG}/scraper/http-client";`;
-    const queryConst = gql && gqlQuery
+    // Browser-flow-only plugins have no direct-HTTP hot path, so none of the
+    // client-construction machinery below (import, rate limiter, cache/client
+    // const) is emitted — it would sit unreferenced and trip Biome's
+    // `noUnusedVariables`.
+    const clientImport = omitExecuteHttp
+        ? ""
+        : gql
+            ? `import { createGraphqlClient } from "${ENGINE_PKG}/scraper/graphql-client";`
+            : `import { createHttpClient } from "${ENGINE_PKG}/scraper/http-client";`;
+    const queryConst = !omitExecuteHttp && gql && gqlQuery
         ? `\n// Lifted verbatim from recon capture — trim UI-only fields before shipping.\nconst ${pascal.toUpperCase()}_QUERY = \`${gqlQuery.trim()}\`;\n`
         : "";
-    const gqlCacheBlock = gql
-        ? `
+    const gqlCacheBlock = omitExecuteHttp
+        ? ""
+        : gql
+            ? `
 type GqlFn = (operationName: string, query: string, variables: Record<string, unknown>) => Promise<${pascal}Response>;
 
 const gqlCache = new Map<string, GqlFn>();
@@ -3511,7 +3530,7 @@ function getGql(baseUrl: string): GqlFn {
   return client;
 }
 `
-        : `
+            : `
 const httpClient = createHttpClient({ schema: ${pascal}ResponseSchema, bottleneck: limiter, baseHeaders: BASE_HEADERS${bindOptionLiteral(headerBindings)} });
 `;
     const gqlOperationNameExpr = gqlOperationName
@@ -3556,7 +3575,7 @@ const httpClient = createHttpClient({ schema: ${pascal}ResponseSchema, bottlenec
 export const ${pascal}InternalRequestReference = ${internalRequestReferenceExpr};
 `
         : "";
-    const queryChecklistLine = gql
+    const queryChecklistLine = !omitExecuteHttp && gql
         ? `\n *   [ ] Trim UI-only fields from ${pascal.toUpperCase()}_QUERY (keep only fields you need)`
         : "";
     // Multi-step flows validate each call against its own per-call inferred
@@ -3564,37 +3583,73 @@ export const ${pascal}InternalRequestReference = ${internalRequestReferenceExpr}
     // what executeHttp promises ITS OWN caller, never a per-call validator, so
     // the checklist item must say that explicitly. Single-endpoint plugins have
     // exactly one call, so the client schema and that call's validator are the
-    // same schema and the shorter wording stays accurate.
-    const narrowSchemaChecklistLine = multiStepBody
-        ? `\n *   [ ] Narrow ${pascal}ResponseSchema to match what executeHttp should promise ITS CALLER — this is the plugin's own return-value contract, not a per-call validator (each call in the flow is already checked against its own inferred schema)`
-        : `\n *   [ ] Narrow ${pascal}ResponseSchema to match the real response shape`;
+    // same schema and the shorter wording stays accurate. Browser-flow-only
+    // plugins have no executeHttp at all, so ResponseSchema is only ever the
+    // browser flow's own return-value contract.
+    const narrowSchemaChecklistLine = omitExecuteHttp
+        ? `\n *   [ ] Narrow ${pascal}ResponseSchema to match what the browser flow should promise ITS CALLER — this flow could not synthesize a trustworthy executeHttp (a required value from the captured sequence never resolved), so it ships browser-only`
+        : multiStepBody
+            ? `\n *   [ ] Narrow ${pascal}ResponseSchema to match what executeHttp should promise ITS CALLER — this is the plugin's own return-value contract, not a per-call validator (each call in the flow is already checked against its own inferred schema)`
+            : `\n *   [ ] Narrow ${pascal}ResponseSchema to match the real response shape`;
     const camel = siteId.replace(/-([a-z])/g, (_, c) => c.toUpperCase());
+    // Browser-flow-only plugins need neither Bottleneck (no rate-limited HTTP
+    // client) nor BASE_HEADERS (no per-call headers to bake in) — both would
+    // sit unreferenced and trip Biome's `noUnusedVariables`.
+    const bottleneckImport = omitExecuteHttp ? "" : `import Bottleneck from "bottleneck";\n`;
+    const baseHeadersBlock = omitExecuteHttp
+        ? ""
+        : `
+const BASE_HEADERS: Record<string, string> = {
+${headersLiteral},
+};
+`;
+    const limiterBlock = omitExecuteHttp
+        ? ""
+        : `
+// Safe ceiling: ${safeRps} rps — from recon rate-limit probe.
+const limiter = new Bottleneck({ minTime: ${minTime} });
+`;
+    const executeHttpMethodBlock = omitExecuteHttp
+        ? ""
+        : `
+  /** Hot path: direct HTTP — no browser, no LLM tokens. */
+  async executeHttp(
+    payload: ${pascal}Payload,
+    ${executeHttpBody.includes("context.") ? "context" : "_context"}: SitePluginContext
+  ): Promise<SitePluginResult<${pascal}Response>> {
+${executeHttpBody}
+  },
+`;
+    const pluginDocComment = omitExecuteHttp
+        ? `/**
+ * Plugin for ${siteId}. Browser-flow-only: the captured multi-step submission
+ * sequence could not be synthesized into a trustworthy direct-HTTP hot path
+ * (see the checklist above), so this always runs via Stagehand.
+ */`
+        : `/**
+ * Plugin for ${siteId}. Tries the direct-HTTP hot path first; falls back to
+ * Stagehand automatically on schema drift or bot challenge.
+ */`;
+    const baseHeadersChecklistLine = omitExecuteHttp
+        ? ""
+        : `\n *   [ ] Verify BASE_HEADERS — remove any that aren't load-bearing`;
+    const outOfTreeChecklistLine = omitExecuteHttp
+        ? `\n *   [ ] Out-of-tree: \`pnpm add zod\` — this file imports it directly, and a\n *       strict node_modules layout (pnpm) won't resolve it as a transitive\n *       dep of @enricai/barnacle alone`
+        : `\n *   [ ] Out-of-tree: \`pnpm add bottleneck zod\` — this file imports both\n *       directly, and a strict node_modules layout (pnpm) won't resolve\n *       them as transitive deps of @enricai/barnacle alone`;
     return `/**
  * Generated by recon-generate.ts — review before shipping.
  *
  * Checklist:${queryChecklistLine}${narrowSchemaChecklistLine}
- *   [ ] Adjust ${pascal}PayloadSchema to your actual request parameters
- *   [ ] Verify BASE_HEADERS — remove any that aren't load-bearing
- *   [ ] Out-of-tree: \`pnpm add bottleneck zod\` — this file imports both
- *       directly, and a strict node_modules layout (pnpm) won't resolve
- *       them as transitive deps of @enricai/barnacle alone
+ *   [ ] Adjust ${pascal}PayloadSchema to your actual request parameters${baseHeadersChecklistLine}${outOfTreeChecklistLine}
  */
 
-import Bottleneck from "bottleneck";
-import { z } from "zod/v4";
+${bottleneckImport}import { z } from "zod/v4";
 
 ${fixtureImport}${applicantContactImport}${caseInsensitiveHeadersImport}${multipartBoolImport}${clientImport}
 import type { BrowserSession } from "${ENGINE_PKG}/scraper/session";
 import type { SitePlugin, SitePluginContext, SitePluginResult } from "${ENGINE_PKG}/site-plugin";
 import { run${pascal}BrowserFlow } from "@/sites/${siteId}/flows/browser-flow";
-
-const BASE_HEADERS: Record<string, string> = {
-${headersLiteral},
-};
-
-// Safe ceiling: ${safeRps} rps — from recon rate-limit probe.
-const limiter = new Bottleneck({ minTime: ${minTime} });
-
+${baseHeadersBlock}${limiterBlock}
 const ${pascal}ResponseSchema = ${responseSchemaExpr};
 
 export type ${pascal}Response = z.infer<typeof ${pascal}ResponseSchema>;
@@ -3605,10 +3660,7 @@ const ${pascal}PayloadSchema = ${payloadSchemaExpr};
 
 export type ${pascal}Payload = z.infer<typeof ${pascal}PayloadSchema>;
 ${internalRequestReferenceBlock}${queryConst}${gqlCacheBlock}${fixtureComments}
-/**
- * Plugin for ${siteId}. Tries the direct-HTTP hot path first; falls back to
- * Stagehand automatically on schema drift or bot challenge.
- */
+${pluginDocComment}
 export const ${camel}Plugin: SitePlugin<${pascal}Payload, ${pascal}Response> = {
   meta: {
     siteId: ${JSON.stringify(siteId)},
@@ -3626,15 +3678,7 @@ export const ${camel}Plugin: SitePlugin<${pascal}Payload, ${pascal}Response> = {
     // encoding parseable.
     apiVersion: ${JSON.stringify(plugin_api_version_1.PLUGIN_API_VERSION)},${payloadNeedsMultipart || inputBody ? "\n    multipart: true," : ""}
   },
-
-  /** Hot path: direct HTTP — no browser, no LLM tokens. */
-  async executeHttp(
-    payload: ${pascal}Payload,
-    ${executeHttpBody.includes("context.") ? "context" : "_context"}: SitePluginContext
-  ): Promise<SitePluginResult<${pascal}Response>> {
-${executeHttpBody}
-  },
-
+${executeHttpMethodBlock}
   /** Browser fallback: Stagehand + Steel — invoked only when hot path fails. */
   async execute(
     payload: ${pascal}Payload,
@@ -4150,15 +4194,60 @@ async function main() {
     // Selection precedence: (A) the authoritative submit-manifest recon-browser
     // wrote from the verified submission; else (B/C) pattern/heuristic extraction.
     // The manifest is the only signal that separates a submission POST from a
-    // page-chrome POST sharing its URL, so it wins when present.
+    // page-chrome POST sharing its URL, so it normally wins when present — but
+    // a manifest built from a single flow-declared submit step cannot represent
+    // a wizard whose every section saves independently, so it is only trusted
+    // when it isn't a strict undercount of what the same captures' own
+    // heuristic extraction finds.
+    const patternedHeuristicActionCaptures = gql
+        ? graphqlActionSequence
+        : collapseRedundantPatches(extractActionSequence(captures, baseUrl, submitPatterns));
+    // The same undercount hazard applies one layer below the manifest: a
+    // flow-declared submitEndpointPattern that matches only one section's URL
+    // (the natural way to describe "the button that finishes the wizard")
+    // filters the heuristic sequence down to that one capture even though the
+    // same captures, read without the pattern, show every section saving
+    // independently. A pattern that undercounts what the unfiltered heuristic
+    // finds is therefore not trusted either — it falls back to the richer,
+    // unfiltered sequence instead of collapsing a real multi-call flow into
+    // the generic single-endpoint fallback.
+    const unfilteredHeuristicActionCaptures = submitPatterns.endpoint === null && submitPatterns.body === null
+        ? patternedHeuristicActionCaptures
+        : gql
+            ? extractGraphQLActionSequence(captures, baseUrl, null)
+            : collapseRedundantPatches(extractActionSequence(captures, baseUrl, null));
+    const patternUndercounts = patternedHeuristicActionCaptures.length < unfilteredHeuristicActionCaptures.length;
+    if (patternUndercounts) {
+        logger.info(`submission selection: ignoring submitEndpointPattern/submitBodyPattern (${patternedHeuristicActionCaptures.length} capture(s)) as an undercount of the unfiltered heuristic action sequence (${unfilteredHeuristicActionCaptures.length} capture(s))`);
+    }
+    const heuristicActionCaptures = patternUndercounts
+        ? unfilteredHeuristicActionCaptures
+        : patternedHeuristicActionCaptures;
     const manifestActionCaptures = resolveManifestActionSequence(runRoot, captures);
-    if (manifestActionCaptures !== null) {
+    const manifestUndercounts = manifestActionCaptures !== null &&
+        manifestActionCaptures.length < heuristicActionCaptures.length;
+    if (manifestActionCaptures !== null && manifestUndercounts) {
+        logger.info(`submission selection: ignoring submit-manifest.json (${manifestActionCaptures.length} capture(s)) as an undercount of the heuristic action sequence (${heuristicActionCaptures.length} capture(s))`);
+    }
+    else if (manifestActionCaptures !== null) {
         logger.info(`submission selection: using submit-manifest.json (${manifestActionCaptures.length} authoritative capture(s))`);
     }
-    const rawActionCaptures = manifestActionCaptures ??
-        (gql
-            ? graphqlActionSequence
-            : collapseRedundantPatches(extractActionSequence(captures, baseUrl, submitPatterns)));
+    const rawActionCaptures = manifestActionCaptures !== null && !manifestUndercounts
+        ? manifestActionCaptures
+        : heuristicActionCaptures;
+    // The declared submitEndpointPattern's own under-match: unlike manifestUndercounts
+    // (which compares an authoritative submit-manifest.json against the heuristic
+    // sequence), this compares the pattern-filtered heuristic sequence against the SAME
+    // captures with no submitPatterns filter at all — the true raw same-host non-GET 2xx
+    // action set. A pattern that matches conspicuously fewer calls than that raw set is a
+    // detection failure (the pattern is too narrow), not evidence the flow is read-only,
+    // and per the no-silent-fallback rule must not be allowed to quietly collapse into the
+    // generic single-endpoint {query} template below.
+    const rawSameHostActionCaptures = submitEndpointPattern === null
+        ? null
+        : gql
+            ? extractGraphQLActionSequence(captures, baseUrl, null)
+            : collapseRedundantPatches(extractActionSequence(captures, baseUrl, null));
     // Form-schema detection runs BEFORE state-indexing so the field-id/option-id
     // UUIDs can be shielded from indexing — those UUIDs are stable schema
     // anchors that T2/T3 substitution depends on remaining literal in body
@@ -4211,6 +4300,19 @@ async function main() {
         : new Map();
     const actionSteps = actionCaptures.length > 1 ? compileActionSteps(actionCaptures, stateIndex) : [];
     const isSubmissionFlow = actionSteps.length > 1;
+    // Loud failure for a submitEndpointPattern that under-matches the raw traffic badly
+    // enough to collapse the flow to the single-endpoint fallback: heuristicActionCaptures
+    // is the pattern-filtered sequence (used both directly and as rawActionCaptures' floor
+    // via manifestUndercounts above), so if it's this thin ONLY because the pattern itself
+    // excluded real action captures the unfiltered raw set still has, emitting the generic
+    // {query} template would misrepresent a genuine multi-call flow as read-only. Exit
+    // rather than degrade quietly, per the no-defensive/no-silent-fallback rule.
+    if (rawSameHostActionCaptures !== null &&
+        !isSubmissionFlow &&
+        rawSameHostActionCaptures.length > heuristicActionCaptures.length) {
+        logger.error(`ERROR declared submitEndpointPattern ${JSON.stringify(submitEndpointPattern)} matched only ${heuristicActionCaptures.length} of ${rawSameHostActionCaptures.length} raw same-host non-GET 2xx action capture(s) — this under-match looks like a detection failure, not a read-only flow; refusing to fall back to the generic single-endpoint {query} template. Fix submitEndpointPattern in recon-flow.json to cover the real submission calls.`);
+        process.exit(1);
+    }
     // Diagnostic for the FAILURE-3 shape (a flowless recon capture): a "submission flow"
     // whose every action capture is landing-phase is almost certainly page-chrome
     // bootstrap (e.g. page-chrome `POST /widgets`) misread as an apply flow, not a walked
@@ -4287,15 +4389,33 @@ async function main() {
             staticBaseHeaders[k] = v;
         }
     }
-    const multiStepBody = isSubmissionFlow
-        ? emitMultiStepExecuteHttp(actionSteps, inputBody, errorSignals, fieldNameMap, discoveredFormFields, fieldOptionsMap, discoveredOptionFields, discoveredRawOptionFields, discoveredAdditionalBodyKeys, baseUrl, baseUrlDerivedHeaders, tenantSubdomainHeaders, formSchema, personaBindings, entryUrlParams, shieldedUuids, selectResolutions, discoveredStructuredKeys, rawCodeFields)
-        : undefined;
+    // Third branch (browser-flow-only): a multi-action flow (isSubmissionFlow)
+    // that crosses hosts mid-sequence (compileActionSteps' isCrossDomain — a
+    // captured redirect off the original domain, e.g. an auth bounce or a
+    // vendor-hosted submission step). A bare `fetch`-based executeHttp can't
+    // reliably replay that: cookies/CSRF/session state minted for one origin
+    // don't automatically carry to the next the way a real browser's redirect
+    // handling does, so synthesizing a same-shape HTTP sequence would silently
+    // drop the session boundary the recon actually walked. Per-step, this
+    // already surfaces as the "cross-domain redirect detected ... likely needs
+    // browser fallback for this step" TODO (see emitMultiStepExecuteHttp); at
+    // the whole-flow level the honest emit is no executeHttp at all — never a
+    // same-host multi-step body that quietly drops the hop, and never a
+    // downgrade to the single-endpoint `{query}` branch either, since that's a
+    // fabrication of its own kind for a flow that isn't a single-action
+    // query/search to begin with.
+    const browserFlowOnly = isSubmissionFlow && actionSteps.some((s) => s.isCrossDomain);
+    const multiStepBody = browserFlowOnly
+        ? undefined
+        : isSubmissionFlow
+            ? emitMultiStepExecuteHttp(actionSteps, inputBody, errorSignals, fieldNameMap, discoveredFormFields, fieldOptionsMap, discoveredOptionFields, discoveredRawOptionFields, discoveredAdditionalBodyKeys, baseUrl, baseUrlDerivedHeaders, tenantSubdomainHeaders, formSchema, personaBindings, entryUrlParams, shieldedUuids, selectResolutions, discoveredStructuredKeys, rawCodeFields)
+            : undefined;
     const hasMultipartStep = actionSteps.some((s) => s.isMultipart);
     const headerBindings = collectHeaderBindings(actionSteps);
     // Shape inference targets the SAME call executeHttp returns — see
     // selectEffectiveResponseBody — so the two surfaces can't describe different calls.
     const effectiveResponseBody = selectEffectiveResponseBody(isSubmissionFlow, actionSteps, responseBody);
-    logger.info(`generating plugin for ${siteId} (${gql ? "GraphQL" : isSubmissionFlow ? `submission flow, ${actionSteps.length} steps` : "single-endpoint REST"}, baseUrl: ${baseUrl})`);
+    logger.info(`generating plugin for ${siteId} (${gql ? "GraphQL" : browserFlowOnly ? `submission flow, ${actionSteps.length} steps, browser-flow-only (cross-domain hop detected)` : isSubmissionFlow ? `submission flow, ${actionSteps.length} steps` : "single-endpoint REST"}, baseUrl: ${baseUrl})`);
     if (emit === "config") {
         (0, node_fs_1.mkdirSync)(outDir, { recursive: true });
         (0, node_fs_1.writeFileSync)(manifestPath, emitConfigManifest({
@@ -4309,7 +4429,9 @@ async function main() {
             // A submission flow is the case where the `.ts` emit carries an
             // executeHttp hot path; point the manifest at where the operator drops
             // the compiled module rather than silently dropping the direct path.
-            httpModulePath: isSubmissionFlow ? `./${siteId}.http.js` : undefined,
+            // Not set for the browser-flow-only branch — there is no hot path to
+            // compile a module for.
+            httpModulePath: isSubmissionFlow && !browserFlowOnly ? `./${siteId}.http.js` : undefined,
         }));
         logger.info(`wrote ${manifestPath}`);
         logger.info(`done — review ${manifestPath}, fill in response/extract schemas, then load via BARNACLE_PLUGINS or BARNACLE_PLUGINS_CONFIG_DIR (no compile step)`);
@@ -4346,6 +4468,7 @@ async function main() {
         gqlVariables: primaryGraphQLOperation?.capture.variables ?? null,
         auxFiles,
         multiStepBody,
+        omitExecuteHttp: browserFlowOnly,
         inputBody,
         hasMultipartStep,
         discoveredFormFields,