@enricai/barnacle 1.12.44 → 1.12.46

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -71,10 +71,12 @@ exports.resolveFoldPlan = resolveFoldPlan;
71
71
  exports.resolveApplicableFoldPlans = resolveApplicableFoldPlans;
72
72
  exports.buildContractChecklist = buildContractChecklist;
73
73
  exports.emitContractTs = emitContractTs;
74
+ exports.unreferencedRequiredUrlFields = unreferencedRequiredUrlFields;
74
75
  exports.assertRequiredUrlFieldsReferenced = assertRequiredUrlFieldsReferenced;
75
76
  exports.emitConfigManifest = emitConfigManifest;
76
77
  exports.emitBrowserFlowTs = emitBrowserFlowTs;
77
78
  exports.emitIndexTs = emitIndexTs;
79
+ exports.identifyNoiseCapturesForFields = identifyNoiseCapturesForFields;
78
80
  const node_fs_1 = require("node:fs");
79
81
  const node_path_1 = require("node:path");
80
82
  const ats_field_vocabulary_1 = require("../lib/ats-field-vocabulary");
@@ -966,6 +968,19 @@ function captureHostname(url) {
966
968
  return "";
967
969
  }
968
970
  }
971
+ /**
972
+ * Same not-guaranteed-parseable caveat as {@link captureHostname}, for the
973
+ * path instead of the host — used to anchor {@link isStructurallyRelevantCapture}
974
+ * on a capture's URL structure.
975
+ */
976
+ function capturePathname(url) {
977
+ try {
978
+ return new URL(url).pathname;
979
+ }
980
+ catch {
981
+ return "";
982
+ }
983
+ }
969
984
  /**
970
985
  * `baseUrl` itself is what {@link registrableDomain}'s fallback would be
971
986
  * derived FROM, so at this point in generation the registrable-domain gate
@@ -1507,7 +1522,6 @@ function resolveManifestActionSequence(runRoot, captures) {
1507
1522
  * chatter and the emitted hot path.
1508
1523
  */
1509
1524
  function extractActionSequence(captures, submitPatterns = null, foldReturnSpec = null, ownBackendHostnames = [], fallbackDomain = null) {
1510
- const matchesSubmit = compileSubmitMatcher(submitPatterns);
1511
1525
  const matchesFoldReturn = compileFoldReturnEndpointMatcher(foldReturnSpec);
1512
1526
  // Callers with no host-provenance data (the exported function's unit
1513
1527
  // tests) pass neither ownBackendHostnames nor fallbackDomain — in that
@@ -1515,7 +1529,21 @@ function extractActionSequence(captures, submitPatterns = null, foldReturnSpec =
1515
1529
  // only applies once the caller has actually resolved a notion of "own
1516
1530
  // backend" to check against.
1517
1531
  const hasHostProvenance = ownBackendHostnames.length > 0 || fallbackDomain !== null;
1518
- return captures
1532
+ // With host-provenance data AND a declared submitEndpointPattern, the
1533
+ // endpoint match's job of isolating the submission is taken over by the
1534
+ // structural-relevance narrowing pass below (anchored on that SAME
1535
+ // pattern) instead of the per-capture endpoint regex test: a literal
1536
+ // per-capture match can't admit an EARLIER multi-step chain capture (auth
1537
+ // mint, paged listing, ...) whose URL never matches the terminal submit
1538
+ // endpoint, only a LATER structurally-related one. `submitBodyPattern`
1539
+ // stays enforced either way — it isn't what structural relevance reasons
1540
+ // about. Absent host provenance, or absent a declared endpoint pattern,
1541
+ // the per-capture endpoint match below is the only submit-isolation
1542
+ // mechanism and stays unconditional (matches the "submit patterns isolate
1543
+ // the submission from same-URL chrome" behavior).
1544
+ const hasSubmitEndpointAnchor = hasHostProvenance && submitPatterns !== null && submitPatterns.endpoint !== null;
1545
+ const matchesSubmit = compileSubmitMatcher(hasSubmitEndpointAnchor ? { endpoint: null, body: submitPatterns.body } : submitPatterns);
1546
+ const hostGated = captures
1519
1547
  .map((capture, index) => ({ capture, index }))
1520
1548
  .filter(({ capture }) => {
1521
1549
  if (capture.method === "GET" && !matchesFoldReturn(capture))
@@ -1531,6 +1559,46 @@ function extractActionSequence(captures, submitPatterns = null, foldReturnSpec =
1531
1559
  return false;
1532
1560
  return true;
1533
1561
  });
1562
+ // Host-gating alone can't tell a genuinely related own-backend endpoint
1563
+ // family apart from a same-host capture that shares nothing but the host
1564
+ // (e.g. a marketing/promotions endpoint fired incidentally by a page load,
1565
+ // admitted above because it too is a 2xx own-backend POST). Drop a
1566
+ // structurally-isolated member of the pool first — a compound-path capture
1567
+ // (e.g. `/site-banner`) sharing no token with anything else host-gated in,
1568
+ // regardless of whether a submitEndpointPattern is declared. Guarded to
1569
+ // pools of 3+: a real chain's own steps are usually plain single-word
1570
+ // paths with no tokens of their own (never flagged, see
1571
+ // isStructurallyIsolatedCapture), so this only ever removes a capture that
1572
+ // is BOTH compound-path AND unrelated to every other admitted capture —
1573
+ // but a 1-2 capture pool has no "everything else" to be isolated from, so
1574
+ // skip it there rather than risk flagging a single-endpoint site's own
1575
+ // hyphenated path.
1576
+ const structurallyGated = hasHostProvenance && hostGated.length > 2
1577
+ ? hostGated.filter(({ capture }, i) => {
1578
+ const path = safeUrlPathname(capture.url);
1579
+ const otherPaths = hostGated
1580
+ .filter((_, j) => j !== i)
1581
+ .map((h) => safeUrlPathname(h.capture.url));
1582
+ return !(0, capture_filters_1.isStructurallyIsolatedCapture)(path, otherPaths);
1583
+ })
1584
+ : hostGated;
1585
+ // Narrow further using structural relevance, anchored on whichever
1586
+ // captures the flow's own declared `submitEndpointPattern` matches — the
1587
+ // report's own preferred anchor, and the only signal here with no false
1588
+ // positives by construction. No declared endpoint pattern gives no
1589
+ // authoritative reference, so this pass is a no-op — mirroring
1590
+ // compileSubmitMatcher's own null-pattern passthrough — rather than
1591
+ // guessing at a reference set.
1592
+ if (!hasSubmitEndpointAnchor)
1593
+ return structurallyGated;
1594
+ const endpointRx = new RegExp(submitPatterns.endpoint);
1595
+ const referencePaths = structurallyGated
1596
+ .filter(({ capture }) => endpointRx.test(capture.url))
1597
+ .map(({ capture }) => safeUrlPathname(capture.url));
1598
+ if (referencePaths.length === 0)
1599
+ return structurallyGated;
1600
+ return structurallyGated.filter(({ capture }) => endpointRx.test(capture.url) ||
1601
+ (0, capture_filters_1.isStructurallyRelevantCapture)(safeUrlPathname(capture.url), referencePaths));
1534
1602
  }
1535
1603
  /**
1536
1604
  * GraphQL-aware analog of {@link extractActionSequence}: REST's URL/method
@@ -1555,6 +1623,18 @@ function extractActionSequence(captures, submitPatterns = null, foldReturnSpec =
1555
1623
  * `resolveFoldPlan` with no primary capture to resolve `resultsPath`
1556
1624
  * against. Every other non-mutation capture is still dropped.
1557
1625
  *
1626
+ * The fold-return matchers above only look at a capture's URL/response
1627
+ * shape, not at whether it belongs to this flow's own endpoint family — a
1628
+ * same-host marketing query can pass them purely by shape coincidence, the
1629
+ * same gap {@link extractActionSequence} closes with
1630
+ * {@link isStructurallyRelevantCapture}. So once host-provenance data is
1631
+ * available, a non-mutation capture admitted only via the fold-return
1632
+ * matchers is kept only when its path shares structural tokens with at
1633
+ * least one admitted mutation's path — mutations are always genuine flow
1634
+ * steps, so they anchor what "this flow's family" means. When no mutation
1635
+ * was admitted there is nothing to anchor against, so the fold-return
1636
+ * admissions above stand unnarrowed.
1637
+ *
1558
1638
  * Exported for tests: this predicate decides what a generated GraphQL plugin
1559
1639
  * will send at a live site.
1560
1640
  */
@@ -1568,7 +1648,8 @@ function extractGraphQLActionSequence(captures, submitPatterns = null, foldRetur
1568
1648
  // only applies once the caller has actually resolved a notion of "own
1569
1649
  // backend" to check against.
1570
1650
  const hasHostProvenance = ownBackendHostnames.length > 0 || fallbackDomain !== null;
1571
- return captures
1651
+ const isMutation = (capture) => capture.query !== null && /^\s*mutation\b/.test(capture.query);
1652
+ const admitted = captures
1572
1653
  .map((capture, index) => ({ capture, index }))
1573
1654
  .filter(({ capture }) => {
1574
1655
  if (capture.status < 200 || capture.status >= 300)
@@ -1580,10 +1661,19 @@ function extractGraphQLActionSequence(captures, submitPatterns = null, foldRetur
1580
1661
  if (hasHostProvenance &&
1581
1662
  !(0, capture_filters_1.isAllowedFixtureHost)(captureHostname(capture.url), ownBackendHostnames, fallbackDomain))
1582
1663
  return false;
1583
- if (capture.query !== null && /^\s*mutation\b/.test(capture.query))
1664
+ if (isMutation(capture))
1584
1665
  return true;
1585
1666
  return matchesFoldReturn(capture) || matchesFoldReturnResults(capture);
1586
1667
  });
1668
+ if (!hasHostProvenance)
1669
+ return admitted;
1670
+ const mutationPaths = admitted
1671
+ .filter(({ capture }) => isMutation(capture))
1672
+ .map(({ capture }) => capturePathname(capture.url));
1673
+ if (mutationPaths.length === 0)
1674
+ return admitted;
1675
+ return admitted.filter(({ capture }) => isMutation(capture) ||
1676
+ (0, capture_filters_1.isStructurallyRelevantCapture)(capturePathname(capture.url), mutationPaths));
1587
1677
  }
1588
1678
  /**
1589
1679
  * Identity of a capture's response shape for dedup purposes: the endpoint it
@@ -7078,19 +7168,46 @@ function emitContractTs(opts) {
7078
7168
  //
7079
7169
  // The captured request body (inputBody) is the SITE's internal request
7080
7170
  // shape (a vendor's ddoKey/formData, a GraphQL worklet's variables, …) — not
7081
- // what the real caller sends. The plugin's buildBarnacleFormData posts
7082
- // the standard candidate payload (ApplicantContactSchema's identity/
7083
- // address/resume fields + Email + job-targeting + a JSON Answers block) to
7084
- // every plugin's /run, so that — not a structural inference over
7085
- // inputBody — is the public contract every submission-flow plugin must
7086
- // declare, unconditionally (see recon-generate-payload-schema-mismatch.md
7087
- // fix option (a)). inputBody remains available to the plugin author as the
7088
- // internal request shape the site's own call needs to be built from; it no
7089
- // longer drives the public schema. A missing inputBody means this is a
7090
- // non-submission (query-type) flow, which keeps its own contract untouched.
7091
- const basePayloadSchemaExpr = inputBody
7171
+ // necessarily the real caller's shape. ApplicantContactSchema (the job-
7172
+ // application template: identity/address/resume fields + Email + a JSON
7173
+ // Answers block) only belongs on flows that are actually job/benefits
7174
+ // applications — selecting it purely because SOME body was captured picks
7175
+ // it for unrelated submission flows too. usesApplicantContactSchema is the
7176
+ // real-evidence gate: it requires the captured inputBody to itself carry
7177
+ // one of ApplicantContactSchema's own field names (case-insensitively,
7178
+ // checked at any depth via walkAllPrimitiveLeaves) before the template
7179
+ // applies (see recon-generate-payload-schema-mismatch.md).
7180
+ const applicantContactFieldNames = new Set([
7181
+ "FirstName",
7182
+ "LastName",
7183
+ "Phone",
7184
+ "AddressLine",
7185
+ "City",
7186
+ "State",
7187
+ "PostalCode",
7188
+ "Country",
7189
+ "County",
7190
+ "Resume",
7191
+ "ResumeContentType",
7192
+ "ResumeFilename",
7193
+ "ResumeBase64",
7194
+ ]);
7195
+ const baseContractFieldNames = new Set(["Email", "ClickUrl", "Answers"]);
7196
+ const applicantContactEvidenceFieldNamesLower = new Set([...applicantContactFieldNames, ...baseContractFieldNames].map((name) => name.toLowerCase()));
7197
+ const usesApplicantContactSchema = inputBody != null &&
7198
+ [...walkAllPrimitiveLeaves(inputBody)].some(({ path }) => path.some((segment) => applicantContactEvidenceFieldNamesLower.has(segment.toLowerCase())));
7199
+ const isReservedByApplicantContactSchema = (name) => usesApplicantContactSchema && applicantContactFieldNames.has(name);
7200
+ // A missing inputBody, or one with no ApplicantContactSchema evidence,
7201
+ // means this is not a job-application submission flow. The former keeps
7202
+ // its own read-flow `{ query }` contract untouched; the latter (a body WAS
7203
+ // captured, it just isn't an application) gets a plain empty base instead
7204
+ // — extend() below still layers on every genuinely-discovered field from
7205
+ // the other sources unchanged.
7206
+ const basePayloadSchemaExpr = usesApplicantContactSchema
7092
7207
  ? `ApplicantContactSchema`
7093
- : `z.object({\n query: z.string().min(1),\n})`;
7208
+ : inputBody
7209
+ ? `z.object({})`
7210
+ : `z.object({\n query: z.string().min(1),\n})`;
7094
7211
  // Only the single-endpoint GraphQL read path (a real primary operation, no
7095
7212
  // multi-step flow) is a candidate for a paging signal — multiStepBody
7096
7213
  // already owns its own per-call semantics.
@@ -7153,34 +7270,12 @@ function emitContractTs(opts) {
7153
7270
  if (paginationSignal) {
7154
7271
  addExtendField("maxPages", " maxPages: z.number().int().positive().optional(),");
7155
7272
  }
7156
- // The base extend's own keys — submission flows only.
7157
- if (inputBody) {
7273
+ // The base extend's own keys — job-application submission flows only.
7274
+ if (usesApplicantContactSchema) {
7158
7275
  addExtendField("Email", " Email: z.email(),");
7159
7276
  addExtendField("ClickUrl", " ClickUrl: z.string().min(1),");
7160
7277
  addExtendField("Answers", " Answers: multipartJsonObject(z.record(z.string(), z.unknown())),");
7161
7278
  }
7162
- // ApplicantContactSchema's own merged identity/address/resume field names
7163
- // (see src/lib/application-identity.ts, application-address.ts,
7164
- // application-resume.ts, applicant-payload.ts) — reserved so no discovered/
7165
- // spliced source can redeclare (and silently shadow) a field the base
7166
- // ApplicantContactSchema already supplies. Only relevant for submission
7167
- // flows, where basePayloadSchemaExpr actually is ApplicantContactSchema.
7168
- const applicantContactFieldNames = new Set([
7169
- "FirstName",
7170
- "LastName",
7171
- "Phone",
7172
- "AddressLine",
7173
- "City",
7174
- "State",
7175
- "PostalCode",
7176
- "Country",
7177
- "County",
7178
- "Resume",
7179
- "ResumeContentType",
7180
- "ResumeFilename",
7181
- "ResumeBase64",
7182
- ]);
7183
- const isReservedByApplicantContactSchema = (name) => Boolean(inputBody) && applicantContactFieldNames.has(name);
7184
7279
  // A declared foldReturn.drillParamBindings names drill query params that
7185
7280
  // are caller-driven instead of frozen literals (see
7186
7281
  // recon-generate-foldreturn-cannot-bind-drill-query-param-to-caller-payload.md)
@@ -7201,9 +7296,10 @@ function emitContractTs(opts) {
7201
7296
  addExtendField(binding.payloadField, ` ${binding.payloadField}: ${zod},`);
7202
7297
  }
7203
7298
  // Multi-step flows that include a multipart upload need the binary asset
7204
- // on the payload. A query-type flow (no ApplicantContactSchema base) still
7205
- // needs these fields spelled out explicitly.
7206
- if (hasMultipartStep && !inputBody) {
7299
+ // on the payload. A non-applicant flow (no ApplicantContactSchema base,
7300
+ // whether or not a body was captured) still needs these fields spelled
7301
+ // out explicitly.
7302
+ if (hasMultipartStep && !usesApplicantContactSchema) {
7207
7303
  addExtendField("Resume", " Resume: z.instanceof(Buffer),");
7208
7304
  addExtendField("ResumeContentType", " ResumeContentType: z.string(),");
7209
7305
  addExtendField("ResumeFilename", " ResumeFilename: z.string(),");
@@ -7345,9 +7441,8 @@ function emitContractTs(opts) {
7345
7441
  // comment above) — a GraphQL mutation that happens to declare an
7346
7442
  // unpopulated variable with a matching name (e.g. `$email`) must not
7347
7443
  // downgrade that required base field.
7348
- const baseContractFieldNames = new Set(["Email", "ClickUrl", "Answers"]);
7349
7444
  for (const [fieldName, line] of extendFields) {
7350
- if (inputBody && baseContractFieldNames.has(fieldName))
7445
+ if (usesApplicantContactSchema && baseContractFieldNames.has(fieldName))
7351
7446
  continue;
7352
7447
  if (unpopulatedDeclaredVariables.some((name) => name.toLowerCase() === fieldName.toLowerCase())) {
7353
7448
  extendFields.set(fieldName, line.replace(/,\s*$/, ".optional(),"));
@@ -7356,23 +7451,25 @@ function emitContractTs(opts) {
7356
7451
  const mergedExtension = extendFields.size > 0 ? `.extend({\n${[...extendFields.values()].join("\n")}\n})` : "";
7357
7452
  const payloadSchemaExpr = `${basePayloadSchemaExpr}${mergedExtension}`;
7358
7453
  // basePayloadSchemaExpr's own Answers field always wraps in
7359
- // multipartJsonObject() for submission flows (inputBody set);
7360
- // multipartBoolean() is only imported when a boolean field was actually
7361
- // wrapped in it above (an additional-body-key or an inputBody field under
7362
- // multipartCoerce) — payloadNeedsMultipart alone doesn't imply that.
7454
+ // multipartJsonObject() for job-application submission flows
7455
+ // (usesApplicantContactSchema); multipartBoolean() is only imported when a
7456
+ // boolean field was actually wrapped in it above (an additional-body-key
7457
+ // or an inputBody field under multipartCoerce) — payloadNeedsMultipart
7458
+ // alone doesn't imply that.
7363
7459
  // Named imports from the same module are combined into one import statement.
7364
7460
  const zodMultipartNamedImports = [
7365
7461
  ...(usesMultipartBoolean ? ["multipartBoolean"] : []),
7366
- ...(inputBody || (payloadNeedsMultipart && sortedStructuredEntries.length > 0)
7462
+ ...(usesApplicantContactSchema || (payloadNeedsMultipart && sortedStructuredEntries.length > 0)
7367
7463
  ? ["multipartJsonObject"]
7368
7464
  : []),
7369
7465
  ];
7370
7466
  const multipartBoolImport = zodMultipartNamedImports.length > 0
7371
7467
  ? `import { ${zodMultipartNamedImports.join(", ")} } from "${ENGINE_PKG}/lib/zod-multipart";\n`
7372
7468
  : "";
7373
- // ApplicantContactSchema backs the default submission-flow payload schema
7374
- // (see basePayloadSchemaExpr above); only referenced when inputBody is set.
7375
- const applicantContactImport = inputBody
7469
+ // ApplicantContactSchema backs the job-application submission-flow payload
7470
+ // schema (see basePayloadSchemaExpr above); only referenced when
7471
+ // usesApplicantContactSchema is true.
7472
+ const applicantContactImport = usesApplicantContactSchema
7376
7473
  ? `import { ApplicantContactSchema } from "${ENGINE_PKG}/lib/applicant-payload";\n`
7377
7474
  : "";
7378
7475
  // Content-Type must be absent from multipart fetch calls so FormData can inject the boundary.
@@ -7773,17 +7870,17 @@ export const ${camel}Plugin: SitePlugin<${pascal}Payload, ${pascal}Response> = {
7773
7870
  bodySchema: ${pascal}PayloadSchema,
7774
7871
  responseSchema: ${pascal}ResponseSchema,
7775
7872
  defaultBaseUrl: ${JSON.stringify(baseUrl)},
7776
- ${payloadNeedsMultipart || inputBody
7873
+ ${payloadNeedsMultipart || usesApplicantContactSchema || hasMultipartStep
7777
7874
  ? `// multipart is required whenever the flow itself uploads a file
7778
- // (hasMultipartStep), OR this is a submission flow (inputBody set) since
7779
- // basePayloadSchemaExpr always requires a real Resume Buffer via
7780
- // ApplicantContactSchema regardless of whether the recorded browser flow
7781
- // contained an upload step, OR the payload has a non-scalar
7782
- // discoveredStructuredKeys field (payloadNeedsMultipart), since the
7783
- // multipart wire format is what makes that field's JSON-stringified
7784
- // encoding parseable.
7875
+ // (hasMultipartStep), OR this is a job-application submission flow
7876
+ // (usesApplicantContactSchema) since basePayloadSchemaExpr requires a
7877
+ // real Resume Buffer via ApplicantContactSchema regardless of whether
7878
+ // the recorded browser flow contained an upload step, OR the payload
7879
+ // has a non-scalar discoveredStructuredKeys field (payloadNeedsMultipart),
7880
+ // since the multipart wire format is what makes that field's
7881
+ // JSON-stringified encoding parseable.
7785
7882
  `
7786
- : ""}apiVersion: ${JSON.stringify(plugin_api_version_1.PLUGIN_API_VERSION)},${payloadNeedsMultipart || inputBody ? "\n multipart: true," : ""}
7883
+ : ""}apiVersion: ${JSON.stringify(plugin_api_version_1.PLUGIN_API_VERSION)},${payloadNeedsMultipart || usesApplicantContactSchema || hasMultipartStep ? "\n multipart: true," : ""}
7787
7884
  },
7788
7885
  ${executeHttpMethodBlock}
7789
7886
  /** Browser fallback: Stagehand + Steel — invoked only when hot path fails. */
@@ -7792,7 +7889,7 @@ ${executeHttpMethodBlock}
7792
7889
  session: BrowserSession,
7793
7890
  context: SitePluginContext
7794
7891
  ): Promise<SitePluginResult<${pascal}Response>> {
7795
- const raw = await run${pascal}BrowserFlow(session.stagehand, ${inputBody ? "payload.ClickUrl" : "context.baseUrl"}, payload, session.sessionProxy ?? null);
7892
+ const raw = await run${pascal}BrowserFlow(session.stagehand, ${usesApplicantContactSchema ? "payload.ClickUrl" : "context.baseUrl"}, payload, session.sessionProxy ?? null);
7796
7893
  return { data: raw as ${pascal}Response };
7797
7894
  },
7798
7895
  };
@@ -7813,14 +7910,23 @@ export { ${camel}Plugin as plugin };
7813
7910
  * defect 1) — this generalizes the check past ClickUrl so any future
7814
7911
  * required URL field regresses loudly instead of silently.
7815
7912
  */
7816
- function assertRequiredUrlFieldsReferenced(contractCode, browserFlowCode) {
7913
+ /**
7914
+ * The field-name computation {@link assertRequiredUrlFieldsReferenced} guards
7915
+ * with. Exposed separately so a caller catching the guard's failure can
7916
+ * re-derive the exact offending field names to drive a narrowed retry,
7917
+ * instead of parsing them back out of the thrown error's message.
7918
+ */
7919
+ function unreferencedRequiredUrlFields(contractCode, browserFlowCode) {
7817
7920
  const urlFieldLinePattern = /^\s*(\w*Url\w*):\s*z\.\w+\(.*$/gm;
7818
7921
  const emittedCode = `${contractCode}\n${browserFlowCode}`;
7819
- const unreferenced = [...contractCode.matchAll(urlFieldLinePattern)]
7922
+ return [...contractCode.matchAll(urlFieldLinePattern)]
7820
7923
  .filter((match) => !match[0].includes(".optional("))
7821
7924
  .map((match) => match[1])
7822
7925
  .filter((name) => name !== undefined)
7823
7926
  .filter((name) => !emittedCode.includes(`payload.${name}`));
7927
+ }
7928
+ function assertRequiredUrlFieldsReferenced(contractCode, browserFlowCode) {
7929
+ const unreferenced = unreferencedRequiredUrlFields(contractCode, browserFlowCode);
7824
7930
  if (unreferenced.length === 0)
7825
7931
  return;
7826
7932
  throw new Error(`recon-generate: required URL field(s) ${unreferenced.join(", ")} declared on the payload schema ` +
@@ -8314,6 +8420,124 @@ async function resolveFormSchema(specifier) {
8314
8420
  logger.info(`form-schema: ${specifier === load_form_schema_1.FORM_SCHEMA_NONE ? "none (no ATS form recovery)" : `custom keys from ${specifier}`}`);
8315
8421
  return formSchema;
8316
8422
  }
8423
+ /**
8424
+ * The capture whose own response `assertRequiredUrlFieldsReferenced`'s
8425
+ * scan actually describes: a submission flow returns
8426
+ * {@link selectReturnAction}'s pick (the terminal/return call
8427
+ * `executeHttp` hands back); a read flow returns `winningCapture`. This is
8428
+ * the one capture the narrowing pass must never exclude — if an offending
8429
+ * field's sole source turns out to be this capture, the field is genuinely
8430
+ * required and the guard must still hard-fail.
8431
+ */
8432
+ function resolvedPrimaryResponseCapture(result) {
8433
+ if (!result.isSubmissionFlow)
8434
+ return result.winningCapture;
8435
+ return selectReturnAction(result.actionSteps)?.capture ?? null;
8436
+ }
8437
+ /** True when `capture`'s own top-level response JSON is the field's source —
8438
+ * the same "does this capture's response carry the field" signal
8439
+ * {@link assertRequiredUrlFieldsReferenced} implicitly relies on, applied per
8440
+ * capture instead of to the merged emitted code. */
8441
+ function captureOwnsTopLevelField(capture, fieldName) {
8442
+ const body = capture.responseBody;
8443
+ if (typeof body !== "object" || body === null || Array.isArray(body))
8444
+ return false;
8445
+ return Object.hasOwn(body, fieldName);
8446
+ }
8447
+ /**
8448
+ * Builds the "field(s) ... capture(s) ..." detail the intent requires a
8449
+ * surviving hard-fail to name — {@link assertRequiredUrlFieldsReferenced}
8450
+ * itself only knows field names (it never sees the resolved pool), so this
8451
+ * is what turns that into an actionable "which capture" answer when the
8452
+ * narrowing pass can't clear the violation.
8453
+ */
8454
+ function describeOffendingFieldSources(fields, pool) {
8455
+ return fields
8456
+ .map((fieldName) => {
8457
+ const sourceUrls = pool
8458
+ .filter(({ capture }) => captureOwnsTopLevelField(capture, fieldName))
8459
+ .map(({ capture }) => capture.url);
8460
+ return sourceUrls.length > 0
8461
+ ? `${fieldName} (from ${sourceUrls.join(", ")})`
8462
+ : `${fieldName} (source capture not found in resolved pool)`;
8463
+ })
8464
+ .join(", ");
8465
+ }
8466
+ /**
8467
+ * The narrowing pass's core relevance decision, isolated for direct
8468
+ * testing: every capture in `resolvedPool` — other than `primaryCapture`,
8469
+ * which must never be dropped — whose own top-level response JSON owns at
8470
+ * least one of `offendingFields`. This is the actual "is this capture
8471
+ * structurally part of the resolved chain or incidental noise" call;
8472
+ * {@link healUnreferencedUrlFieldsOnce} only wires it into the regenerate
8473
+ * retry loop.
8474
+ *
8475
+ * Exported for tests: this predicate decides which captures the required-
8476
+ * URL-field guard's self-heal excludes before regenerating.
8477
+ */
8478
+ function identifyNoiseCapturesForFields(offendingFields, resolvedPool, primaryCapture) {
8479
+ const noiseCaptures = new Set();
8480
+ for (const fieldName of offendingFields) {
8481
+ for (const { capture } of resolvedPool) {
8482
+ if (capture === primaryCapture)
8483
+ continue;
8484
+ if (captureOwnsTopLevelField(capture, fieldName))
8485
+ noiseCaptures.add(capture);
8486
+ }
8487
+ }
8488
+ return noiseCaptures;
8489
+ }
8490
+ /**
8491
+ * When `assertRequiredUrlFieldsReferenced` would abort on the first pass,
8492
+ * this identifies the capture(s) in the resolved pool that are the SOLE
8493
+ * source of each offending field — excluding the resolved submit/primary
8494
+ * capture, which must never be dropped — removes exactly those captures from
8495
+ * the pool driving generation, and re-runs `generateFromCaptures` exactly
8496
+ * once. Hard-fails (naming the field(s) and, when found, the offending
8497
+ * capture URL(s)) if the violation survives the narrowed pool: either no
8498
+ * noise capture explains it (so nothing was excluded and re-running would be
8499
+ * a no-op), or the field's only source is the resolved primary capture
8500
+ * itself, meaning the field is genuinely required.
8501
+ */
8502
+ function healUnreferencedUrlFieldsOnce(allCaptures, firstAttempt, regenerate) {
8503
+ const offending = unreferencedRequiredUrlFields(firstAttempt.contractCode, firstAttempt.browserFlow.code);
8504
+ if (offending.length === 0)
8505
+ return firstAttempt;
8506
+ const primaryCapture = resolvedPrimaryResponseCapture(firstAttempt);
8507
+ const noiseCaptures = identifyNoiseCapturesForFields(offending, firstAttempt.resolvedPool, primaryCapture);
8508
+ if (noiseCaptures.size === 0) {
8509
+ throw new Error(`recon-generate: required URL field(s) ${describeOffendingFieldSources(offending, firstAttempt.resolvedPool)} ` +
8510
+ `declared on the payload schema but never referenced by the emitted contract or browser flow, and no ` +
8511
+ `unrelated capture explains them — the field is genuinely required by the resolved submit/primary capture ` +
8512
+ `itself; the flow would enter on the wrong page`);
8513
+ }
8514
+ logger.warn(`required URL field(s) ${offending.join(", ")} traced to ${noiseCaptures.size} unrelated capture(s) ` +
8515
+ `(${[...noiseCaptures].map((c) => c.url).join(", ")}) not referenced or threaded by the resolved action ` +
8516
+ `sequence — excluding them and re-generating once`);
8517
+ // Excluded by INDEX into allCaptures, not by capture reference: a
8518
+ // resolvedPool entry produced by the form-schema-fetch insertion (see
8519
+ // generateFromCaptures' schemaFetchCleaned) wraps a shallow clone of its
8520
+ // source capture (url stripped of cache-buster params), so that entry's
8521
+ // `.capture` can never be `===` anything in allCaptures even though its
8522
+ // `.index` still correctly names the source capture's position — every
8523
+ // ActionCapture-producing extractor in this file threads `index` from the
8524
+ // original captures array, so it survives the clone where the reference
8525
+ // doesn't.
8526
+ const noiseIndices = new Set(firstAttempt.resolvedPool.filter((a) => noiseCaptures.has(a.capture)).map((a) => a.index));
8527
+ const narrowedCaptures = allCaptures.filter((_, i) => !noiseIndices.has(i));
8528
+ const retried = regenerate(narrowedCaptures);
8529
+ if (retried === null) {
8530
+ throw new Error("recon-generate: narrowed retry unexpectedly resolved to the config-manifest branch");
8531
+ }
8532
+ const stillOffending = unreferencedRequiredUrlFields(retried.contractCode, retried.browserFlow.code);
8533
+ if (stillOffending.length > 0) {
8534
+ throw new Error(`recon-generate: required URL field(s) ${describeOffendingFieldSources(stillOffending, retried.resolvedPool)} ` +
8535
+ `declared on the payload schema but never referenced by the emitted contract or browser flow, even after ` +
8536
+ `excluding unrelated capture(s) (${[...noiseCaptures].map((c) => c.url).join(", ")}) — the flow would ` +
8537
+ `enter on the wrong page`);
8538
+ }
8539
+ return retried;
8540
+ }
8317
8541
  async function main() {
8318
8542
  const args = process.argv.slice(2);
8319
8543
  let siteId = "";
@@ -8484,428 +8708,454 @@ async function main() {
8484
8708
  // isn't available to deriveBaseUrl yet -- see deriveBaseUrl's own doc
8485
8709
  // comment for how it copes when ownBackendHostnames is empty.
8486
8710
  const ownBackendHostnames = (0, recon_shared_1.readOwnBackendHostnames)(flowFile);
8487
- const baseUrl = deriveBaseUrl(captures, ownBackendHostnames);
8488
- // Gate on the flow's declared own-backend hosts (or the registrable-domain
8489
- // fallback of baseUrl) via the same predicate recon-http.ts applies at
8490
- // write time, so a stale aux/ directory from before that filter existed
8491
- // — or one written by a filter someone bypassed — can never smuggle a
8492
- // third party's JSON into a generated plugin's fixtures/. A file with no
8493
- // manifest entry is unverifiable provenance, not proven safe, so it is
8494
- // excluded rather than assumed to have passed the write-time filter.
8495
- const fallbackDomain = baseUrl.length > 0 ? (0, capture_filters_1.registrableDomain)(new URL(baseUrl).hostname) : null;
8496
- const auxFiles = auxManifest
8497
- .filter((entry) => {
8498
- const allowed = (0, capture_filters_1.isAllowedFixtureHost)(entry.hostname, ownBackendHostnames, fallbackDomain);
8499
- if (!allowed) {
8500
- logger.warn(`excluding aux fixture '${entry.filename}' — host '${entry.hostname}' is not an own-backend host`);
8501
- }
8502
- return allowed;
8503
- })
8504
- .map((entry) => entry.filename)
8505
- .sort();
8506
- // A file on disk with no manifest entry predates the write-time provenance
8507
- // record (bugfix-002) or otherwise landed outside probeAuxiliaryEndpoints —
8508
- // its source host is unverifiable, so it is excluded, not assumed safe.
8509
- const manifestedFilenames = new Set(auxManifest.map((entry) => entry.filename));
8510
- const unmanifestedFiles = (() => {
8511
- try {
8512
- return (0, node_fs_1.readdirSync)(auxDir).filter((f) => f.endsWith(".json") && f !== "aux-manifest.json" && !manifestedFilenames.has(f));
8513
- }
8514
- catch {
8515
- return [];
8516
- }
8517
- })();
8518
- for (const f of unmanifestedFiles) {
8519
- logger.warn(`excluding aux fixture '${f}' — no aux-manifest.json entry, provenance unverifiable`);
8520
- }
8521
- const baseHeaders = deriveRequestHeaders(captures, replays, baseUrl, submitPatterns, ownBackendHostnames, fallbackDomain);
8522
- const minTime = deriveMinTime(rateLimits);
8523
- const hasRateLimitProbeData = rateLimits.some((f) => f.safeRps !== null);
8524
- const safeRps = rateLimits.find((f) => f.safeRps !== null)?.safeRps ?? Math.floor(1000 / minTime);
8525
- const gql = isGraphQL(captures);
8526
- // Hoisted so both the primary-operation gate below and rawActionCaptures
8527
- // (further down) read the same computed sequence instead of calling the
8528
- // extractor twice. Computed unfiltered (submitPatterns: null) — a
8529
- // flow-declared submit pattern must truncate this sequence, not filter
8530
- // it, so every gate reading it sees the full host-gated chain.
8531
- const graphqlActionSequence = gql
8532
- ? extractGraphQLActionSequence(captures, null, foldReturnSpec, ownBackendHostnames, fallbackDomain)
8533
- : [];
8534
- // A foldReturn-admitted read/drill capture (see extractGraphQLActionSequence's
8535
- // doc comment) can put 2+ entries in graphqlActionSequence with none of them
8536
- // an actual `mutation` — a GraphQL-primary query plus its drill-down, not a
8537
- // transactional multi-step submission. Only a real mutation makes this a
8538
- // submission flow; an admitted read/drill capture must not, on its own, null
8539
- // out primaryGraphQLOperation below or flip isSubmissionFlow further down.
8540
- const graphqlActionSequenceHasMutation = graphqlActionSequence.some((a) => a.capture.query !== null && /^\s*mutation\b/.test(a.capture.query));
8541
- const primaryGraphQLOperation = gql && !graphqlActionSequenceHasMutation
8542
- ? selectPrimaryGraphQLOperation(captures, flowSteps, vocabulary, process.env, ownBackendHostnames, fallbackDomain, submitPatterns)
8543
- : null;
8544
- if (primaryGraphQLOperation && primaryGraphQLOperation.unpopulatedDeclaredVariables.length > 0) {
8545
- const operationLabel = primaryGraphQLOperation.capture.operationName ?? "(anonymous)";
8546
- logger.warn(`WARN primary GraphQL operation '${operationLabel}' declares filter variable(s) ${primaryGraphQLOperation.unpopulatedDeclaredVariables.join(", ")} that are never populated in any capture — the generated executeHttp's payload field(s) for ${primaryGraphQLOperation.unpopulatedDeclaredVariables.join(", ")} have no wiring target and will replay the captured frozen value instead of the caller's input`);
8547
- }
8548
- // When there's no primaryGraphQLOperation winner but the flow is still
8549
- // GraphQL, the query/endpointPath/responseBody/operationName fallbacks
8550
- // must all trace back to the SAME capture (firstGraphQLCapture) rather
8551
- // than resolving independently — a non-GraphQL-shaped own-backend
8552
- // capture could otherwise win the endpoint/body fallback while an
8553
- // unrelated capture supplies the query text.
8554
- const fallbackGraphQLCapture = gql
8555
- ? firstGraphQLCapture(captures, ownBackendHostnames, fallbackDomain, submitPatterns)
8556
- : null;
8557
- const gqlQuery = primaryGraphQLOperation?.capture.query ?? fallbackGraphQLCapture?.query ?? null;
8558
- const endpointPath = primaryGraphQLOperation?.endpointPath ??
8559
- (fallbackGraphQLCapture
8560
- ? safeUrlPathname(fallbackGraphQLCapture.url)
8561
- : firstEndpointPath(captures, ownBackendHostnames, fallbackDomain, submitPatterns));
8562
- // Derived from the primary operation's own Phase-1 capture, never from
8563
- // replay array order -- a replay's body reflects whichever endpoint fired
8564
- // first, not necessarily the primary operation, and only exists once
8565
- // recon:http has run.
8566
- const winningCapture = primaryGraphQLOperation?.capture ??
8567
- fallbackGraphQLCapture ??
8568
- firstEndpointCapture(captures, ownBackendHostnames, fallbackDomain, submitPatterns);
8569
- const responseBody = winningCapture?.responseBody ?? null;
8570
- // Every 2xx capture sharing the winning capture's operation identity, not
8571
- // just the one that happened to win selection -- a paginated/re-filtered
8572
- // re-fire of the same operation can omit a field the winning capture
8573
- // happened to have (or vice versa), and inferZodSchemaFromSamples needs
8574
- // that presence evidence across occurrences to mark the field .optional()
8575
- // instead of required from a single observation.
8576
- const responseBodySamples = gatherResponseBodySamples(winningCapture, primaryGraphQLOperation !== null || fallbackGraphQLCapture !== null, captures);
8577
- // Detect a multi-step submission flow (transactional sites like apply forms,
8578
- // checkout, etc.). When the action sequence has 2+ POSTs, switch the
8579
- // contract template to emit a state-threaded executeHttp.
8580
- //
8581
- // Selection precedence: (A) the authoritative submit-manifest recon-browser
8582
- // wrote from the verified submission; else (B/C) pattern/heuristic extraction.
8583
- // The manifest is the only signal that separates a submission POST from a
8584
- // page-chrome POST sharing its URL, so it normally wins when present — but
8585
- // a manifest built from a single flow-declared submit step cannot represent
8586
- // a wizard whose every section saves independently, so it is only trusted
8587
- // when it isn't a strict undercount of what the same captures' own
8588
- // heuristic extraction finds.
8589
- const unfilteredHeuristicActionCaptures = gql
8590
- ? dedupRedundantSameOperationCaptures(graphqlActionSequence, primaryGraphQLOperation)
8591
- : collapseRedundantPatches(extractActionSequence(captures, null, foldReturnSpec, ownBackendHostnames, fallbackDomain));
8592
- // A flow-declared submitEndpointPattern is authoritative: it may match only
8593
- // the final step's URL (the natural way to describe "the button that
8594
- // finishes the wizard") even though the earlier steps of the same chain
8595
- // (auth mint, paged listing, ...) are what state-threading depends on.
8596
- // Truncating at the last match — instead of filtering to matches only —
8597
- // keeps that whole chain; the gap between this and the unfiltered
8598
- // sequence is logged below for visibility, but the declared pattern is
8599
- // never overridden by the richer unfiltered sequence.
8600
- const patternedHeuristicActionCaptures = submitPatterns.endpoint === null && submitPatterns.body === null
8601
- ? unfilteredHeuristicActionCaptures
8602
- : truncateActionSequenceAtSubmitPattern(unfilteredHeuristicActionCaptures, submitPatterns);
8603
- const patternUndercounts = patternedHeuristicActionCaptures.length < unfilteredHeuristicActionCaptures.length;
8604
- if (patternUndercounts && requireSubmitEndpointMatch) {
8605
- // Distinct wording from the non-required case below: this pattern is never
8606
- // discarded, so a message that says "ignoring"/"undercount" would misstate
8607
- // what happened. The disagreement is still worth a warn-level surface —
8608
- // the flow author should know the declared pattern covers fewer captures
8609
- // than the unfiltered heuristic sequence finds.
8610
- logger.warn(`submission selection: declared submitEndpointPattern/submitBodyPattern (${patternedHeuristicActionCaptures.length} capture(s)) disagrees with the unfiltered heuristic action sequence (${unfilteredHeuristicActionCaptures.length} capture(s)); using the declared pattern because requireSubmitEndpointMatch is set`);
8611
- }
8612
- else if (patternUndercounts) {
8613
- logger.info(`submission selection: ignoring submitEndpointPattern/submitBodyPattern (${patternedHeuristicActionCaptures.length} capture(s)) as an undercount of the unfiltered heuristic action sequence (${unfilteredHeuristicActionCaptures.length} capture(s))`);
8614
- }
8615
- const heuristicActionCaptures = patternedHeuristicActionCaptures;
8616
- const manifestActionCaptures = resolveManifestActionSequence(runRoot, captures);
8617
- const manifestUndercounts = manifestActionCaptures !== null &&
8618
- manifestActionCaptures.length < heuristicActionCaptures.length;
8619
- if (manifestActionCaptures !== null && manifestUndercounts) {
8620
- logger.info(`submission selection: ignoring submit-manifest.json (${manifestActionCaptures.length} capture(s)) as an undercount of the heuristic action sequence (${heuristicActionCaptures.length} capture(s))`);
8621
- }
8622
- else if (manifestActionCaptures !== null) {
8623
- logger.info(`submission selection: using submit-manifest.json (${manifestActionCaptures.length} authoritative capture(s))`);
8624
- }
8625
- const rawActionCaptures = manifestActionCaptures !== null && !manifestUndercounts
8626
- ? manifestActionCaptures
8627
- : heuristicActionCaptures;
8628
- // Form-schema detection runs BEFORE state-indexing so the field-id/option-id
8629
- // UUIDs can be shielded from indexing — those UUIDs are stable schema
8630
- // anchors that T2/T3 substitution depends on remaining literal in body
8631
- // templates.
8632
- const { fieldNameMap, fieldOptionsMap, allSchemaUuids } = detectFormSchemaFieldNames(captures, formSchema);
8633
- // Shield ALL field-id/option-id UUIDs that appear in any schema response, not
8634
- // just the ones that detectFormSchemaFieldNames emits a payload name for.
8635
- // Some fields have names too long for the naming heuristic (>80 chars) and
8636
- // would be skipped by fieldNameMap; their field-ids still need shielding
8637
- // because they appear as anchors in the T2-substituted body templates.
8638
- const shieldedUuids = new Set(allSchemaUuids);
8639
- // Persona identity bindings + entry-URL job coordinates — the value→payload
8640
- // reconciliation the body emitter merges into its substitution map so nested
8641
- // applicant fields and job context reach the caller's data instead of the
8642
- // recon persona's. Both are site-agnostic: persona mapping comes from the
8643
- // consumer vocabulary, job coordinates from the entry URL's own query keys.
8644
- const personaBindings = harvestPersonaBindings(flowSteps, vocabulary, process.env);
8645
- const entryUrlParams = extractEntryUrlParams(captures[0]?.url ?? "");
8646
- // T4 — Phase B+C: detect a form-schema GET capture and insert it into the
8647
- // action sequence at the position observed during recon, so the existing
8648
- // state-threading machinery can produce its FormHistoryId / section UUIDs /
8649
- // etc. as state values for downstream POSTs. Strip cache-buster query
8650
- // params (recon timestamps) from the captured URL so the emitted runtime
8651
- // fetch uses a clean template. Sites without a schema-fetch capture
8652
- // (rawSchemaFetch === null) get unchanged behavior.
8653
- const rawSchemaFetch = gql || formSchema === null ? null : detectFormSchemaFetchCapture(captures, baseUrl, formSchema);
8654
- const schemaFetchCleaned = rawSchemaFetch
8655
- ? { ...rawSchemaFetch.capture, url: stripCacheBusterParams(rawSchemaFetch.capture.url) }
8656
- : null;
8657
- const actionCaptures = (() => {
8658
- if (rawActionCaptures.length === 0 || schemaFetchCleaned === null || rawSchemaFetch === null) {
8659
- return rawActionCaptures;
8660
- }
8661
- let insertAt = rawActionCaptures.length;
8662
- for (let i = 0; i < rawActionCaptures.length; i++) {
8663
- if (rawActionCaptures[i].index >= rawSchemaFetch.index) {
8664
- insertAt = i;
8665
- break;
8711
+ /**
8712
+ * Runs the capture-to-contract pipeline once for a given capture pool.
8713
+ * Nested (not top-level) so `assertRequiredUrlFieldsReferenced`'s
8714
+ * self-healing retry can invoke it a second time against a narrowed
8715
+ * `activeCaptures` without duplicating ~500 lines of deterministic
8716
+ * derivation logic. Returns null once the config-manifest branch has
8717
+ * already written its output and there is nothing left for main() to do.
8718
+ */
8719
+ function generateFromCaptures(activeCaptures) {
8720
+ const baseUrl = deriveBaseUrl(activeCaptures, ownBackendHostnames);
8721
+ // Gate on the flow's declared own-backend hosts (or the registrable-domain
8722
+ // fallback of baseUrl) via the same predicate recon-http.ts applies at
8723
+ // write time, so a stale aux/ directory from before that filter existed
8724
+ // — or one written by a filter someone bypassed — can never smuggle a
8725
+ // third party's JSON into a generated plugin's fixtures/. A file with no
8726
+ // manifest entry is unverifiable provenance, not proven safe, so it is
8727
+ // excluded rather than assumed to have passed the write-time filter.
8728
+ const fallbackDomain = baseUrl.length > 0 ? (0, capture_filters_1.registrableDomain)(new URL(baseUrl).hostname) : null;
8729
+ const auxFiles = auxManifest
8730
+ .filter((entry) => {
8731
+ const allowed = (0, capture_filters_1.isAllowedFixtureHost)(entry.hostname, ownBackendHostnames, fallbackDomain);
8732
+ if (!allowed) {
8733
+ logger.warn(`excluding aux fixture '${entry.filename}' — host '${entry.hostname}' is not an own-backend host`);
8666
8734
  }
8667
- }
8668
- return [
8669
- ...rawActionCaptures.slice(0, insertAt),
8670
- { capture: schemaFetchCleaned, index: rawSchemaFetch.index },
8671
- ...rawActionCaptures.slice(insertAt),
8672
- ];
8673
- })();
8674
- const actionCaptureIndices = new Set(actionCaptures.map((a) => a.index));
8675
- // Resolved off raw actionCaptures — fold-plan DETECTION depends only on
8676
- // each action's capture, so this runs before compileActionSteps/
8677
- // indexStateValues even exist — so a short numeric join value threaded
8678
- // through a dependent-drill-down chain hop still gets indexed as
8679
- // producible state (see collectDependentDrillDownChainValues).
8680
- const dependentDrillDownChainValues = actionCaptures.length > 1
8681
- ? collectDependentDrillDownChainValues(actionCaptures, foldReturnSpec)
8682
- : new Set();
8683
- const stateIndex = actionCaptures.length > 1
8684
- ? indexStateValues(captures, shieldedUuids, actionCaptureIndices, dependentDrillDownChainValues)
8685
- : new Map();
8686
- const actionSteps = actionCaptures.length > 1 ? compileActionSteps(actionCaptures, stateIndex) : [];
8687
- const isSubmissionFlow = actionSteps.length > 1 && (!gql || graphqlActionSequenceHasMutation);
8688
- // Diagnostic for the FAILURE-3 shape (a flowless recon capture): a "submission flow"
8689
- // whose every action capture is landing-phase is almost certainly page-chrome
8690
- // bootstrap (e.g. page-chrome `POST /widgets`) misread as an apply flow, not a walked
8691
- // wizard. Real wizard steps carry a step-slug phase; single-endpoint search runs
8692
- // are `length <= 1` and never reach here. We do not filter (that would delete the
8693
- // sole search POST of legitimate `--url`-only single-endpoint runs, which is also
8694
- // landing-phase) — we only surface the suspicious shape.
8695
- if (isSubmissionFlow && actionCaptures.every((a) => a.capture.phase === "home")) {
8696
- logger.warn(`WARN all ${actionCaptures.length} action captures are landing-phase (phase="home") — this may be page-chrome bootstrap misread as a submission flow, not a walked apply wizard; verify the recon --flow actually advanced the form`);
8697
- }
8698
- const inputBody = isSubmissionFlow
8699
- ? (() => {
8735
+ return allowed;
8736
+ })
8737
+ .map((entry) => entry.filename)
8738
+ .sort();
8739
+ // A file on disk with no manifest entry predates the write-time provenance
8740
+ // record (bugfix-002) or otherwise landed outside probeAuxiliaryEndpoints —
8741
+ // its source host is unverifiable, so it is excluded, not assumed safe.
8742
+ const manifestedFilenames = new Set(auxManifest.map((entry) => entry.filename));
8743
+ const unmanifestedFiles = (() => {
8700
8744
  try {
8701
- const payloadAction = selectPayloadAction(actionSteps);
8702
- return JSON.parse(payloadAction?.capture.requestPostData ?? "null");
8745
+ return (0, node_fs_1.readdirSync)(auxDir).filter((f) => f.endsWith(".json") && f !== "aux-manifest.json" && !manifestedFilenames.has(f));
8703
8746
  }
8704
8747
  catch {
8705
- return null;
8748
+ return [];
8749
+ }
8750
+ })();
8751
+ for (const f of unmanifestedFiles) {
8752
+ logger.warn(`excluding aux fixture '${f}' — no aux-manifest.json entry, provenance unverifiable`);
8753
+ }
8754
+ const baseHeaders = deriveRequestHeaders(activeCaptures, replays, baseUrl, submitPatterns, ownBackendHostnames, fallbackDomain);
8755
+ const minTime = deriveMinTime(rateLimits);
8756
+ const hasRateLimitProbeData = rateLimits.some((f) => f.safeRps !== null);
8757
+ const safeRps = rateLimits.find((f) => f.safeRps !== null)?.safeRps ?? Math.floor(1000 / minTime);
8758
+ const gql = isGraphQL(activeCaptures);
8759
+ // Hoisted so both the primary-operation gate below and rawActionCaptures
8760
+ // (further down) read the same computed sequence instead of calling the
8761
+ // extractor twice. Computed unfiltered (submitPatterns: null) — a
8762
+ // flow-declared submit pattern must truncate this sequence, not filter
8763
+ // it, so every gate reading it sees the full host-gated chain.
8764
+ const graphqlActionSequence = gql
8765
+ ? extractGraphQLActionSequence(activeCaptures, null, foldReturnSpec, ownBackendHostnames, fallbackDomain)
8766
+ : [];
8767
+ // A foldReturn-admitted read/drill capture (see extractGraphQLActionSequence's
8768
+ // doc comment) can put 2+ entries in graphqlActionSequence with none of them
8769
+ // an actual `mutation` — a GraphQL-primary query plus its drill-down, not a
8770
+ // transactional multi-step submission. Only a real mutation makes this a
8771
+ // submission flow; an admitted read/drill capture must not, on its own, null
8772
+ // out primaryGraphQLOperation below or flip isSubmissionFlow further down.
8773
+ const graphqlActionSequenceHasMutation = graphqlActionSequence.some((a) => a.capture.query !== null && /^\s*mutation\b/.test(a.capture.query));
8774
+ const primaryGraphQLOperation = gql && !graphqlActionSequenceHasMutation
8775
+ ? selectPrimaryGraphQLOperation(activeCaptures, flowSteps, vocabulary, process.env, ownBackendHostnames, fallbackDomain, submitPatterns)
8776
+ : null;
8777
+ if (primaryGraphQLOperation &&
8778
+ primaryGraphQLOperation.unpopulatedDeclaredVariables.length > 0) {
8779
+ const operationLabel = primaryGraphQLOperation.capture.operationName ?? "(anonymous)";
8780
+ logger.warn(`WARN primary GraphQL operation '${operationLabel}' declares filter variable(s) ${primaryGraphQLOperation.unpopulatedDeclaredVariables.join(", ")} that are never populated in any capture — the generated executeHttp's payload field(s) for ${primaryGraphQLOperation.unpopulatedDeclaredVariables.join(", ")} have no wiring target and will replay the captured frozen value instead of the caller's input`);
8781
+ }
8782
+ // When there's no primaryGraphQLOperation winner but the flow is still
8783
+ // GraphQL, the query/endpointPath/responseBody/operationName fallbacks
8784
+ // must all trace back to the SAME capture (firstGraphQLCapture) rather
8785
+ // than resolving independently — a non-GraphQL-shaped own-backend
8786
+ // capture could otherwise win the endpoint/body fallback while an
8787
+ // unrelated capture supplies the query text.
8788
+ const fallbackGraphQLCapture = gql
8789
+ ? firstGraphQLCapture(activeCaptures, ownBackendHostnames, fallbackDomain, submitPatterns)
8790
+ : null;
8791
+ const gqlQuery = primaryGraphQLOperation?.capture.query ?? fallbackGraphQLCapture?.query ?? null;
8792
+ const endpointPath = primaryGraphQLOperation?.endpointPath ??
8793
+ (fallbackGraphQLCapture
8794
+ ? safeUrlPathname(fallbackGraphQLCapture.url)
8795
+ : firstEndpointPath(activeCaptures, ownBackendHostnames, fallbackDomain, submitPatterns));
8796
+ // Derived from the primary operation's own Phase-1 capture, never from
8797
+ // replay array order -- a replay's body reflects whichever endpoint fired
8798
+ // first, not necessarily the primary operation, and only exists once
8799
+ // recon:http has run.
8800
+ const winningCapture = primaryGraphQLOperation?.capture ??
8801
+ fallbackGraphQLCapture ??
8802
+ firstEndpointCapture(activeCaptures, ownBackendHostnames, fallbackDomain, submitPatterns);
8803
+ const responseBody = winningCapture?.responseBody ?? null;
8804
+ // Every 2xx capture sharing the winning capture's operation identity, not
8805
+ // just the one that happened to win selection -- a paginated/re-filtered
8806
+ // re-fire of the same operation can omit a field the winning capture
8807
+ // happened to have (or vice versa), and inferZodSchemaFromSamples needs
8808
+ // that presence evidence across occurrences to mark the field .optional()
8809
+ // instead of required from a single observation.
8810
+ const responseBodySamples = gatherResponseBodySamples(winningCapture, primaryGraphQLOperation !== null || fallbackGraphQLCapture !== null, activeCaptures);
8811
+ // Detect a multi-step submission flow (transactional sites like apply forms,
8812
+ // checkout, etc.). When the action sequence has 2+ POSTs, switch the
8813
+ // contract template to emit a state-threaded executeHttp.
8814
+ //
8815
+ // Selection precedence: (A) the authoritative submit-manifest recon-browser
8816
+ // wrote from the verified submission; else (B/C) pattern/heuristic extraction.
8817
+ // The manifest is the only signal that separates a submission POST from a
8818
+ // page-chrome POST sharing its URL, so it normally wins when present — but
8819
+ // a manifest built from a single flow-declared submit step cannot represent
8820
+ // a wizard whose every section saves independently, so it is only trusted
8821
+ // when it isn't a strict undercount of what the same activeCaptures' own
8822
+ // heuristic extraction finds.
8823
+ const unfilteredHeuristicActionCaptures = gql
8824
+ ? dedupRedundantSameOperationCaptures(graphqlActionSequence, primaryGraphQLOperation)
8825
+ : collapseRedundantPatches(extractActionSequence(activeCaptures, null, foldReturnSpec, ownBackendHostnames, fallbackDomain));
8826
+ // A flow-declared submitEndpointPattern is authoritative: it may match only
8827
+ // the final step's URL (the natural way to describe "the button that
8828
+ // finishes the wizard") even though the earlier steps of the same chain
8829
+ // (auth mint, paged listing, ...) are what state-threading depends on.
8830
+ // Truncating at the last match — instead of filtering to matches only —
8831
+ // keeps that whole chain; the gap between this and the unfiltered
8832
+ // sequence is logged below for visibility, but the declared pattern is
8833
+ // never overridden by the richer unfiltered sequence.
8834
+ const patternedHeuristicActionCaptures = submitPatterns.endpoint === null && submitPatterns.body === null
8835
+ ? unfilteredHeuristicActionCaptures
8836
+ : truncateActionSequenceAtSubmitPattern(unfilteredHeuristicActionCaptures, submitPatterns);
8837
+ const patternUndercounts = patternedHeuristicActionCaptures.length < unfilteredHeuristicActionCaptures.length;
8838
+ if (patternUndercounts && requireSubmitEndpointMatch) {
8839
+ // Distinct wording from the non-required case below: this pattern is never
8840
+ // discarded, so a message that says "ignoring"/"undercount" would misstate
8841
+ // what happened. The disagreement is still worth a warn-level surface —
8842
+ // the flow author should know the declared pattern covers fewer activeCaptures
8843
+ // than the unfiltered heuristic sequence finds.
8844
+ logger.warn(`submission selection: declared submitEndpointPattern/submitBodyPattern (${patternedHeuristicActionCaptures.length} capture(s)) disagrees with the unfiltered heuristic action sequence (${unfilteredHeuristicActionCaptures.length} capture(s)); using the declared pattern because requireSubmitEndpointMatch is set`);
8845
+ }
8846
+ else if (patternUndercounts) {
8847
+ logger.info(`submission selection: ignoring submitEndpointPattern/submitBodyPattern (${patternedHeuristicActionCaptures.length} capture(s)) as an undercount of the unfiltered heuristic action sequence (${unfilteredHeuristicActionCaptures.length} capture(s))`);
8848
+ }
8849
+ const heuristicActionCaptures = patternedHeuristicActionCaptures;
8850
+ const manifestActionCaptures = resolveManifestActionSequence(runRoot, activeCaptures);
8851
+ const manifestUndercounts = manifestActionCaptures !== null &&
8852
+ manifestActionCaptures.length < heuristicActionCaptures.length;
8853
+ if (manifestActionCaptures !== null && manifestUndercounts) {
8854
+ logger.info(`submission selection: ignoring submit-manifest.json (${manifestActionCaptures.length} capture(s)) as an undercount of the heuristic action sequence (${heuristicActionCaptures.length} capture(s))`);
8855
+ }
8856
+ else if (manifestActionCaptures !== null) {
8857
+ logger.info(`submission selection: using submit-manifest.json (${manifestActionCaptures.length} authoritative capture(s))`);
8858
+ }
8859
+ const rawActionCaptures = manifestActionCaptures !== null && !manifestUndercounts
8860
+ ? manifestActionCaptures
8861
+ : heuristicActionCaptures;
8862
+ // Form-schema detection runs BEFORE state-indexing so the field-id/option-id
8863
+ // UUIDs can be shielded from indexing — those UUIDs are stable schema
8864
+ // anchors that T2/T3 substitution depends on remaining literal in body
8865
+ // templates.
8866
+ const { fieldNameMap, fieldOptionsMap, allSchemaUuids } = detectFormSchemaFieldNames(activeCaptures, formSchema);
8867
+ // Shield ALL field-id/option-id UUIDs that appear in any schema response, not
8868
+ // just the ones that detectFormSchemaFieldNames emits a payload name for.
8869
+ // Some fields have names too long for the naming heuristic (>80 chars) and
8870
+ // would be skipped by fieldNameMap; their field-ids still need shielding
8871
+ // because they appear as anchors in the T2-substituted body templates.
8872
+ const shieldedUuids = new Set(allSchemaUuids);
8873
+ // Persona identity bindings + entry-URL job coordinates — the value→payload
8874
+ // reconciliation the body emitter merges into its substitution map so nested
8875
+ // applicant fields and job context reach the caller's data instead of the
8876
+ // recon persona's. Both are site-agnostic: persona mapping comes from the
8877
+ // consumer vocabulary, job coordinates from the entry URL's own query keys.
8878
+ const personaBindings = harvestPersonaBindings(flowSteps, vocabulary, process.env);
8879
+ const entryUrlParams = extractEntryUrlParams(activeCaptures[0]?.url ?? "");
8880
+ // T4 — Phase B+C: detect a form-schema GET capture and insert it into the
8881
+ // action sequence at the position observed during recon, so the existing
8882
+ // state-threading machinery can produce its FormHistoryId / section UUIDs /
8883
+ // etc. as state values for downstream POSTs. Strip cache-buster query
8884
+ // params (recon timestamps) from the captured URL so the emitted runtime
8885
+ // fetch uses a clean template. Sites without a schema-fetch capture
8886
+ // (rawSchemaFetch === null) get unchanged behavior.
8887
+ const rawSchemaFetch = gql || formSchema === null
8888
+ ? null
8889
+ : detectFormSchemaFetchCapture(activeCaptures, baseUrl, formSchema);
8890
+ const schemaFetchCleaned = rawSchemaFetch
8891
+ ? { ...rawSchemaFetch.capture, url: stripCacheBusterParams(rawSchemaFetch.capture.url) }
8892
+ : null;
8893
+ const actionCaptures = (() => {
8894
+ if (rawActionCaptures.length === 0 ||
8895
+ schemaFetchCleaned === null ||
8896
+ rawSchemaFetch === null) {
8897
+ return rawActionCaptures;
8898
+ }
8899
+ let insertAt = rawActionCaptures.length;
8900
+ for (let i = 0; i < rawActionCaptures.length; i++) {
8901
+ if (rawActionCaptures[i].index >= rawSchemaFetch.index) {
8902
+ insertAt = i;
8903
+ break;
8904
+ }
8905
+ }
8906
+ return [
8907
+ ...rawActionCaptures.slice(0, insertAt),
8908
+ { capture: schemaFetchCleaned, index: rawSchemaFetch.index },
8909
+ ...rawActionCaptures.slice(insertAt),
8910
+ ];
8911
+ })();
8912
+ const actionCaptureIndices = new Set(actionCaptures.map((a) => a.index));
8913
+ // Resolved off raw actionCaptures — fold-plan DETECTION depends only on
8914
+ // each action's capture, so this runs before compileActionSteps/
8915
+ // indexStateValues even exist — so a short numeric join value threaded
8916
+ // through a dependent-drill-down chain hop still gets indexed as
8917
+ // producible state (see collectDependentDrillDownChainValues).
8918
+ const dependentDrillDownChainValues = actionCaptures.length > 1
8919
+ ? collectDependentDrillDownChainValues(actionCaptures, foldReturnSpec)
8920
+ : new Set();
8921
+ const stateIndex = actionCaptures.length > 1
8922
+ ? indexStateValues(activeCaptures, shieldedUuids, actionCaptureIndices, dependentDrillDownChainValues)
8923
+ : new Map();
8924
+ const actionSteps = actionCaptures.length > 1 ? compileActionSteps(actionCaptures, stateIndex) : [];
8925
+ const isSubmissionFlow = actionSteps.length > 1 && (!gql || graphqlActionSequenceHasMutation);
8926
+ // Diagnostic for the FAILURE-3 shape (a flowless recon capture): a "submission flow"
8927
+ // whose every action capture is landing-phase is almost certainly page-chrome
8928
+ // bootstrap (e.g. page-chrome `POST /widgets`) misread as an apply flow, not a walked
8929
+ // wizard. Real wizard steps carry a step-slug phase; single-endpoint search runs
8930
+ // are `length <= 1` and never reach here. We do not filter (that would delete the
8931
+ // sole search POST of legitimate `--url`-only single-endpoint runs, which is also
8932
+ // landing-phase) — we only surface the suspicious shape.
8933
+ if (isSubmissionFlow && actionCaptures.every((a) => a.capture.phase === "home")) {
8934
+ logger.warn(`WARN all ${actionCaptures.length} action captures are landing-phase (phase="home") — this may be page-chrome bootstrap misread as a submission flow, not a walked apply wizard; verify the recon --flow actually advanced the form`);
8935
+ }
8936
+ const inputBody = isSubmissionFlow
8937
+ ? (() => {
8938
+ try {
8939
+ const payloadAction = selectPayloadAction(actionSteps);
8940
+ return JSON.parse(payloadAction?.capture.requestPostData ?? "null");
8941
+ }
8942
+ catch {
8943
+ return null;
8944
+ }
8945
+ })()
8946
+ : undefined;
8947
+ const errorSignals = detectErrorSignals(actionSteps);
8948
+ const discoveredFormFields = new Set();
8949
+ const discoveredOptionFields = new Set();
8950
+ // Phase E: maps label-derived raw-option payload field name (e.g.
8951
+ // "AreYouOverTheAgeOf18OptionId") → recon-observed option-id UUID. Used to
8952
+ // emit `<Name>OptionId: z.string()` payload fields with TSDoc docs.
8953
+ const discoveredRawOptionFields = new Map();
8954
+ // Phase F: keys from additional action POST bodies (beyond inputBody/r0)
8955
+ // that get parameterized. Recorded with their value type so the contract
8956
+ // emitter can add them to the payload schema with appropriate Zod types.
8957
+ const discoveredAdditionalBodyKeys = new Map();
8958
+ // Mechanism A: reconcile flow SELECT steps to submitted option codes. The
8959
+ // resolutions drive a wire-key-anchored body rewrite (label→code dropdowns);
8960
+ // i18n-only dropdowns (labels all templated, e.g. gender) fall through to the
8961
+ // existing raw-option channel so their frozen code is still parameterized.
8962
+ const { resolutions: selectResolutions, rawCodeFields } = buildSelectOptionResolutions(flowSteps, activeCaptures, vocabulary, process.env);
8963
+ for (const [semanticName, { code }] of rawCodeFields) {
8964
+ const fieldName = `${semanticName}Code`;
8965
+ if (!discoveredRawOptionFields.has(fieldName))
8966
+ discoveredRawOptionFields.set(fieldName, code);
8967
+ }
8968
+ // Mechanism B: nested caller structures (experienceData/educationData
8969
+ // history, opaque eventData) discovered during the body emit, surfaced to the
8970
+ // contract's payload schema.
8971
+ const discoveredStructuredKeys = new Map();
8972
+ // G1+G2: partition baseHeaders into three buckets:
8973
+ // - static: values that don't reference baseUrl or tenant subdomain
8974
+ // - baseUrl-derived: values containing the recon's baseUrl as substring
8975
+ // (e.g. Origin, Referer) — emit per-call from payload.BaseUrl
8976
+ // - tenant-subdomain: values that EXACTLY equal the first subdomain
8977
+ // (e.g. API-ShortName: "addus") — emit per-call from a payload field
8978
+ const staticBaseHeaders = {};
8979
+ const baseUrlDerivedHeaders = new Map();
8980
+ const tenantSubdomainHeaders = new Map();
8981
+ const firstSubdomain = (() => {
8982
+ try {
8983
+ const host = new URL(baseUrl).hostname;
8984
+ const firstDot = host.indexOf(".");
8985
+ return firstDot === -1 ? host : host.slice(0, firstDot);
8986
+ }
8987
+ catch {
8988
+ return "";
8989
+ }
8990
+ })();
8991
+ for (const [k, v] of Object.entries(baseHeaders)) {
8992
+ if (firstSubdomain.length > 0 && v === firstSubdomain) {
8993
+ tenantSubdomainHeaders.set(k, v);
8994
+ }
8995
+ else if (baseUrl.length > 0 && v.includes(baseUrl)) {
8996
+ baseUrlDerivedHeaders.set(k, v);
8997
+ }
8998
+ else {
8999
+ staticBaseHeaders[k] = v;
8706
9000
  }
8707
- })()
8708
- : undefined;
8709
- const errorSignals = detectErrorSignals(actionSteps);
8710
- const discoveredFormFields = new Set();
8711
- const discoveredOptionFields = new Set();
8712
- // Phase E: maps label-derived raw-option payload field name (e.g.
8713
- // "AreYouOverTheAgeOf18OptionId") → recon-observed option-id UUID. Used to
8714
- // emit `<Name>OptionId: z.string()` payload fields with TSDoc docs.
8715
- const discoveredRawOptionFields = new Map();
8716
- // Phase F: keys from additional action POST bodies (beyond inputBody/r0)
8717
- // that get parameterized. Recorded with their value type so the contract
8718
- // emitter can add them to the payload schema with appropriate Zod types.
8719
- const discoveredAdditionalBodyKeys = new Map();
8720
- // Mechanism A: reconcile flow SELECT steps to submitted option codes. The
8721
- // resolutions drive a wire-key-anchored body rewrite (label→code dropdowns);
8722
- // i18n-only dropdowns (labels all templated, e.g. gender) fall through to the
8723
- // existing raw-option channel so their frozen code is still parameterized.
8724
- const { resolutions: selectResolutions, rawCodeFields } = buildSelectOptionResolutions(flowSteps, captures, vocabulary, process.env);
8725
- for (const [semanticName, { code }] of rawCodeFields) {
8726
- const fieldName = `${semanticName}Code`;
8727
- if (!discoveredRawOptionFields.has(fieldName))
8728
- discoveredRawOptionFields.set(fieldName, code);
8729
- }
8730
- // Mechanism B: nested caller structures (experienceData/educationData
8731
- // history, opaque eventData) discovered during the body emit, surfaced to the
8732
- // contract's payload schema.
8733
- const discoveredStructuredKeys = new Map();
8734
- // G1+G2: partition baseHeaders into three buckets:
8735
- // - static: values that don't reference baseUrl or tenant subdomain
8736
- // - baseUrl-derived: values containing the recon's baseUrl as substring
8737
- // (e.g. Origin, Referer) — emit per-call from payload.BaseUrl
8738
- // - tenant-subdomain: values that EXACTLY equal the first subdomain
8739
- // (e.g. API-ShortName: "addus") — emit per-call from a payload field
8740
- const staticBaseHeaders = {};
8741
- const baseUrlDerivedHeaders = new Map();
8742
- const tenantSubdomainHeaders = new Map();
8743
- const firstSubdomain = (() => {
8744
- try {
8745
- const host = new URL(baseUrl).hostname;
8746
- const firstDot = host.indexOf(".");
8747
- return firstDot === -1 ? host : host.slice(0, firstDot);
8748
- }
8749
- catch {
8750
- return "";
8751
- }
8752
- })();
8753
- for (const [k, v] of Object.entries(baseHeaders)) {
8754
- if (firstSubdomain.length > 0 && v === firstSubdomain) {
8755
- tenantSubdomainHeaders.set(k, v);
8756
9001
  }
8757
- else if (baseUrl.length > 0 && v.includes(baseUrl)) {
8758
- baseUrlDerivedHeaders.set(k, v);
9002
+ // Third branch (browser-flow-only): a multi-action flow (isSubmissionFlow)
9003
+ // that crosses hosts mid-sequence (compileActionSteps' isCrossDomain — a
9004
+ // captured redirect off the original domain, e.g. an auth bounce or a
9005
+ // vendor-hosted submission step). A bare `fetch`-based executeHttp can't
9006
+ // reliably replay that: cookies/CSRF/session state minted for one origin
9007
+ // don't automatically carry to the next the way a real browser's redirect
9008
+ // handling does, so synthesizing a same-shape HTTP sequence would silently
9009
+ // drop the session boundary the recon actually walked. Per-step, this
9010
+ // already surfaces as the "cross-domain redirect detected ... likely needs
9011
+ // browser fallback for this step" TODO (see emitMultiStepExecuteHttp); at
9012
+ // the whole-flow level the honest emit is no executeHttp at all — never a
9013
+ // same-host multi-step body that quietly drops the hop, and never a
9014
+ // downgrade to the single-endpoint `{query}` branch either, since that's a
9015
+ // fabrication of its own kind for a flow that isn't a single-action
9016
+ // query/search to begin with.
9017
+ const browserFlowOnly = isSubmissionFlow && actionSteps.some((s) => s.isCrossDomain);
9018
+ const multiStepBody = browserFlowOnly
9019
+ ? undefined
9020
+ : isSubmissionFlow
9021
+ ? emitMultiStepExecuteHttp(actionSteps, inputBody, errorSignals, fieldNameMap, discoveredFormFields, fieldOptionsMap, discoveredOptionFields, discoveredRawOptionFields, discoveredAdditionalBodyKeys, baseUrl, baseUrlDerivedHeaders, tenantSubdomainHeaders, formSchema, personaBindings, entryUrlParams, shieldedUuids, selectResolutions, discoveredStructuredKeys, rawCodeFields, foldReturnSpec)
9022
+ : undefined;
9023
+ const hasMultipartStep = actionSteps.some((s) => s.isMultipart);
9024
+ const headerBindings = collectHeaderBindings(actionSteps);
9025
+ // Shape inference targets the SAME call executeHttp returns — see
9026
+ // selectEffectiveResponseBody — so the two surfaces can't describe different calls.
9027
+ const effectiveResponseBody = selectEffectiveResponseBody(isSubmissionFlow, actionSteps, responseBody, foldReturnSpec);
9028
+ // A declared foldReturn that resolves to no plan is a silent no-op otherwise
9029
+ // — the flow author gets the discarding selectReturnAction path with nothing
9030
+ // in the output saying their declaration never applied. A multi-step
9031
+ // (submission) flow applies its fold via emitMultiStepExecuteHttp's own
9032
+ // resolveFoldPlan call, entirely independent of resolveApplicableFoldPlans
9033
+ // (which exists only to gate emitContractTs's single-primary hot path, and
9034
+ // unconditionally reports zero plans once multiStepBody is set) — so this
9035
+ // diagnostic must consult the SAME resolution each path actually applies,
9036
+ // or it falsely reports "no fold plan resolved" for every multi-step flow
9037
+ // with a working foldReturn.
9038
+ const effectiveFoldPlanCount = multiStepBody
9039
+ ? resolveFoldPlan(actionSteps, foldReturnSpec).length
9040
+ : resolveApplicableFoldPlans(actionSteps, foldReturnSpec, multiStepBody).length;
9041
+ if (foldReturnSpec !== null && effectiveFoldPlanCount === 0) {
9042
+ logger.warn(`flow declares foldReturn (endpointPattern: ${foldReturnSpec.endpointPattern}, resultsPath: ${foldReturnSpec.resultsPath}, joinFields: ${foldReturnSpec.joinFields.join(", ")}) but no fold plan resolved — no later capture matched the endpoint pattern, resultsPath resolved to no object array, or the matched drill-down is multipart; the drill-down's response will not be folded`);
9043
+ }
9044
+ logger.info(`generating plugin for ${siteId} (${gql ? "GraphQL" : browserFlowOnly ? `submission flow, ${actionSteps.length} steps, browser-flow-only (cross-domain hop detected)` : isSubmissionFlow ? `submission flow, ${actionSteps.length} steps` : "single-endpoint REST"}, baseUrl: ${baseUrl})`);
9045
+ if (emit === "config") {
9046
+ (0, node_fs_1.mkdirSync)(outDir, { recursive: true });
9047
+ (0, node_fs_1.writeFileSync)(manifestPath, emitConfigManifest({
9048
+ siteId,
9049
+ displayName,
9050
+ baseUrl,
9051
+ flowSteps,
9052
+ vocabulary,
9053
+ inputBody,
9054
+ recoveredFields: [...discoveredFormFields, ...discoveredOptionFields],
9055
+ isSubmissionFlow,
9056
+ // A submission flow is the case where the `.ts` emit carries an
9057
+ // executeHttp hot path; point the manifest at where the operator drops
9058
+ // the compiled module rather than silently dropping the direct path.
9059
+ // Not set for the browser-flow-only branch — there is no hot path to
9060
+ // compile a module for.
9061
+ httpModulePath: isSubmissionFlow && !browserFlowOnly ? `./${siteId}.http.js` : undefined,
9062
+ }));
9063
+ logger.info(`wrote ${manifestPath}`);
9064
+ logger.info(`done — review ${manifestPath}, fill in response/extract schemas, then load via BARNACLE_PLUGINS or BARNACLE_PLUGINS_CONFIG_DIR (no compile step)`);
9065
+ return null;
8759
9066
  }
8760
- else {
8761
- staticBaseHeaders[k] = v;
8762
- }
8763
- }
8764
- // Third branch (browser-flow-only): a multi-action flow (isSubmissionFlow)
8765
- // that crosses hosts mid-sequence (compileActionSteps' isCrossDomain — a
8766
- // captured redirect off the original domain, e.g. an auth bounce or a
8767
- // vendor-hosted submission step). A bare `fetch`-based executeHttp can't
8768
- // reliably replay that: cookies/CSRF/session state minted for one origin
8769
- // don't automatically carry to the next the way a real browser's redirect
8770
- // handling does, so synthesizing a same-shape HTTP sequence would silently
8771
- // drop the session boundary the recon actually walked. Per-step, this
8772
- // already surfaces as the "cross-domain redirect detected ... likely needs
8773
- // browser fallback for this step" TODO (see emitMultiStepExecuteHttp); at
8774
- // the whole-flow level the honest emit is no executeHttp at all — never a
8775
- // same-host multi-step body that quietly drops the hop, and never a
8776
- // downgrade to the single-endpoint `{query}` branch either, since that's a
8777
- // fabrication of its own kind for a flow that isn't a single-action
8778
- // query/search to begin with.
8779
- const browserFlowOnly = isSubmissionFlow && actionSteps.some((s) => s.isCrossDomain);
8780
- const multiStepBody = browserFlowOnly
8781
- ? undefined
8782
- : isSubmissionFlow
8783
- ? emitMultiStepExecuteHttp(actionSteps, inputBody, errorSignals, fieldNameMap, discoveredFormFields, fieldOptionsMap, discoveredOptionFields, discoveredRawOptionFields, discoveredAdditionalBodyKeys, baseUrl, baseUrlDerivedHeaders, tenantSubdomainHeaders, formSchema, personaBindings, entryUrlParams, shieldedUuids, selectResolutions, discoveredStructuredKeys, rawCodeFields, foldReturnSpec)
8784
- : undefined;
8785
- const hasMultipartStep = actionSteps.some((s) => s.isMultipart);
8786
- const headerBindings = collectHeaderBindings(actionSteps);
8787
- // Shape inference targets the SAME call executeHttp returns — see
8788
- // selectEffectiveResponseBody — so the two surfaces can't describe different calls.
8789
- const effectiveResponseBody = selectEffectiveResponseBody(isSubmissionFlow, actionSteps, responseBody, foldReturnSpec);
8790
- // A declared foldReturn that resolves to no plan is a silent no-op otherwise
8791
- // — the flow author gets the discarding selectReturnAction path with nothing
8792
- // in the output saying their declaration never applied. A multi-step
8793
- // (submission) flow applies its fold via emitMultiStepExecuteHttp's own
8794
- // resolveFoldPlan call, entirely independent of resolveApplicableFoldPlans
8795
- // (which exists only to gate emitContractTs's single-primary hot path, and
8796
- // unconditionally reports zero plans once multiStepBody is set) — so this
8797
- // diagnostic must consult the SAME resolution each path actually applies,
8798
- // or it falsely reports "no fold plan resolved" for every multi-step flow
8799
- // with a working foldReturn.
8800
- const effectiveFoldPlanCount = multiStepBody
8801
- ? resolveFoldPlan(actionSteps, foldReturnSpec).length
8802
- : resolveApplicableFoldPlans(actionSteps, foldReturnSpec, multiStepBody).length;
8803
- if (foldReturnSpec !== null && effectiveFoldPlanCount === 0) {
8804
- logger.warn(`flow declares foldReturn (endpointPattern: ${foldReturnSpec.endpointPattern}, resultsPath: ${foldReturnSpec.resultsPath}, joinFields: ${foldReturnSpec.joinFields.join(", ")}) but no fold plan resolved — no later capture matched the endpoint pattern, resultsPath resolved to no object array, or the matched drill-down is multipart; the drill-down's response will not be folded`);
8805
- }
8806
- logger.info(`generating plugin for ${siteId} (${gql ? "GraphQL" : browserFlowOnly ? `submission flow, ${actionSteps.length} steps, browser-flow-only (cross-domain hop detected)` : isSubmissionFlow ? `submission flow, ${actionSteps.length} steps` : "single-endpoint REST"}, baseUrl: ${baseUrl})`);
8807
- if (emit === "config") {
8808
- (0, node_fs_1.mkdirSync)(outDir, { recursive: true });
8809
- (0, node_fs_1.writeFileSync)(manifestPath, emitConfigManifest({
9067
+ (0, node_fs_1.mkdirSync)(`${outDir}/flows`, { recursive: true });
9068
+ // Emit the browser flow first so the SAME payloadFieldNames set that drives
9069
+ // its `payload.<field>` splices also extends the contract's payload schema —
9070
+ // the two artifacts can't drift because one accumulator feeds both.
9071
+ const browserFlow = emitBrowserFlowTs({
8810
9072
  siteId,
8811
- displayName,
9073
+ pascal,
8812
9074
  baseUrl,
8813
9075
  flowSteps,
9076
+ isSubmissionFlow,
9077
+ hasMultipartStep,
8814
9078
  vocabulary,
9079
+ frameSelector,
9080
+ });
9081
+ const contractOpts = {
9082
+ siteId,
9083
+ displayName,
9084
+ pascal,
9085
+ baseUrl,
9086
+ // G1+G2: only the static headers (no baseUrl/tenant-subdomain references)
9087
+ // get baked into BASE_HEADERS. The rest are emitted per-call from payload.
9088
+ baseHeaders: isSubmissionFlow ? staticBaseHeaders : baseHeaders,
9089
+ minTime,
9090
+ safeRps,
9091
+ hasRateLimitProbeData,
9092
+ responseBody: effectiveResponseBody,
9093
+ // selectEffectiveResponseBody already resolves a submission flow's own
9094
+ // single terminal capture -- the multi-sample evidence gathered above
9095
+ // describes the (possibly different) read-flow winning capture and
9096
+ // doesn't apply once a submission flow has picked its own return call.
9097
+ responseBodySamples: isSubmissionFlow ? [effectiveResponseBody] : responseBodySamples,
9098
+ gql,
9099
+ gqlQuery,
9100
+ endpointPath,
9101
+ gqlOperationName: primaryGraphQLOperation
9102
+ ? (primaryGraphQLOperation.capture.operationName ??
9103
+ parsedOperationName(primaryGraphQLOperation.capture.query ?? ""))
9104
+ : (fallbackGraphQLCapture?.operationName ??
9105
+ parsedOperationName(fallbackGraphQLCapture?.query ?? "")),
9106
+ gqlVariables: primaryGraphQLOperation?.capture.variables ?? null,
9107
+ allCaptures: activeCaptures,
9108
+ auxFiles,
9109
+ multiStepBody,
9110
+ omitExecuteHttp: browserFlowOnly,
9111
+ isSubmissionFlow,
8815
9112
  inputBody,
8816
- recoveredFields: [...discoveredFormFields, ...discoveredOptionFields],
9113
+ hasMultipartStep,
9114
+ actionSteps,
9115
+ foldReturnSpec,
9116
+ discoveredFormFields,
9117
+ fieldOptionsMap,
9118
+ discoveredOptionFields,
9119
+ discoveredRawOptionFields,
9120
+ discoveredAdditionalBodyKeys,
9121
+ discoveredStructuredKeys,
9122
+ payloadFieldNames: browserFlow.payloadFieldNames,
9123
+ optionalPayloadFieldNames: browserFlow.optionalPayloadFieldNames,
9124
+ headerBindings,
9125
+ unpopulatedDeclaredVariables: primaryGraphQLOperation?.unpopulatedDeclaredVariables ?? [],
9126
+ };
9127
+ const contractCode = emitContractTs(contractOpts);
9128
+ return {
9129
+ contractCode,
9130
+ contractOpts,
9131
+ browserFlow,
9132
+ auxFiles,
9133
+ resolvedPool: actionCaptures,
8817
9134
  isSubmissionFlow,
8818
- // A submission flow is the case where the `.ts` emit carries an
8819
- // executeHttp hot path; point the manifest at where the operator drops
8820
- // the compiled module rather than silently dropping the direct path.
8821
- // Not set for the browser-flow-only branch — there is no hot path to
8822
- // compile a module for.
8823
- httpModulePath: isSubmissionFlow && !browserFlowOnly ? `./${siteId}.http.js` : undefined,
8824
- }));
8825
- logger.info(`wrote ${manifestPath}`);
8826
- logger.info(`done — review ${manifestPath}, fill in response/extract schemas, then load via BARNACLE_PLUGINS or BARNACLE_PLUGINS_CONFIG_DIR (no compile step)`);
8827
- return;
9135
+ actionSteps,
9136
+ winningCapture,
9137
+ };
8828
9138
  }
8829
- (0, node_fs_1.mkdirSync)(`${outDir}/flows`, { recursive: true });
8830
- // Emit the browser flow first so the SAME payloadFieldNames set that drives
8831
- // its `payload.<field>` splices also extends the contract's payload schema —
8832
- // the two artifacts can't drift because one accumulator feeds both.
8833
- const browserFlow = emitBrowserFlowTs({
8834
- siteId,
8835
- pascal,
8836
- baseUrl,
8837
- flowSteps,
8838
- isSubmissionFlow,
8839
- hasMultipartStep,
8840
- vocabulary,
8841
- frameSelector,
8842
- });
8843
- const contractOpts = {
8844
- siteId,
8845
- displayName,
8846
- pascal,
8847
- baseUrl,
8848
- // G1+G2: only the static headers (no baseUrl/tenant-subdomain references)
8849
- // get baked into BASE_HEADERS. The rest are emitted per-call from payload.
8850
- baseHeaders: isSubmissionFlow ? staticBaseHeaders : baseHeaders,
8851
- minTime,
8852
- safeRps,
8853
- hasRateLimitProbeData,
8854
- responseBody: effectiveResponseBody,
8855
- // selectEffectiveResponseBody already resolves a submission flow's own
8856
- // single terminal capture -- the multi-sample evidence gathered above
8857
- // describes the (possibly different) read-flow winning capture and
8858
- // doesn't apply once a submission flow has picked its own return call.
8859
- responseBodySamples: isSubmissionFlow ? [effectiveResponseBody] : responseBodySamples,
8860
- gql,
8861
- gqlQuery,
8862
- endpointPath,
8863
- gqlOperationName: primaryGraphQLOperation
8864
- ? (primaryGraphQLOperation.capture.operationName ??
8865
- parsedOperationName(primaryGraphQLOperation.capture.query ?? ""))
8866
- : (fallbackGraphQLCapture?.operationName ??
8867
- parsedOperationName(fallbackGraphQLCapture?.query ?? "")),
8868
- gqlVariables: primaryGraphQLOperation?.capture.variables ?? null,
8869
- allCaptures: captures,
8870
- auxFiles,
8871
- multiStepBody,
8872
- omitExecuteHttp: browserFlowOnly,
8873
- isSubmissionFlow,
8874
- inputBody,
8875
- hasMultipartStep,
8876
- actionSteps,
8877
- foldReturnSpec,
8878
- discoveredFormFields,
8879
- fieldOptionsMap,
8880
- discoveredOptionFields,
8881
- discoveredRawOptionFields,
8882
- discoveredAdditionalBodyKeys,
8883
- discoveredStructuredKeys,
8884
- payloadFieldNames: browserFlow.payloadFieldNames,
8885
- optionalPayloadFieldNames: browserFlow.optionalPayloadFieldNames,
8886
- headerBindings,
8887
- unpopulatedDeclaredVariables: primaryGraphQLOperation?.unpopulatedDeclaredVariables ?? [],
8888
- };
8889
- const contractCode = emitContractTs(contractOpts);
8890
- // Fails loudly rather than shipping a flow that requires a URL field it
8891
- // never reads (see assertRequiredUrlFieldsReferenced doc comment).
8892
- assertRequiredUrlFieldsReferenced(contractCode, browserFlow.code);
8893
- (0, node_fs_1.writeFileSync)(`${outDir}/contract.ts`, contractCode);
9139
+ const result = generateFromCaptures(captures);
9140
+ if (result === null)
9141
+ return; // emit === "config": already written above.
9142
+ const final = healUnreferencedUrlFieldsOnce(captures, result, generateFromCaptures);
9143
+ (0, node_fs_1.writeFileSync)(`${outDir}/contract.ts`, final.contractCode);
8894
9144
  logger.info(`wrote ${outDir}/contract.ts`);
8895
9145
  // Same opts fed to emitContractTs above, so this can never drift from what
8896
9146
  // the header used to embed.
8897
- const checklist = buildContractChecklist(contractOpts);
9147
+ const checklist = buildContractChecklist(final.contractOpts);
8898
9148
  logger.info(`review checklist for ${outDir}/contract.ts:\n${checklist.map((item) => ` [ ] ${item}`).join("\n")}`);
8899
- (0, node_fs_1.writeFileSync)(`${outDir}/flows/browser-flow.ts`, browserFlow.code);
9149
+ (0, node_fs_1.writeFileSync)(`${outDir}/flows/browser-flow.ts`, final.browserFlow.code);
8900
9150
  logger.info(`wrote ${outDir}/flows/browser-flow.ts`);
8901
9151
  (0, node_fs_1.writeFileSync)(`${outDir}/index.ts`, emitIndexTs({ siteId, pascal }));
8902
9152
  logger.info(`wrote ${outDir}/index.ts`);
8903
- if (auxFiles.length > 0) {
9153
+ if (final.auxFiles.length > 0) {
8904
9154
  (0, node_fs_1.mkdirSync)(`${outDir}/fixtures`, { recursive: true });
8905
- for (const f of auxFiles) {
9155
+ for (const f of final.auxFiles) {
8906
9156
  (0, node_fs_1.copyFileSync)((0, node_path_1.join)(auxDir, f), `${outDir}/fixtures/${f}`);
8907
9157
  }
8908
- logger.info(`copied ${auxFiles.length} fixture(s) to ${outDir}/fixtures/`);
9158
+ logger.info(`copied ${final.auxFiles.length} fixture(s) to ${outDir}/fixtures/`);
8909
9159
  }
8910
9160
  logger.info(`done — review ${outDir}/, build the package, then point BARNACLE_PLUGINS at the compiled module (no core edits required): BARNACLE_PLUGINS=./dist/sites/${siteId}/index.js pnpm start`);
8911
9161
  }