@enricai/barnacle 1.4.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/scraper/flow-runner.d.ts +12 -0
- package/dist/scraper/flow-runner.d.ts.map +1 -1
- package/dist/scraper/flow-runner.js +37 -0
- package/dist/scraper/flow-runner.js.map +1 -1
- package/dist/scripts/recon-generate.d.ts +22 -2
- package/dist/scripts/recon-generate.d.ts.map +1 -1
- package/dist/scripts/recon-generate.js +121 -25
- package/dist/scripts/recon-generate.js.map +1 -1
- package/package.json +5 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"recon-generate.d.ts","sourceRoot":"","sources":["../../src/scripts/recon-generate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAiBH,OAAO,EAAoB,KAAK,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAGL,KAAK,OAAO,EAKb,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"recon-generate.d.ts","sourceRoot":"","sources":["../../src/scripts/recon-generate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAiBH,OAAO,EAAoB,KAAK,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,EAGL,KAAK,OAAO,EAKb,MAAM,wBAAwB,CAAC;AA2EhC;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,uBAAuB,CACrC,WAAW,EAAE,MAAM,EACnB,QAAQ,CAAC,EAAE,MAAM,EACjB,SAAS,CAAC,EAAE,OAAO,EACnB,UAAU,GAAE,eAAmD,GAC9D,MAAM,GAAG,IAAI,CAoBf;AAUD,UAAU,SAAS;IACjB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,SAAS,OAAO,EAAE,EAC3B,KAAK,SAAI,EACT,MAAM,SAAK,EACX,IAAI,GAAE,SAAc,GACnB,MAAM,CA+DR;AAoCD;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,mBAAmB,CAAC,CAAC,SAAS;IAAE,OAAO,EAAE,OAAO,CAAA;CAAE,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,GAAG,CAAC,GAAG,IAAI,CAqBjG;AA6JD,UAAU,aAAa;IACrB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf;AAED;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,MAAM,GAAG,aAAa,EAAE,CA4B3F;AAyBD,UAAU,UAAU;IAClB,6EAA6E;IAC7E,KAAK,EAAE,MAAM,CAAC;IACd,gFAAgF;IAChF,WAAW,EAAE,MAAM,CAAC;IACpB;4DACwD;IACxD,IAAI,EAAE,MAAM,EAAE,CAAC;IACf;;uEAEmE;IACnE,YAAY,CAAC,EAAE;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9D;AAmFD,UAAU,YAAY;IACpB;iFAC6E;IAC7E,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC;;uDAEmD;IACnD,gBAAgB,EAAE,KAAK,CAAC;QAAE,UAAU,EAAE,MAAM,EAAE,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACrE;AAiFD;;;;;GAKG;AACH,KAAK,YAAY,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAExC;;;;;;;;;GASG;AACH,UAAU,mBAAmB;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACrD;AACD,KAAK,eAAe,GAAG,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC;AAsfxD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH;8EAC8E;AAC9E,wBAAgB,gBAAgB,CAC9B,QAAQ,EAAE,OAAO,EAAE,EACnB,aAAa,GAAE,GAAG,CAAC,MAAM,CAAa,EACtC,oBAAoB,GAAE,GAAG,CAAC,MAAM,CAAa,GAC5C,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CA2DzB;AAED;4EAC4E;AAC5E,UAAU,WAAW;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AAED;;;;;;0BAM0B;AAC1B,UAAU,aAAa;IACrB,IAAI,EAAE,QAAQ,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,KAAK,OAAO,GAAG,WAAW,GAAG,aAAa,CAAC;AAE3C,UAAU,UAAU;IAClB,4CAA4C;IAC5C,OAAO,EAAE,OAAO,CAAC;IACjB,mEAAmE;IACnE,OAAO,EAAE,MAAM,CAAC;IAChB;;sFAEkF;IAClF,QAAQ,EAAE,OAAO,EAAE,CAAC;IACpB,yEAAyE;IACzE,WAAW,EAAE,OAAO,CAAC;IACrB,kFAAkF;IAClF,aAAa,EAAE,OAAO,CAAC;CACxB;AAmDD;;;;GAIG;AACH,0DAA0D;AAC1D,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,aAAa,EAAE,EACxB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,GAClC,UAAU,EAAE,CAoGd;AAwKD;;;;;;;;GAQG;AACH,wBAAgB,0BAA0B,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CASrE;AA2QD;6DAC6D;AAC7D,wBAAgB,wBAAwB,CACtC,OAAO,EAAE,UAAU,EAAE,EACrB,SAAS,EAAE,OAAO,EAClB,YAAY,EAAE,YAAY,EAC1B,YAAY,EAAE,YAAY,EAC1B,mBAAmB,EAAE,GAAG,CAAC,MAAM,CAAC,EAChC,eAAe,EAAE,eAAe,EAChC,yBAAyB,EAAE,GAAG,CAAC,MAAM,CAAC,EACtC,4BAA4B,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EACjD,+BAA+B,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC,EAC7E,OAAO,EAAE,MAAM,EACf,qBAAqB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAC1C,sBAAsB,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAC3C,mBAAmB,GAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAa,GACnD,MAAM,CAkTR;AAkCD;yEACyE;AACzE,wBAAgB,cAAc,CAAC,IAAI,EAAE;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,GAAG,EAAE,OAAO,CAAC;IACb,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,iGAAiG;IACjG,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,mGAAmG;IACnG,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,gGAAgG;IAChG,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;qDAEiD;IACjD,oBAAoB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACnC;;;mCAG+B;IAC/B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC;sEACkE;IAClE,sBAAsB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACrC;;;;sDAIkD;IAClD,yBAAyB,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChD;;4EAEwE;IACxE,4BAA4B,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC,CAAC;IAC5E;;;+DAG2D;IAC3D,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAChC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;sEAGkE;IAClE,cAAc,CAAC,EAAE,aAAa,EAAE,CAAC;CAClC,GAAG,MAAM,CAmUT;AAED,qFAAqF;AACrF,KAAK,aAAa,GACd,MAAM,GACN;IACE,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,CAAC;AAyFN;;;;;;;;;;;;;GAaG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE;IACvC,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,aAAa,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,eAAe,CAAC;IAC7B,kFAAkF;IAClF,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,gFAAgF;IAChF,eAAe,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;CACpC,GAAG,MAAM,CAiET;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,aAAa,EAAE,CAAC;IAC3B,gBAAgB,EAAE,OAAO,CAAC;IAC1B,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,UAAU,CAAC,EAAE,eAAe,CAAC;CAC9B,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,iBAAiB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;CAAE,CAiInD;AAED;mEACmE;AACnE,wBAAgB,WAAW,CAAC,IAAI,EAAE;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAgB5E"}
|
|
@@ -20,6 +20,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
20
20
|
exports.resolveStepPayloadField = resolveStepPayloadField;
|
|
21
21
|
exports.inferZodSchemaFromSamples = inferZodSchemaFromSamples;
|
|
22
22
|
exports.selectPayloadAction = selectPayloadAction;
|
|
23
|
+
exports.extractActionSequence = extractActionSequence;
|
|
23
24
|
exports.indexStateValues = indexStateValues;
|
|
24
25
|
exports.compileActionSteps = compileActionSteps;
|
|
25
26
|
exports.loadQuestionPromptKeywords = loadQuestionPromptKeywords;
|
|
@@ -38,6 +39,19 @@ const load_vocabulary_1 = require("../recon/load-vocabulary");
|
|
|
38
39
|
const vocabulary_1 = require("../recon/vocabulary");
|
|
39
40
|
const recon_shared_1 = require("../scripts/recon-shared");
|
|
40
41
|
const logger = (0, logging_1.getScriptLogger)("recon-generate");
|
|
42
|
+
/**
|
|
43
|
+
* Engine imports in GENERATED code must be package subpaths, never the `@/`
|
|
44
|
+
* alias, and the reason is not visible from the source: `tsc-alias` rewrites by
|
|
45
|
+
* text, so it cannot tell an import this module *uses* from one it *emits as a
|
|
46
|
+
* string*. Written as `@/scraper/session`, the build silently rewrote the
|
|
47
|
+
* template literal itself — shipping `dist/` emitters that generated
|
|
48
|
+
* `../scraper/session` and left every out-of-tree consumer with TS2307.
|
|
49
|
+
* (`@/sites/...` survived only because `src/sites/` is empty, so it resolved to
|
|
50
|
+
* no file.) A bare specifier has nothing to resolve against, so the build leaves
|
|
51
|
+
* it alone. `out-of-tree-e2e.test.ts` asserts this against the BUILT dist —
|
|
52
|
+
* asserting it against the source would pass while the shipped artifact is broken.
|
|
53
|
+
*/
|
|
54
|
+
const ENGINE_PKG = "@enricai/barnacle";
|
|
41
55
|
// ── helpers ──────────────────────────────────────────────────────────────────
|
|
42
56
|
function toPascalCase(siteId) {
|
|
43
57
|
return siteId
|
|
@@ -410,9 +424,29 @@ const TELEMETRY_URL_PATTERNS = [
|
|
|
410
424
|
.map((p) => p.trim())
|
|
411
425
|
.filter(Boolean),
|
|
412
426
|
];
|
|
427
|
+
/**
|
|
428
|
+
* A POST to a path whose own segment is `error`/`errors` is a client-side
|
|
429
|
+
* reporting sink, never a call a caller wants replayed.
|
|
430
|
+
*
|
|
431
|
+
* Emitting one is worse than noise. A browser's error reports are frozen at
|
|
432
|
+
* recon time, so the generated plugin re-POSTs a crash that never happened —
|
|
433
|
+
* a stack trace and timestamp from the recon run, sent to the site on every
|
|
434
|
+
* invocation, describing a failure in a page the plugin never loaded.
|
|
435
|
+
*
|
|
436
|
+
* Matched on a whole path segment rather than by substring so `/error-codes`
|
|
437
|
+
* and `/terrorism-screening` stay data endpoints, and kept out of
|
|
438
|
+
* TELEMETRY_URL_PATTERNS because that list is literal substrings — a site's own
|
|
439
|
+
* sink is structural, not an ad-tech domain the operator must enumerate.
|
|
440
|
+
*/
|
|
441
|
+
const ERROR_SINK_PATH_SEGMENT = /(^|\/)errors?(\/|$)/i;
|
|
413
442
|
/**
|
|
414
443
|
* Extracts the ordered sequence of meaningful POSTs that represent the
|
|
415
|
-
* transactional flow
|
|
444
|
+
* transactional flow: same-host 2xx POSTs, minus telemetry and error-reporting
|
|
445
|
+
* sinks. Assets need no filter of their own — they arrive as GETs.
|
|
446
|
+
*
|
|
447
|
+
* Exported for tests: this predicate decides what a generated plugin will POST
|
|
448
|
+
* at a live site, and it is the only gate between a browser's incidental
|
|
449
|
+
* chatter and the emitted hot path.
|
|
416
450
|
*/
|
|
417
451
|
function extractActionSequence(captures, baseUrl) {
|
|
418
452
|
let host;
|
|
@@ -440,6 +474,13 @@ function extractActionSequence(captures, baseUrl) {
|
|
|
440
474
|
return false;
|
|
441
475
|
if (TELEMETRY_URL_PATTERNS.some((p) => capture.url.includes(p)))
|
|
442
476
|
return false;
|
|
477
|
+
try {
|
|
478
|
+
if (ERROR_SINK_PATH_SEGMENT.test(new URL(capture.url).pathname))
|
|
479
|
+
return false;
|
|
480
|
+
}
|
|
481
|
+
catch {
|
|
482
|
+
return false;
|
|
483
|
+
}
|
|
443
484
|
return true;
|
|
444
485
|
});
|
|
445
486
|
}
|
|
@@ -667,13 +708,16 @@ function detectFormSchemaFieldNames(captures) {
|
|
|
667
708
|
return { fieldNameMap, fieldOptionsMap, allSchemaUuids };
|
|
668
709
|
}
|
|
669
710
|
/**
|
|
670
|
-
* Walks a response body collecting
|
|
671
|
-
*
|
|
672
|
-
* `Id` of an object that has `OptionSourceCode`/`StringKey`/`FieldOptions`
|
|
673
|
-
* sibling — i.e. an OptionId in the form schema. These UUIDs are stable
|
|
711
|
+
* Walks a response body collecting UUID-shaped strings under a `FieldId` key, or
|
|
712
|
+
* under the `Id` of an entry in a sibling `FieldOptions` array. These are stable
|
|
674
713
|
* schema anchors that must be shielded from state-threading even when
|
|
675
|
-
* detectFormSchemaFieldNames
|
|
676
|
-
*
|
|
714
|
+
* detectFormSchemaFieldNames emits no payload-mappable name for the field (e.g.
|
|
715
|
+
* when the field's FieldName is too long for our naming heuristic).
|
|
716
|
+
*
|
|
717
|
+
* The key names are exact by design, not an oversight: a differing wire format
|
|
718
|
+
* (a lowercase `fieldId`, another vendor's option key) is the consumer's to
|
|
719
|
+
* declare, not the engine's to guess — see issue #57. Matching case variants
|
|
720
|
+
* here would re-broaden the very fingerprint that issue exists to narrow.
|
|
677
721
|
*/
|
|
678
722
|
function walkForSchemaUuids(value, out) {
|
|
679
723
|
if (value === null || typeof value !== "object")
|
|
@@ -2013,6 +2057,13 @@ function emitContractTs(opts) {
|
|
|
2013
2057
|
// Multi-step plugins thread responses through many different shapes that a
|
|
2014
2058
|
// single Zod schema can't cover — use z.unknown() so each per-step access
|
|
2015
2059
|
// compiles cleanly. Single-endpoint plugins keep the inferred schema.
|
|
2060
|
+
//
|
|
2061
|
+
// This is deliberate, not an unfinished schema: a submission flow's terminal
|
|
2062
|
+
// shape is the plugin's OWN contract with its caller (e.g. { verified: boolean }),
|
|
2063
|
+
// a field that appears in zero captured responses. Inferring a schema from the
|
|
2064
|
+
// captures would emit the wrong shape with false confidence. z.unknown() plus
|
|
2065
|
+
// the generated `[ ] Narrow ResponseSchema` checklist item is the intended
|
|
2066
|
+
// hand-off to the plugin author, who alone knows that contract.
|
|
2016
2067
|
const responseSchemaExpr = multiStepBody ? `z.unknown()` : inferZodSchema(responseBody);
|
|
2017
2068
|
// Multi-step flows that include a multipart upload need the binary asset
|
|
2018
2069
|
// on the payload. Add Resume/ResumeContentType/ResumeFilename as required
|
|
@@ -2125,24 +2176,24 @@ function emitContractTs(opts) {
|
|
|
2125
2176
|
? `${basePayloadSchemaExpr}.extend({\n Resume: z.instanceof(Buffer),\n ResumeContentType: z.string(),\n ResumeFilename: z.string(),\n})${formFieldsExtension}${splicedFieldsExtension}${optionSchemaExtension}${rawOptionSchemaExtension}${additionalBodyKeysExtension}${answersExtension}`
|
|
2126
2177
|
: `${basePayloadSchemaExpr}${formFieldsExtension}${splicedFieldsExtension}${optionSchemaExtension}${rawOptionSchemaExtension}${additionalBodyKeysExtension}${answersExtension}`;
|
|
2127
2178
|
// When the payload schema uses multipartBoolean(), import the shared helper
|
|
2128
|
-
//
|
|
2129
|
-
//
|
|
2179
|
+
// so the generated file resolves the reference and doesn't re-inline the
|
|
2180
|
+
// preprocess expression per boolean field.
|
|
2130
2181
|
const multipartBoolImport = hasMultipartStep
|
|
2131
|
-
? `import { multipartBoolean } from "
|
|
2182
|
+
? `import { multipartBoolean } from "${ENGINE_PKG}/lib/zod-multipart";\n`
|
|
2132
2183
|
: "";
|
|
2133
2184
|
// Content-Type must be absent from multipart fetch calls so FormData can inject the boundary.
|
|
2134
2185
|
const caseInsensitiveHeadersImport = hasMultipartStep
|
|
2135
|
-
? `import { omitHeaderCaseInsensitive } from "
|
|
2186
|
+
? `import { omitHeaderCaseInsensitive } from "${ENGINE_PKG}/lib/case-insensitive-headers";\n`
|
|
2136
2187
|
: "";
|
|
2137
2188
|
// Emit identifier-shaped keys unquoted so Biome's formatter doesn't rewrite
|
|
2138
2189
|
// the generated file on first lint:fix.
|
|
2139
2190
|
const headersLiteral = Object.entries(baseHeaders)
|
|
2140
2191
|
.map(([k, v]) => ` ${isValidJsIdentifier(k) ? k : JSON.stringify(k)}: ${JSON.stringify(v)}`)
|
|
2141
2192
|
.join(",\n");
|
|
2142
|
-
const fixtureImport = auxFiles.length > 0 ? `// import { loadFixture } from "
|
|
2193
|
+
const fixtureImport = auxFiles.length > 0 ? `// import { loadFixture } from "${ENGINE_PKG}/scraper/fixtures";\n` : "";
|
|
2143
2194
|
const clientImport = gql
|
|
2144
|
-
? `import { createGraphqlClient } from "
|
|
2145
|
-
: `import { createHttpClient } from "
|
|
2195
|
+
? `import { createGraphqlClient } from "${ENGINE_PKG}/scraper/graphql-client";`
|
|
2196
|
+
: `import { createHttpClient } from "${ENGINE_PKG}/scraper/http-client";`;
|
|
2146
2197
|
const queryConst = gql && gqlQuery
|
|
2147
2198
|
? `\n// Lifted verbatim from recon capture — trim UI-only fields before shipping.\nconst ${pascal.toUpperCase()}_QUERY = \`${gqlQuery.trim()}\`;\n`
|
|
2148
2199
|
: "";
|
|
@@ -2206,8 +2257,8 @@ import Bottleneck from "bottleneck";
|
|
|
2206
2257
|
import { z } from "zod/v4";
|
|
2207
2258
|
|
|
2208
2259
|
${fixtureImport}${caseInsensitiveHeadersImport}${multipartBoolImport}${clientImport}
|
|
2209
|
-
import type { BrowserSession } from "
|
|
2210
|
-
import type { SitePlugin, SitePluginContext, SitePluginResult } from "
|
|
2260
|
+
import type { BrowserSession } from "${ENGINE_PKG}/scraper/session";
|
|
2261
|
+
import type { SitePlugin, SitePluginContext, SitePluginResult } from "${ENGINE_PKG}/site-plugin";
|
|
2211
2262
|
import { run${pascal}BrowserFlow } from "@/sites/${siteId}/flows/browser-flow";
|
|
2212
2263
|
|
|
2213
2264
|
const BASE_HEADERS: Record<string, string> = {
|
|
@@ -2333,16 +2384,43 @@ function buildManifestInstruction(instruction, field) {
|
|
|
2333
2384
|
`{{ .request.${field} }}` +
|
|
2334
2385
|
instruction.slice(m.index + m[0].length));
|
|
2335
2386
|
}
|
|
2387
|
+
/**
|
|
2388
|
+
* The JSON Schema `type` keyword for a sample value. Just the keyword, not a
|
|
2389
|
+
* full schema: the manifest is a scaffold a human narrows, so it needs the real
|
|
2390
|
+
* type a caller must send (`page` is a number, `filters` an array) without
|
|
2391
|
+
* duplicating {@link inferZodSchema}'s recursive shape inference. `null` and
|
|
2392
|
+
* `undefined` fall back to `string`, the safe default for a field a caller fills.
|
|
2393
|
+
*/
|
|
2394
|
+
function jsonSchemaTypeOf(value) {
|
|
2395
|
+
if (Array.isArray(value))
|
|
2396
|
+
return "array";
|
|
2397
|
+
if (value === null || value === undefined)
|
|
2398
|
+
return "string";
|
|
2399
|
+
const t = typeof value;
|
|
2400
|
+
if (t === "number")
|
|
2401
|
+
return "number";
|
|
2402
|
+
if (t === "boolean")
|
|
2403
|
+
return "boolean";
|
|
2404
|
+
if (t === "object")
|
|
2405
|
+
return "object";
|
|
2406
|
+
return "string";
|
|
2407
|
+
}
|
|
2336
2408
|
/**
|
|
2337
2409
|
* Emits a config-only plugin manifest (`<siteId>.plugin.json`) from the recon
|
|
2338
2410
|
* flow, as an alternative to the `.ts` trio for browser-only sites. Reuses the
|
|
2339
2411
|
* SAME `resolveStepPayloadField` splice logic as the browser-flow emitter, so
|
|
2340
2412
|
* every `{{ .request.<field> }}` reference also lands in the manifest's request
|
|
2341
|
-
* schema — the two cannot drift.
|
|
2342
|
-
*
|
|
2413
|
+
* schema — the two cannot drift.
|
|
2414
|
+
*
|
|
2415
|
+
* `recovered` carries the request contract the `.ts` path infers from real
|
|
2416
|
+
* captures — the first POST body's fields plus form-schema discoveries — so
|
|
2417
|
+
* `--emit config` no longer throws that away and emit a request schema built
|
|
2418
|
+
* only from the handful of flow-step splice hints. The direct-HTTP hot path is
|
|
2419
|
+
* still omitted; a site that needs it keeps the `.ts` path or wires
|
|
2420
|
+
* `spec.httpModule` by hand.
|
|
2343
2421
|
*/
|
|
2344
2422
|
function emitConfigManifest(opts) {
|
|
2345
|
-
const { siteId, displayName, baseUrl, flowSteps, vocabulary } = opts;
|
|
2423
|
+
const { siteId, displayName, baseUrl, flowSteps, vocabulary, inputBody, recoveredFields } = opts;
|
|
2346
2424
|
const payloadFieldNames = new Set();
|
|
2347
2425
|
const steps = flowSteps.map((step) => {
|
|
2348
2426
|
const isObj = typeof step !== "string";
|
|
@@ -2358,14 +2436,30 @@ function emitConfigManifest(opts) {
|
|
|
2358
2436
|
return rewritten;
|
|
2359
2437
|
return { step: rewritten, optional, upload, submitStep };
|
|
2360
2438
|
});
|
|
2361
|
-
|
|
2439
|
+
// The request surface, widest wins: a flow splice, a recovered form field, or
|
|
2440
|
+
// a key from the first POST body all name something a caller controls. Splices
|
|
2441
|
+
// and recovered fields are strings (the browser flow fills them as text); a
|
|
2442
|
+
// body key keeps its captured type so a caller sends `page: 1`, not `"1"`.
|
|
2443
|
+
const requestProperties = {};
|
|
2444
|
+
for (const name of payloadFieldNames)
|
|
2445
|
+
requestProperties[name] = { type: "string" };
|
|
2446
|
+
for (const name of recoveredFields ?? [])
|
|
2447
|
+
requestProperties[name] = { type: "string" };
|
|
2448
|
+
if (inputBody !== null && typeof inputBody === "object" && !Array.isArray(inputBody)) {
|
|
2449
|
+
for (const [name, value] of Object.entries(inputBody)) {
|
|
2450
|
+
requestProperties[name] = { type: jsonSchemaTypeOf(value) };
|
|
2451
|
+
}
|
|
2452
|
+
}
|
|
2453
|
+
const sortedRequestProperties = Object.fromEntries(Object.keys(requestProperties)
|
|
2454
|
+
.sort()
|
|
2455
|
+
.map((name) => [name, requestProperties[name]]));
|
|
2362
2456
|
const manifest = {
|
|
2363
2457
|
apiVersion: plugin_manifest_envelope_1.CONFIG_PLUGIN_API_VERSION,
|
|
2364
2458
|
kind: plugin_manifest_envelope_1.CONFIG_PLUGIN_KIND,
|
|
2365
2459
|
metadata: { siteId, displayName },
|
|
2366
2460
|
spec: {
|
|
2367
2461
|
defaultBaseUrl: baseUrl,
|
|
2368
|
-
request: { type: "object", properties:
|
|
2462
|
+
request: { type: "object", properties: sortedRequestProperties },
|
|
2369
2463
|
response: {
|
|
2370
2464
|
type: "object",
|
|
2371
2465
|
description: "TODO: declare the fields this site returns (recon leaves this empty).",
|
|
@@ -2440,10 +2534,10 @@ function emitBrowserFlowTs(opts) {
|
|
|
2440
2534
|
import type { Stagehand } from "@browserbasehq/stagehand";
|
|
2441
2535
|
import { z } from "zod/v4";
|
|
2442
2536
|
|
|
2443
|
-
import { buildAnthropicClient } from "
|
|
2444
|
-
import { getLogger } from "
|
|
2445
|
-
import { type HealingFlowStep, runHealingFlow, waitForSpaReady } from "
|
|
2446
|
-
import { guardedExtract } from "
|
|
2537
|
+
import { buildAnthropicClient } from "${ENGINE_PKG}/lib/llm/anthropic-client";
|
|
2538
|
+
import { getLogger } from "${ENGINE_PKG}/lib/logging";
|
|
2539
|
+
import { type HealingFlowStep, runHealingFlow, waitForSpaReady } from "${ENGINE_PKG}/scraper/flow-runner";
|
|
2540
|
+
import { guardedExtract } from "${ENGINE_PKG}/scraper/stagehand-guard";
|
|
2447
2541
|
import type { ${pascal}Payload, ${pascal}Response } from "@/sites/${siteId}/contract";
|
|
2448
2542
|
|
|
2449
2543
|
const logger = getLogger({ name: "${siteId}-browser-flow" });
|
|
@@ -2861,6 +2955,8 @@ async function main() {
|
|
|
2861
2955
|
baseUrl,
|
|
2862
2956
|
flowSteps,
|
|
2863
2957
|
vocabulary,
|
|
2958
|
+
inputBody,
|
|
2959
|
+
recoveredFields: [...discoveredFormFields, ...discoveredOptionFields],
|
|
2864
2960
|
}));
|
|
2865
2961
|
logger.info(`wrote ${manifestPath}`);
|
|
2866
2962
|
logger.info(`done — review ${manifestPath}, fill in response/extract schemas, then load via BARNACLE_PLUGINS or BARNACLE_PLUGINS_CONFIG_DIR (no compile step)`);
|