@opencraw/core 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/README.md +67 -0
  2. package/dist/index.d.ts +1 -0
  3. package/dist/index.esm.js +5302 -0
  4. package/dist/src/access/access-broker.use-case.d.ts +43 -0
  5. package/dist/src/access/access-config.error.d.ts +5 -0
  6. package/dist/src/access/access-config.repository.d.ts +11 -0
  7. package/dist/src/access/access-json-schema.mapper.d.ts +8 -0
  8. package/dist/src/access/access-plugin.contract.d.ts +48 -0
  9. package/dist/src/access/access-preset.store.d.ts +19 -0
  10. package/dist/src/access/access-profile.contract.d.ts +91 -0
  11. package/dist/src/access/index.d.ts +11 -0
  12. package/dist/src/access/render-access.mapper.d.ts +41 -0
  13. package/dist/src/access/session-id.algorithm.d.ts +8 -0
  14. package/dist/src/api-steps/extract-from-document.use-case.d.ts +27 -0
  15. package/dist/src/api-steps/index.d.ts +4 -0
  16. package/dist/src/api-steps/run-api-step.use-case.d.ts +20 -0
  17. package/dist/src/api-steps/send-request.use-case.d.ts +22 -0
  18. package/dist/src/browser-session/browser-session.config.d.ts +21 -0
  19. package/dist/src/browser-session/browser.client.d.ts +67 -0
  20. package/dist/src/browser-session/index.d.ts +5 -0
  21. package/dist/src/crawl-events/crawl-event.contract.d.ts +120 -0
  22. package/dist/src/crawl-events/event-bus.store.d.ts +10 -0
  23. package/dist/src/crawl-events/index.d.ts +4 -0
  24. package/dist/src/crawl-events/trace-line.mapper.d.ts +14 -0
  25. package/dist/src/crawl-execution/bootstrap-session.use-case.d.ts +55 -0
  26. package/dist/src/crawl-execution/crawl-options.config.d.ts +37 -0
  27. package/dist/src/crawl-execution/crawl-report.model.d.ts +25 -0
  28. package/dist/src/crawl-execution/create-crawler.use-case.d.ts +19 -0
  29. package/dist/src/crawl-execution/index.d.ts +8 -0
  30. package/dist/src/crawl-execution/rotating-runner.use-case.d.ts +57 -0
  31. package/dist/src/crawl-execution/run-crawl.use-case.d.ts +13 -0
  32. package/dist/src/crawl-execution/run-input-recipe.use-case.d.ts +40 -0
  33. package/dist/src/extraction-scope/extraction-scope.model.d.ts +73 -0
  34. package/dist/src/extraction-scope/index.d.ts +5 -0
  35. package/dist/src/extraction-scope/live-element.model.d.ts +28 -0
  36. package/dist/src/hooks/hook-registry.store.d.ts +16 -0
  37. package/dist/src/hooks/hook.contract.d.ts +17 -0
  38. package/dist/src/hooks/hook.error.d.ts +7 -0
  39. package/dist/src/hooks/index.d.ts +4 -0
  40. package/dist/src/http-session/http-response.contract.d.ts +44 -0
  41. package/dist/src/http-session/http.client.d.ts +42 -0
  42. package/dist/src/http-session/index.d.ts +5 -0
  43. package/dist/src/index.d.ts +24 -0
  44. package/dist/src/output-mapping/coerce-field.mapper.d.ts +19 -0
  45. package/dist/src/output-mapping/generated-field.mapper.d.ts +16 -0
  46. package/dist/src/output-mapping/index.d.ts +9 -0
  47. package/dist/src/output-mapping/map-record.use-case.d.ts +27 -0
  48. package/dist/src/output-mapping/mapping.error.d.ts +17 -0
  49. package/dist/src/output-mapping/missing-value.policy.d.ts +14 -0
  50. package/dist/src/output-mapping/output-field.validator.d.ts +10 -0
  51. package/dist/src/output-mapping/output-record.model.d.ts +20 -0
  52. package/dist/src/recipe-loading/index.d.ts +9 -0
  53. package/dist/src/recipe-loading/load-recipe-set.use-case.d.ts +45 -0
  54. package/dist/src/recipe-loading/read-recipe-source.use-case.d.ts +14 -0
  55. package/dist/src/recipe-loading/recipe-binding.error.d.ts +14 -0
  56. package/dist/src/recipe-loading/recipe-binding.validator.d.ts +32 -0
  57. package/dist/src/recipe-loading/recipe-file.repository.d.ts +11 -0
  58. package/dist/src/recipe-loading/recipe-set.model.d.ts +12 -0
  59. package/dist/src/recipe-loading/recipe-source.contract.d.ts +28 -0
  60. package/dist/src/recipe-loading/recipe-text.mapper.d.ts +14 -0
  61. package/dist/src/recipe-schema/index.d.ts +15 -0
  62. package/dist/src/recipe-schema/input-recipe.contract.d.ts +103 -0
  63. package/dist/src/recipe-schema/json-schema.mapper.d.ts +15 -0
  64. package/dist/src/recipe-schema/output-recipe.contract.d.ts +47 -0
  65. package/dist/src/recipe-schema/recipe-kind.enum.d.ts +31 -0
  66. package/dist/src/recipe-schema/recipe-validation.error.d.ts +17 -0
  67. package/dist/src/recipe-schema/recipe.validator.d.ts +28 -0
  68. package/dist/src/recipe-schema/step.contract.d.ts +152 -0
  69. package/dist/src/recipe-schema/transform-rule.contract.d.ts +120 -0
  70. package/dist/src/record-sink/dedupe.policy.d.ts +17 -0
  71. package/dist/src/record-sink/index.d.ts +8 -0
  72. package/dist/src/record-sink/json-lines-sink.repository.d.ts +19 -0
  73. package/dist/src/record-sink/memory-sink.repository.d.ts +9 -0
  74. package/dist/src/record-sink/record-sink.contract.d.ts +17 -0
  75. package/dist/src/selection/html-selector.algorithm.d.ts +18 -0
  76. package/dist/src/selection/index.d.ts +8 -0
  77. package/dist/src/selection/json-path.algorithm.d.ts +9 -0
  78. package/dist/src/selection/json-text.algorithm.d.ts +31 -0
  79. package/dist/src/selection/regex.algorithm.d.ts +12 -0
  80. package/dist/src/selection/take-value.mapper.d.ts +27 -0
  81. package/dist/src/step-flow/block-rule.policy.d.ts +21 -0
  82. package/dist/src/step-flow/blocked.error.d.ts +9 -0
  83. package/dist/src/step-flow/for-each.use-case.d.ts +18 -0
  84. package/dist/src/step-flow/index.d.ts +10 -0
  85. package/dist/src/step-flow/paginate.use-case.d.ts +17 -0
  86. package/dist/src/step-flow/retry.policy.d.ts +21 -0
  87. package/dist/src/step-flow/run-gate.policy.d.ts +39 -0
  88. package/dist/src/step-flow/run-steps.use-case.d.ts +39 -0
  89. package/dist/src/step-flow/step-failure.error.d.ts +14 -0
  90. package/dist/src/step-flow/step-runner.contract.d.ts +39 -0
  91. package/dist/src/template/expression.algorithm.d.ts +53 -0
  92. package/dist/src/template/index.d.ts +7 -0
  93. package/dist/src/template/template.algorithm.d.ts +53 -0
  94. package/dist/src/template/value-path.algorithm.d.ts +29 -0
  95. package/dist/src/template/value-text.algorithm.d.ts +18 -0
  96. package/dist/src/transformation/apply-transform-chain.use-case.d.ts +17 -0
  97. package/dist/src/transformation/collection.algorithm.d.ts +15 -0
  98. package/dist/src/transformation/currency.algorithm.d.ts +15 -0
  99. package/dist/src/transformation/date.algorithm.d.ts +13 -0
  100. package/dist/src/transformation/index.d.ts +10 -0
  101. package/dist/src/transformation/lookup.algorithm.d.ts +24 -0
  102. package/dist/src/transformation/number.algorithm.d.ts +30 -0
  103. package/dist/src/transformation/string.algorithm.d.ts +18 -0
  104. package/dist/src/transformation/transform-registry.store.d.ts +36 -0
  105. package/dist/src/transformation/transform.error.d.ts +8 -0
  106. package/dist/src/transformation/url.algorithm.d.ts +20 -0
  107. package/dist/src/web-steps/evaluate-script.use-case.d.ts +10 -0
  108. package/dist/src/web-steps/extract-from-page.use-case.d.ts +10 -0
  109. package/dist/src/web-steps/index.d.ts +2 -0
  110. package/dist/src/web-steps/interact.use-case.d.ts +36 -0
  111. package/dist/src/web-steps/navigate.use-case.d.ts +14 -0
  112. package/dist/src/web-steps/run-web-step.use-case.d.ts +22 -0
  113. package/dist/src/web-steps/snapshot-elements.use-case.d.ts +13 -0
  114. package/package.json +75 -0
@@ -0,0 +1,14 @@
1
+ import type { FieldSpec, MappingRule, MissingPolicy, OutputRecipe } from '../recipe-schema/index.js';
2
+ /** Whether a mapped value counts as missing. An empty list is a value. */
3
+ export declare function isMissing(value: unknown): boolean;
4
+ /**
5
+ * The policy for a missing value: mapping rule, then field, then recipe, then
6
+ * `default` when the field has one, `fail` when it is required, `null` otherwise.
7
+ *
8
+ * @param field - The output field.
9
+ * @param recipe - The output recipe.
10
+ * @param rule - The mapping rule that produced the value, if any.
11
+ * @returns The policy to apply.
12
+ */
13
+ export declare function resolveMissingPolicy(field: FieldSpec, recipe: OutputRecipe, rule?: MappingRule): MissingPolicy;
14
+ //# sourceMappingURL=missing-value.policy.d.ts.map
@@ -0,0 +1,10 @@
1
+ import type { FieldSpec } from '../recipe-schema/index.js';
2
+ /**
3
+ * Checks a present, coerced value against the field's quality rules.
4
+ *
5
+ * @param value - A non-missing coerced value.
6
+ * @param field - The output field.
7
+ * @returns Every violated rule, as messages; empty when valid.
8
+ */
9
+ export declare function validateField(value: unknown, field: FieldSpec): string[];
10
+ //# sourceMappingURL=output-field.validator.d.ts.map
@@ -0,0 +1,20 @@
1
+ /** One record a crawl produced, validated against its output recipe. */
2
+ export interface OutputRecord {
3
+ data: Record<string, unknown>;
4
+ /** The identity from the output's `key` fields, `null` when the output declares none. */
5
+ key: string | null;
6
+ source: {
7
+ recipeId: string;
8
+ url: string;
9
+ emittedAt: string;
10
+ };
11
+ }
12
+ /**
13
+ * The identity of a record: its key field values, in output order, as one string.
14
+ *
15
+ * @param data - The validated record data.
16
+ * @param keyFields - The output fields marked `key`.
17
+ * @returns The key, or `null` when there are no key fields.
18
+ */
19
+ export declare function recordKey(data: Record<string, unknown>, keyFields: readonly string[]): string | null;
20
+ //# sourceMappingURL=output-record.model.d.ts.map
@@ -0,0 +1,9 @@
1
+ export { loadRecipeSet, loadRecipes, bindRecipeSet } from './load-recipe-set.use-case.js';
2
+ export type { RecipeSetSource } from './load-recipe-set.use-case.js';
3
+ export { readRecipeSource } from './read-recipe-source.use-case.js';
4
+ export type { RecipeSource, RecipeBytes, RecipeDocument } from './recipe-source.contract.js';
5
+ export { RecipeSet } from './recipe-set.model.js';
6
+ export { validateBinding, fieldAt } from './recipe-binding.validator.js';
7
+ export { RecipeBindingError } from './recipe-binding.error.js';
8
+ export type { BindingIssue } from './recipe-binding.error.js';
9
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1,45 @@
1
+ import type { InputRecipe, OutputRecipe } from '../recipe-schema/index.js';
2
+ import type { RecipeSource } from './recipe-source.contract.js';
3
+ import { RecipeSet } from './recipe-set.model.js';
4
+ /** What to load, with the output recipe kept apart from the inputs. */
5
+ export interface RecipeSetSource {
6
+ /** The output recipe, or a source holding exactly one output recipe among others. */
7
+ output: RecipeSource;
8
+ /**
9
+ * The input recipes. The output recipe itself is skipped when found here, so
10
+ * one directory or bundle holding both can be passed as both.
11
+ */
12
+ inputs: readonly RecipeSource[];
13
+ }
14
+ /**
15
+ * Loads, validates and binds one output recipe with its input recipes.
16
+ *
17
+ * @param source - Where the recipes are; each part takes any {@link RecipeSource}.
18
+ * @returns A bound set, inputs in the order given.
19
+ * @throws RecipeValidationError when a recipe does not match its contract.
20
+ * @throws RecipeBindingError when an input cannot feed the output.
21
+ */
22
+ export declare function loadRecipeSet(source: RecipeSetSource): Promise<RecipeSet>;
23
+ /**
24
+ * Loads, validates and binds recipes from one source holding all of them, the
25
+ * output recipe told apart from the inputs by its `kind`: a directory, a JSON
26
+ * Lines file or string, a `Blob` or `Buffer` of either, or an array of recipe
27
+ * objects.
28
+ *
29
+ * @param source - Where the recipes are.
30
+ * @returns A bound set, inputs in the order found.
31
+ * @throws Error when the source does not hold exactly one output recipe.
32
+ * @throws RecipeValidationError when a recipe does not match its contract.
33
+ * @throws RecipeBindingError when an input cannot feed the output.
34
+ */
35
+ export declare function loadRecipes(source: RecipeSource): Promise<RecipeSet>;
36
+ /**
37
+ * Binds already-parsed recipes into a set.
38
+ *
39
+ * @param output - The output recipe.
40
+ * @param inputs - The input recipes.
41
+ * @returns A bound set.
42
+ * @throws RecipeBindingError when an input cannot feed the output.
43
+ */
44
+ export declare function bindRecipeSet(output: OutputRecipe, inputs: readonly InputRecipe[]): RecipeSet;
45
+ //# sourceMappingURL=load-recipe-set.use-case.d.ts.map
@@ -0,0 +1,14 @@
1
+ import type { RecipeDocument, RecipeSource } from './recipe-source.contract.js';
2
+ /**
3
+ * Reads recipes from any {@link RecipeSource}: paths, JSON or JSON Lines text,
4
+ * bytes, decoded objects, or an array mixing them. Nothing is validated yet;
5
+ * the documents keep where they came from, for error messages.
6
+ *
7
+ * @param source - Where the recipes are.
8
+ * @param label - What to call recipes that did not come from a file.
9
+ * @returns The decoded recipes, in the order given.
10
+ * @throws Error naming the source when a path is missing, text does not
11
+ * decode, or a value is not a recipe source at all.
12
+ */
13
+ export declare function readRecipeSource(source: RecipeSource, label?: string): Promise<RecipeDocument[]>;
14
+ //# sourceMappingURL=read-recipe-source.use-case.d.ts.map
@@ -0,0 +1,14 @@
1
+ /** One problem in how an input recipe binds to its output recipe. */
2
+ export interface BindingIssue {
3
+ recipeId: string;
4
+ /** Where in the input recipe: `mapping.price`, `steps.1.steps.0`, `session.bootstrap.steps.2`. */
5
+ path: string;
6
+ message: string;
7
+ }
8
+ /** An input recipe that parses but cannot feed its output recipe. */
9
+ export declare class RecipeBindingError extends Error {
10
+ readonly issues: readonly BindingIssue[];
11
+ readonly name = "RecipeBindingError";
12
+ constructor(issues: readonly BindingIssue[]);
13
+ }
14
+ //# sourceMappingURL=recipe-binding.error.d.ts.map
@@ -0,0 +1,32 @@
1
+ import type { FieldSpec, InputRecipe, OutputRecipe } from '../recipe-schema/index.js';
2
+ import type { BindingIssue } from './recipe-binding.error.js';
3
+ /**
4
+ * Checks that an input recipe can actually feed its output recipe:
5
+ *
6
+ * - every mapping key names an output field (dotted for nested objects), and an
7
+ * `each` rule targets an array of objects;
8
+ * - every `from` / `each` / `over` root is a step id bound anywhere in the
9
+ * recipe, a `forEach` variable, a `paginate` `as` binding, or one of `page`,
10
+ * `start`, `vars` (`emit` snapshots the whole scope chain, so any id can feed
11
+ * the mapping);
12
+ * - every required output field is mapped, defaulted, or generated;
13
+ * - step ids are unique along any path;
14
+ * - web-only steps appear only in web recipes or inside a bootstrap, api-only
15
+ * steps only in api recipes, and `next.selector` only in web mode;
16
+ * - exactly one emitting construct exists on any path (the two branches of an
17
+ * `if` are separate paths).
18
+ *
19
+ * @param input - A parsed input recipe.
20
+ * @param output - The parsed output recipe it names.
21
+ * @returns Every problem found; empty when the recipes bind.
22
+ */
23
+ export declare function validateBinding(input: InputRecipe, output: OutputRecipe): BindingIssue[];
24
+ /**
25
+ * The output field at a dotted target, walking into `object` members.
26
+ *
27
+ * @param fields - The output recipe's fields.
28
+ * @param target - A mapping key.
29
+ * @returns The field, or `undefined`.
30
+ */
31
+ export declare function fieldAt(fields: Record<string, FieldSpec>, target: string): FieldSpec | undefined;
32
+ //# sourceMappingURL=recipe-binding.validator.d.ts.map
@@ -0,0 +1,11 @@
1
+ import type { RecipeDocument } from './recipe-source.contract.js';
2
+ /**
3
+ * Reads one recipe file, or every `.json` and `.jsonl` file directly inside a
4
+ * directory. A file holds one recipe, an array of them, or JSON Lines.
5
+ *
6
+ * @param location - A file or directory path.
7
+ * @returns The decoded recipes, files sorted by path.
8
+ * @throws When the path does not exist or a file is neither JSON nor JSON Lines.
9
+ */
10
+ export declare function readRecipeFiles(location: string): Promise<RecipeDocument[]>;
11
+ //# sourceMappingURL=recipe-file.repository.d.ts.map
@@ -0,0 +1,12 @@
1
+ import type { InputRecipe, OutputRecipe } from '../recipe-schema/index.js';
2
+ /**
3
+ * One output recipe and the input recipes that feed it. The invariant that
4
+ * every input names this output is checked on construction; deeper binding
5
+ * rules live in the binding validator.
6
+ */
7
+ export declare class RecipeSet {
8
+ readonly output: OutputRecipe;
9
+ readonly inputs: readonly InputRecipe[];
10
+ constructor(output: OutputRecipe, inputs: readonly InputRecipe[]);
11
+ }
12
+ //# sourceMappingURL=recipe-set.model.d.ts.map
@@ -0,0 +1,28 @@
1
+ /**
2
+ * Raw bytes holding recipe JSON or JSON Lines, UTF-8: a `Buffer` or any other
3
+ * typed array, an `ArrayBuffer`, a `Blob` or `File`, or a stream of chunks (a
4
+ * Node `Readable`, a web `ReadableStream`, any async iterable of bytes or text).
5
+ */
6
+ export type RecipeBytes = Blob | ArrayBuffer | ArrayBufferView | AsyncIterable<Uint8Array | string>;
7
+ /**
8
+ * Where recipes come from. Each form holds one recipe or many:
9
+ *
10
+ * - a string starting with `{` or `[` (after whitespace): JSON text (one recipe
11
+ * or an array of them) or JSON Lines text (one recipe per line);
12
+ * - any other string: a path to a `.json` or `.jsonl` file, or a directory whose
13
+ * `.json` and `.jsonl` files are read (sorted, not recursive);
14
+ * - {@link RecipeBytes}: the same text, still encoded;
15
+ * - an object: one recipe, already decoded;
16
+ * - an array: any mix of the above.
17
+ */
18
+ export type RecipeSource = string | RecipeBytes | object | readonly RecipeSource[];
19
+ /** One decoded recipe before validation, with where it came from. */
20
+ export interface RecipeDocument {
21
+ /**
22
+ * A file path, `path:line` for a JSON Lines file, a `File`'s name, or a
23
+ * positional label (`recipes[2]`, `recipes:3`) for recipes that came from memory.
24
+ */
25
+ source: string;
26
+ content: unknown;
27
+ }
28
+ //# sourceMappingURL=recipe-source.contract.d.ts.map
@@ -0,0 +1,14 @@
1
+ import type { RecipeDocument } from './recipe-source.contract.js';
2
+ /**
3
+ * Decodes recipe text: one JSON document (a recipe, or an array of recipes) or
4
+ * JSON Lines (one recipe per non-blank line).
5
+ *
6
+ * @param text - The text.
7
+ * @param label - Where it came from, prefixed to each document's source.
8
+ * @returns The decoded recipes: `label` for a single one, `label[i]` for array
9
+ * items, `label:line` for JSON Lines.
10
+ * @throws Error naming the label (and the line, for JSON Lines) when the text
11
+ * is neither.
12
+ */
13
+ export declare function parseRecipeText(text: string, label: string): RecipeDocument[];
14
+ //# sourceMappingURL=recipe-text.mapper.d.ts.map
@@ -0,0 +1,15 @@
1
+ export * from './recipe-kind.enum.js';
2
+ export { fieldSpecSchema, outputRecipeSchema } from './output-recipe.contract.js';
3
+ export type { FieldSpec, OutputRecipe } from './output-recipe.contract.js';
4
+ export { stepSchema, errorPolicySchema, paginateNextSchema } from './step.contract.js';
5
+ export type { Step, StepType, StepBaseFields, TargetFields, ErrorPolicy, PaginateNext, TakeKind, GotoStep, ClickStep, FillStep, PressStep, SelectStep, ScrollStep, WaitStep, EvaluateStep, ScreenshotStep, RequestStep, ExtractStep, SetStep, CollectStep, ForEachStep, IfStep, PaginateStep, EmitStep, HookStep, } from './step.contract.js';
6
+ export { transformRuleSchema, mappingRuleSchema } from './transform-rule.contract.js';
7
+ export type { TransformRule, TransformOp, MappingRule, FromRule, EachRule } from './transform-rule.contract.js';
8
+ export { inputRecipeSchema, sessionSpecSchema, startPointSchema } from './input-recipe.contract.js';
9
+ export type { InputRecipe, SessionSpec, SessionBootstrap, SessionAccess, BlockRule, BlockRotation, StartPoint, CrawlLimits, RecipeCookie } from './input-recipe.contract.js';
10
+ export { parseInputRecipe, parseOutputRecipe, recipeKindOf } from './recipe.validator.js';
11
+ export { RecipeValidationError } from './recipe-validation.error.js';
12
+ export type { RecipeIssue } from './recipe-validation.error.js';
13
+ export { inputRecipeJsonSchema, outputRecipeJsonSchema } from './json-schema.mapper.js';
14
+ export type { JsonSchemaDocument } from './json-schema.mapper.js';
15
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1,103 @@
1
+ import { z } from 'zod';
2
+ import type { CrawlMode, KeepKind } from './recipe-kind.enum.js';
3
+ import type { ErrorPolicy, Step } from './step.contract.js';
4
+ import type { MappingRule } from './transform-rule.contract.js';
5
+ /** A URL the crawl starts from, with variables visible to its templates as `vars.*`. */
6
+ export interface StartPoint {
7
+ url: string;
8
+ vars?: Record<string, string | number | boolean>;
9
+ }
10
+ /** A cookie in Playwright's shape. */
11
+ export interface RecipeCookie {
12
+ name: string;
13
+ value: string;
14
+ domain: string;
15
+ path?: string;
16
+ expires?: number;
17
+ httpOnly?: boolean;
18
+ secure?: boolean;
19
+ sameSite?: 'Strict' | 'Lax' | 'None';
20
+ }
21
+ /** Runs before the crawl, always in a browser, to obtain cookies or storage (a login, a consent wall). */
22
+ export interface SessionBootstrap {
23
+ steps: Step[];
24
+ keep: KeepKind[];
25
+ /** Persist the resulting storage state for the next run. */
26
+ saveTo?: string;
27
+ }
28
+ /**
29
+ * What the site needs from the network, never how to get it: the runner's
30
+ * access config maps a profile name to a provider and credentials.
31
+ */
32
+ export interface SessionAccess {
33
+ /** An access profile of the runner's config; its default when omitted. */
34
+ profile?: string;
35
+ /** ISO 3166 country the traffic should come from, for profiles that target by country. */
36
+ country?: string;
37
+ /** `false` lets the provider rotate IPs per request; the profile decides when omitted. */
38
+ sticky?: boolean;
39
+ }
40
+ /**
41
+ * What counts as the site refusing the crawl, checked on every navigation and
42
+ * request. Any condition that matches is a block. When omitted: status 403 or
43
+ * 429, or an AWS WAF challenge (`x-amzn-waf-action: challenge`).
44
+ */
45
+ export interface BlockRule {
46
+ status?: number[];
47
+ /** Header name to a regular expression its value must match (case-insensitive). */
48
+ header?: Record<string, string>;
49
+ /** A regular expression the response body must match (case-insensitive). */
50
+ text?: string;
51
+ }
52
+ /** What to do when blocked: take a new access lease (a new IP), reopen the session, and retry the step. */
53
+ export interface BlockRotation {
54
+ rotate: boolean;
55
+ /** How many rotations a recipe run may use. Default 2. */
56
+ attempts?: number;
57
+ }
58
+ export interface SessionSpec {
59
+ headers?: Record<string, string>;
60
+ cookies?: RecipeCookie[];
61
+ userAgent?: string;
62
+ viewport?: {
63
+ width: number;
64
+ height: number;
65
+ };
66
+ /** Reuse a storage state saved by a previous bootstrap. */
67
+ storageStatePath?: string;
68
+ bootstrap?: SessionBootstrap;
69
+ access?: SessionAccess;
70
+ blockedWhen?: BlockRule;
71
+ onBlock?: BlockRotation;
72
+ }
73
+ export interface CrawlLimits {
74
+ maxRecords?: number;
75
+ /** Minimum interval between two request starts across the recipe, whatever runs in parallel. */
76
+ delayMs?: number;
77
+ timeoutMs?: number;
78
+ /** How many `forEach` iterations may run at once (api mode; a web recipe drives one page). Default 1. */
79
+ concurrency?: number;
80
+ }
81
+ /** Where to start, how to navigate, what to extract, and how it maps to one output recipe. */
82
+ export interface InputRecipe {
83
+ $schema?: string;
84
+ kind: 'input';
85
+ id: string;
86
+ /** The `OutputRecipe.id` this recipe feeds. */
87
+ output: string;
88
+ mode: CrawlMode;
89
+ description?: string;
90
+ start: StartPoint[];
91
+ vars?: Record<string, string | number | boolean>;
92
+ session?: SessionSpec;
93
+ limits?: CrawlLimits;
94
+ /** Default policy for every step. */
95
+ onError?: ErrorPolicy;
96
+ steps: Step[];
97
+ /** Keyed by output field path, dotted for nested fields. */
98
+ mapping: Record<string, MappingRule>;
99
+ }
100
+ export declare const startPointSchema: z.ZodType<StartPoint>;
101
+ export declare const sessionSpecSchema: z.ZodType<SessionSpec>;
102
+ export declare const inputRecipeSchema: z.ZodType<InputRecipe>;
103
+ //# sourceMappingURL=input-recipe.contract.d.ts.map
@@ -0,0 +1,15 @@
1
+ /** A JSON Schema document, as `z.toJSONSchema` produces it. */
2
+ export type JsonSchemaDocument = Record<string, unknown>;
3
+ /**
4
+ * The JSON Schema for output recipe files, for editors and other tools.
5
+ *
6
+ * @returns A draft 2020-12 document.
7
+ */
8
+ export declare function outputRecipeJsonSchema(): JsonSchemaDocument;
9
+ /**
10
+ * The JSON Schema for input recipe files, for editors and other tools.
11
+ *
12
+ * @returns A draft 2020-12 document.
13
+ */
14
+ export declare function inputRecipeJsonSchema(): JsonSchemaDocument;
15
+ //# sourceMappingURL=json-schema.mapper.d.ts.map
@@ -0,0 +1,47 @@
1
+ import { z } from 'zod';
2
+ import type { FieldType, GeneratedValue, MissingPolicy, RecipeMissingPolicy } from './recipe-kind.enum.js';
3
+ /** One field of the records a crawl produces: its type, quality rules and what to do when it is missing. */
4
+ export interface FieldSpec {
5
+ type: FieldType;
6
+ description?: string;
7
+ /** Default `false`. */
8
+ required?: boolean;
9
+ /** Default `false`. */
10
+ nullable?: boolean;
11
+ /** Used when the value is missing and the resolved policy is `default`. */
12
+ default?: unknown;
13
+ onMissing?: MissingPolicy;
14
+ /** Part of the record identity, used to drop duplicates. */
15
+ key?: boolean;
16
+ /** Supplied by the engine, never mapped. */
17
+ generated?: GeneratedValue;
18
+ /** Input format for `date` / `datetime`; output is always ISO 8601. */
19
+ format?: string;
20
+ /** ISO 4217 code for `currency`. */
21
+ currency?: string;
22
+ /** Allowed values for `enum`. */
23
+ values?: string[];
24
+ /** Element spec for `array`. */
25
+ items?: FieldSpec;
26
+ /** Member specs for `object`. A `json` field takes none: it keeps whatever it is given. */
27
+ fields?: Record<string, FieldSpec>;
28
+ min?: number;
29
+ max?: number;
30
+ pattern?: string;
31
+ minLength?: number;
32
+ maxLength?: number;
33
+ }
34
+ /** The schema of the records every input recipe bound to it must produce. */
35
+ export interface OutputRecipe {
36
+ $schema?: string;
37
+ kind: 'output';
38
+ id: string;
39
+ version: number;
40
+ description?: string;
41
+ fields: Record<string, FieldSpec>;
42
+ /** Recipe-wide default: `fail` for required fields, `null` otherwise. */
43
+ onMissing?: RecipeMissingPolicy;
44
+ }
45
+ export declare const fieldSpecSchema: z.ZodType<FieldSpec>;
46
+ export declare const outputRecipeSchema: z.ZodType<OutputRecipe>;
47
+ //# sourceMappingURL=output-recipe.contract.d.ts.map
@@ -0,0 +1,31 @@
1
+ /** The closed vocabularies a recipe file can use. Each is a `readonly` tuple so zod and TypeScript share it. */
2
+ export declare const RECIPE_KINDS: readonly ["input", "output"];
3
+ export declare const CRAWL_MODES: readonly ["web", "api"];
4
+ export declare const SELECTOR_KINDS: readonly ["css", "xpath", "jsonpath", "regex"];
5
+ /** `take` also accepts `attr:<name>`, which is validated by pattern rather than listed. */
6
+ export declare const TAKE_KINDS: readonly ["text", "html", "value", "json"];
7
+ export declare const BODY_KINDS: readonly ["json", "html", "text"];
8
+ export declare const FIELD_TYPES: readonly ["string", "number", "integer", "boolean", "date", "datetime", "currency", "url", "enum", "array", "object", "json"];
9
+ export declare const MISSING_POLICIES: readonly ["fail", "skip-record", "null", "default"];
10
+ export declare const RECIPE_MISSING_POLICIES: readonly ["fail", "skip-record", "null"];
11
+ export declare const ERROR_POLICIES: readonly ["fail", "skip", "retry"];
12
+ export declare const GENERATED_VALUES: readonly ["now", "uuid", "sourceUrl", "recipeId"];
13
+ export declare const KEEP_KINDS: readonly ["cookies", "localStorage"];
14
+ export declare const WAIT_UNTIL: readonly ["load", "domcontentloaded", "networkidle", "commit"];
15
+ export declare const HTTP_METHODS: readonly ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD"];
16
+ /** Steps that only make sense with a live browser page. */
17
+ export declare const WEB_ONLY_STEPS: readonly ["goto", "click", "fill", "press", "select", "scroll", "wait", "evaluate", "screenshot"];
18
+ /** Steps that only make sense against an HTTP request context. */
19
+ export declare const API_ONLY_STEPS: readonly ["request"];
20
+ export type RecipeKind = typeof RECIPE_KINDS[number];
21
+ export type CrawlMode = typeof CRAWL_MODES[number];
22
+ export type SelectorKind = typeof SELECTOR_KINDS[number];
23
+ export type BodyKind = typeof BODY_KINDS[number];
24
+ export type FieldType = typeof FIELD_TYPES[number];
25
+ export type MissingPolicy = typeof MISSING_POLICIES[number];
26
+ export type RecipeMissingPolicy = typeof RECIPE_MISSING_POLICIES[number];
27
+ export type GeneratedValue = typeof GENERATED_VALUES[number];
28
+ export type KeepKind = typeof KEEP_KINDS[number];
29
+ export type WaitUntil = typeof WAIT_UNTIL[number];
30
+ export type HttpMethod = typeof HTTP_METHODS[number];
31
+ //# sourceMappingURL=recipe-kind.enum.d.ts.map
@@ -0,0 +1,17 @@
1
+ /** One problem found in a recipe file, located by its JSON path. */
2
+ export interface RecipeIssue {
3
+ /** Dotted JSON path, `''` for the root. */
4
+ path: string;
5
+ message: string;
6
+ }
7
+ /** A recipe that does not match its contract. `issues` lists every problem, not just the first. */
8
+ export declare class RecipeValidationError extends Error {
9
+ /** Which recipe: a file path or an id when known. */
10
+ readonly source: string;
11
+ readonly issues: readonly RecipeIssue[];
12
+ readonly name = "RecipeValidationError";
13
+ constructor(
14
+ /** Which recipe: a file path or an id when known. */
15
+ source: string, issues: readonly RecipeIssue[]);
16
+ }
17
+ //# sourceMappingURL=recipe-validation.error.d.ts.map
@@ -0,0 +1,28 @@
1
+ import type { InputRecipe } from './input-recipe.contract.js';
2
+ import type { OutputRecipe } from './output-recipe.contract.js';
3
+ /**
4
+ * Parses an unknown value (a decoded JSON file) as an output recipe.
5
+ *
6
+ * @param value - The decoded JSON.
7
+ * @param source - A name for error messages, usually the file path.
8
+ * @returns The typed recipe.
9
+ * @throws RecipeValidationError when it does not match the contract.
10
+ */
11
+ export declare function parseOutputRecipe(value: unknown, source?: string): OutputRecipe;
12
+ /**
13
+ * Parses an unknown value (a decoded JSON file) as an input recipe.
14
+ *
15
+ * @param value - The decoded JSON.
16
+ * @param source - A name for error messages, usually the file path.
17
+ * @returns The typed recipe.
18
+ * @throws RecipeValidationError when it does not match the contract.
19
+ */
20
+ export declare function parseInputRecipe(value: unknown, source?: string): InputRecipe;
21
+ /**
22
+ * Reads the `kind` of a decoded recipe without validating the rest.
23
+ *
24
+ * @param value - The decoded JSON.
25
+ * @returns `'input'`, `'output'`, or `undefined` when it is neither.
26
+ */
27
+ export declare function recipeKindOf(value: unknown): 'input' | 'output' | undefined;
28
+ //# sourceMappingURL=recipe.validator.d.ts.map
@@ -0,0 +1,152 @@
1
+ import { z } from 'zod';
2
+ import { TAKE_KINDS } from './recipe-kind.enum.js';
3
+ import type { BodyKind, HttpMethod, SelectorKind, WaitUntil } from './recipe-kind.enum.js';
4
+ /** What to do when a step fails. Resolved step -> recipe -> `fail`. */
5
+ export type ErrorPolicy = {
6
+ policy: 'fail';
7
+ } | {
8
+ policy: 'skip';
9
+ } | {
10
+ policy: 'retry';
11
+ attempts: number;
12
+ backoffMs?: number;
13
+ };
14
+ /** What a value is taken from an element or JSON node: text, inner HTML, an attribute, an input value, or the node itself. */
15
+ export type TakeKind = typeof TAKE_KINDS[number] | `attr:${string}`;
16
+ /** How `paginate` finds the next page once the page body has run. */
17
+ export type PaginateNext = {
18
+ selector: string;
19
+ } | {
20
+ url: string;
21
+ } | {
22
+ jsonpath: string;
23
+ as?: string;
24
+ };
25
+ export interface StepBaseFields {
26
+ /** Names the value this step produces. */
27
+ id?: string;
28
+ onError?: ErrorPolicy;
29
+ /** A template; the step runs only when it renders truthy. */
30
+ when?: string;
31
+ }
32
+ export interface GotoStep extends StepBaseFields {
33
+ type: 'goto';
34
+ url: string;
35
+ waitUntil?: WaitUntil;
36
+ }
37
+ /** Where an interaction lands: a selector, or a `target` template that renders to a live element (a `forEach` over `selector`) or to a selector string. */
38
+ export interface TargetFields {
39
+ selector?: string;
40
+ target?: string;
41
+ }
42
+ export interface ClickStep extends StepBaseFields, TargetFields {
43
+ type: 'click';
44
+ optional?: boolean;
45
+ }
46
+ export interface FillStep extends StepBaseFields, TargetFields {
47
+ type: 'fill';
48
+ value: string;
49
+ }
50
+ export interface PressStep extends StepBaseFields, TargetFields {
51
+ type: 'press';
52
+ key: string;
53
+ }
54
+ /** Picks an option of a `<select>` by value, label or index. */
55
+ export interface SelectStep extends StepBaseFields, TargetFields {
56
+ type: 'select';
57
+ value?: string;
58
+ label?: string;
59
+ index?: number;
60
+ }
61
+ export interface ScrollStep extends StepBaseFields {
62
+ type: 'scroll';
63
+ to: string;
64
+ times?: number;
65
+ untilStable?: boolean;
66
+ }
67
+ export interface WaitStep extends StepBaseFields {
68
+ type: 'wait';
69
+ selector?: string;
70
+ ms?: number;
71
+ state?: 'networkidle';
72
+ }
73
+ export interface EvaluateStep extends StepBaseFields {
74
+ type: 'evaluate';
75
+ script: string;
76
+ }
77
+ export interface ScreenshotStep extends StepBaseFields {
78
+ type: 'screenshot';
79
+ path: string;
80
+ }
81
+ export interface RequestStep extends StepBaseFields {
82
+ type: 'request';
83
+ method?: HttpMethod;
84
+ url: string;
85
+ query?: Record<string, string>;
86
+ headers?: Record<string, string>;
87
+ body?: unknown;
88
+ as?: BodyKind;
89
+ }
90
+ export interface ExtractStep extends StepBaseFields {
91
+ type: 'extract';
92
+ selector: string;
93
+ kind: SelectorKind;
94
+ take?: TakeKind;
95
+ many?: boolean;
96
+ /** Id of a document or fragment to read instead of the current document. */
97
+ from?: string;
98
+ }
99
+ export interface SetStep extends StepBaseFields {
100
+ type: 'set';
101
+ value: unknown;
102
+ }
103
+ /**
104
+ * Appends to a list bound in an enclosing scope (a `set` to `[]` before the
105
+ * loop), so values gathered page by page or item by item outlive the child
106
+ * scope that found them. A list value is appended item by item.
107
+ */
108
+ export interface CollectStep extends StepBaseFields {
109
+ type: 'collect';
110
+ into: string;
111
+ value: unknown;
112
+ }
113
+ /** Runs a body per item of a list (`over`) or per live element matching `selector` (web mode; the elements are re-resolved on every use). */
114
+ export interface ForEachStep extends StepBaseFields {
115
+ type: 'forEach';
116
+ over?: string;
117
+ selector?: string;
118
+ as: string;
119
+ steps: Step[];
120
+ emit?: true | {
121
+ output: string;
122
+ };
123
+ }
124
+ /** Runs `steps` when `test` renders truthy, else `else`; both in the current scope. */
125
+ export interface IfStep extends StepBaseFields {
126
+ type: 'if';
127
+ test: string;
128
+ steps: Step[];
129
+ else?: Step[];
130
+ }
131
+ export interface PaginateStep extends StepBaseFields {
132
+ type: 'paginate';
133
+ next: PaginateNext;
134
+ until?: string;
135
+ maxPages?: number;
136
+ steps: Step[];
137
+ }
138
+ export interface EmitStep extends StepBaseFields {
139
+ type: 'emit';
140
+ output?: string;
141
+ }
142
+ export interface HookStep extends StepBaseFields {
143
+ type: 'hook';
144
+ name: string;
145
+ args?: Record<string, unknown>;
146
+ }
147
+ export type Step = GotoStep | ClickStep | FillStep | PressStep | SelectStep | ScrollStep | WaitStep | EvaluateStep | ScreenshotStep | RequestStep | ExtractStep | SetStep | CollectStep | ForEachStep | IfStep | PaginateStep | EmitStep | HookStep;
148
+ export type StepType = Step['type'];
149
+ export declare const errorPolicySchema: z.ZodType<ErrorPolicy>;
150
+ export declare const paginateNextSchema: z.ZodType<PaginateNext>;
151
+ export declare const stepSchema: z.ZodType<Step>;
152
+ //# sourceMappingURL=step.contract.d.ts.map