@opencraw/core 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/access/access-broker.use-case.d.ts +43 -0
- package/dist/access/access-config.error.d.ts +5 -0
- package/dist/access/access-config.repository.d.ts +11 -0
- package/dist/access/access-json-schema.mapper.d.ts +8 -0
- package/dist/access/access-plugin.contract.d.ts +48 -0
- package/dist/access/access-preset.store.d.ts +19 -0
- package/dist/access/access-profile.contract.d.ts +95 -0
- package/dist/access/index.d.ts +11 -0
- package/dist/access/render-access.mapper.d.ts +41 -0
- package/dist/access/session-id.algorithm.d.ts +8 -0
- package/dist/api-steps/extract-from-document.use-case.d.ts +41 -0
- package/dist/api-steps/index.d.ts +4 -0
- package/dist/api-steps/run-api-step.use-case.d.ts +20 -0
- package/dist/api-steps/send-request.use-case.d.ts +23 -0
- package/dist/browser-session/browser-profile.store.d.ts +52 -0
- package/dist/browser-session/browser-session.config.d.ts +21 -0
- package/dist/browser-session/browser.client.d.ts +75 -0
- package/dist/browser-session/index.d.ts +6 -0
- package/dist/captcha/captcha-budget.model.d.ts +21 -0
- package/dist/captcha/captcha-detection.client.d.ts +28 -0
- package/dist/captcha/captcha-guard.use-case.d.ts +64 -0
- package/dist/captcha/captcha-solver-registry.store.d.ts +19 -0
- package/dist/captcha/captcha-solver.contract.d.ts +47 -0
- package/dist/captcha/captcha.error.d.ts +13 -0
- package/dist/captcha/index.d.ts +10 -0
- package/dist/captcha/resolve-captcha.use-case.d.ts +40 -0
- package/dist/crawl-events/crawl-event.contract.d.ts +169 -0
- package/dist/crawl-events/event-bus.store.d.ts +10 -0
- package/dist/crawl-events/index.d.ts +4 -0
- package/dist/crawl-events/trace-line.mapper.d.ts +14 -0
- package/dist/crawl-execution/bootstrap-session.use-case.d.ts +79 -0
- package/dist/crawl-execution/crawl-options.config.d.ts +67 -0
- package/dist/crawl-execution/crawl-report.model.d.ts +33 -0
- package/dist/crawl-execution/create-crawler.use-case.d.ts +20 -0
- package/dist/crawl-execution/index.d.ts +8 -0
- package/dist/crawl-execution/rotating-runner.use-case.d.ts +66 -0
- package/dist/crawl-execution/run-crawl.use-case.d.ts +17 -0
- package/dist/crawl-execution/run-input-recipe.use-case.d.ts +51 -0
- package/dist/deck-document/deck-document.model.d.ts +58 -0
- package/dist/deck-document/deck-table.algorithm.d.ts +35 -0
- package/dist/deck-document/index.d.ts +6 -0
- package/dist/deck-document/read-pptx.client.d.ts +16 -0
- package/dist/docx-document/index.d.ts +2 -0
- package/dist/docx-document/read-docx-html.client.d.ts +23 -0
- package/dist/extraction-scope/extraction-scope.model.d.ts +77 -0
- package/dist/extraction-scope/index.d.ts +5 -0
- package/dist/extraction-scope/live-element.model.d.ts +28 -0
- package/dist/hooks/hook-registry.store.d.ts +16 -0
- package/dist/hooks/hook.contract.d.ts +17 -0
- package/dist/hooks/hook.error.d.ts +7 -0
- package/dist/hooks/index.d.ts +4 -0
- package/dist/http-session/http-response.contract.d.ts +58 -0
- package/dist/http-session/http.client.d.ts +42 -0
- package/dist/http-session/index.d.ts +5 -0
- package/dist/http-session/text-decoding.algorithm.d.ts +35 -0
- package/dist/index.esm.js +1410 -95
- package/dist/markdown-document/index.d.ts +3 -0
- package/dist/markdown-document/read-markdown.client.d.ts +39 -0
- package/dist/output-mapping/coerce-field.mapper.d.ts +19 -0
- package/dist/output-mapping/generated-field.mapper.d.ts +16 -0
- package/dist/output-mapping/index.d.ts +9 -0
- package/dist/output-mapping/map-record.use-case.d.ts +27 -0
- package/dist/output-mapping/mapping.error.d.ts +17 -0
- package/dist/output-mapping/missing-value.policy.d.ts +14 -0
- package/dist/output-mapping/output-field.validator.d.ts +10 -0
- package/dist/output-mapping/output-record.model.d.ts +20 -0
- package/dist/pdf-document/index.d.ts +7 -0
- package/dist/pdf-document/pdf-document.model.d.ts +51 -0
- package/dist/pdf-document/pdf-table.algorithm.d.ts +43 -0
- package/dist/pdf-document/read-pdf.client.d.ts +18 -0
- package/dist/pdf-document/row-assembly.algorithm.d.ts +26 -0
- package/dist/recipe-loading/index.d.ts +9 -0
- package/dist/recipe-loading/load-recipe-set.use-case.d.ts +45 -0
- package/dist/recipe-loading/read-recipe-source.use-case.d.ts +14 -0
- package/dist/recipe-loading/recipe-binding.error.d.ts +14 -0
- package/dist/recipe-loading/recipe-binding.validator.d.ts +34 -0
- package/dist/recipe-loading/recipe-file.repository.d.ts +11 -0
- package/dist/recipe-loading/recipe-set.model.d.ts +12 -0
- package/dist/recipe-loading/recipe-source.contract.d.ts +28 -0
- package/dist/recipe-loading/recipe-text.mapper.d.ts +14 -0
- package/dist/recipe-schema/index.d.ts +15 -0
- package/dist/recipe-schema/input-recipe.contract.d.ts +156 -0
- package/dist/recipe-schema/json-schema.mapper.d.ts +15 -0
- package/dist/recipe-schema/output-recipe.contract.d.ts +47 -0
- package/dist/recipe-schema/recipe-kind.enum.d.ts +35 -0
- package/dist/recipe-schema/recipe-validation.error.d.ts +17 -0
- package/dist/recipe-schema/recipe.validator.d.ts +28 -0
- package/dist/recipe-schema/step.contract.d.ts +206 -0
- package/dist/recipe-schema/transform-rule.contract.d.ts +120 -0
- package/dist/record-diff/index.d.ts +4 -0
- package/dist/record-diff/record-diff.algorithm.d.ts +82 -0
- package/dist/record-diff/records-file.repository.d.ts +10 -0
- package/dist/record-sink/dedupe.policy.d.ts +29 -0
- package/dist/record-sink/index.d.ts +9 -0
- package/dist/record-sink/json-lines-sink.repository.d.ts +19 -0
- package/dist/record-sink/memory-sink.repository.d.ts +9 -0
- package/dist/record-sink/record-sink.contract.d.ts +17 -0
- package/dist/selection/html-selector.algorithm.d.ts +19 -0
- package/dist/selection/index.d.ts +8 -0
- package/dist/selection/json-path.algorithm.d.ts +9 -0
- package/dist/selection/json-text.algorithm.d.ts +58 -0
- package/dist/selection/regex.algorithm.d.ts +12 -0
- package/dist/selection/take-value.mapper.d.ts +27 -0
- package/dist/src/access/access-profile.contract.d.ts +4 -0
- package/dist/src/access/index.d.ts +1 -1
- package/dist/src/api-steps/extract-from-document.use-case.d.ts +5 -5
- package/dist/src/api-steps/send-request.use-case.d.ts +3 -2
- package/dist/src/browser-session/browser-profile.store.d.ts +52 -0
- package/dist/src/browser-session/browser.client.d.ts +8 -0
- package/dist/src/browser-session/index.d.ts +1 -0
- package/dist/src/crawl-events/crawl-event.contract.d.ts +8 -0
- package/dist/src/crawl-execution/bootstrap-session.use-case.d.ts +23 -2
- package/dist/src/crawl-execution/crawl-options.config.d.ts +27 -0
- package/dist/src/crawl-execution/rotating-runner.use-case.d.ts +9 -0
- package/dist/src/crawl-execution/run-crawl.use-case.d.ts +6 -2
- package/dist/src/crawl-execution/run-input-recipe.use-case.d.ts +11 -3
- package/dist/src/docx-document/index.d.ts +2 -0
- package/dist/src/docx-document/read-docx-html.client.d.ts +23 -0
- package/dist/src/extraction-scope/extraction-scope.model.d.ts +2 -1
- package/dist/src/http-session/http-response.contract.d.ts +2 -1
- package/dist/src/index.d.ts +9 -4
- package/dist/src/markdown-document/index.d.ts +1 -1
- package/dist/src/markdown-document/read-markdown.client.d.ts +10 -0
- package/dist/src/recipe-schema/index.d.ts +2 -2
- package/dist/src/recipe-schema/input-recipe.contract.d.ts +24 -1
- package/dist/src/recipe-schema/recipe-kind.enum.d.ts +1 -1
- package/dist/src/recipe-schema/step.contract.d.ts +4 -0
- package/dist/src/record-diff/index.d.ts +4 -0
- package/dist/src/record-diff/record-diff.algorithm.d.ts +82 -0
- package/dist/src/record-diff/records-file.repository.d.ts +10 -0
- package/dist/src/record-sink/dedupe.policy.d.ts +19 -7
- package/dist/src/record-sink/index.d.ts +1 -0
- package/dist/src/selection/html-selector.algorithm.d.ts +2 -1
- package/dist/src/step-flow/for-each.use-case.d.ts +4 -2
- package/dist/src/step-flow/host-throttle.policy.d.ts +49 -0
- package/dist/src/step-flow/index.d.ts +5 -0
- package/dist/src/step-flow/run-gate.policy.d.ts +12 -1
- package/dist/src/step-flow/step-runner.contract.d.ts +13 -0
- package/dist/src/step-flow/transport-retry.policy.d.ts +76 -0
- package/dist/src/web-steps/navigate.use-case.d.ts +3 -2
- package/dist/src/web-steps/run-web-step.use-case.d.ts +8 -1
- package/dist/src/xml-document/index.d.ts +6 -0
- package/dist/src/xml-document/xml-document.model.d.ts +13 -0
- package/dist/src/xml-document/xml-parser.client.d.ts +49 -0
- package/dist/src/xml-document/xpath.algorithm.d.ts +30 -0
- package/dist/step-flow/block-rule.policy.d.ts +21 -0
- package/dist/step-flow/blocked.error.d.ts +9 -0
- package/dist/step-flow/for-each.use-case.d.ts +20 -0
- package/dist/step-flow/host-throttle.policy.d.ts +49 -0
- package/dist/step-flow/index.d.ts +15 -0
- package/dist/step-flow/paginate.use-case.d.ts +17 -0
- package/dist/step-flow/retry.policy.d.ts +21 -0
- package/dist/step-flow/run-gate.policy.d.ts +50 -0
- package/dist/step-flow/run-steps.use-case.d.ts +39 -0
- package/dist/step-flow/step-failure.error.d.ts +14 -0
- package/dist/step-flow/step-runner.contract.d.ts +52 -0
- package/dist/step-flow/transport-retry.policy.d.ts +76 -0
- package/dist/template/expression.algorithm.d.ts +53 -0
- package/dist/template/index.d.ts +7 -0
- package/dist/template/template.algorithm.d.ts +53 -0
- package/dist/template/value-path.algorithm.d.ts +29 -0
- package/dist/template/value-text.algorithm.d.ts +18 -0
- package/dist/transformation/apply-transform-chain.use-case.d.ts +17 -0
- package/dist/transformation/collection.algorithm.d.ts +15 -0
- package/dist/transformation/currency.algorithm.d.ts +15 -0
- package/dist/transformation/date.algorithm.d.ts +13 -0
- package/dist/transformation/index.d.ts +10 -0
- package/dist/transformation/lookup.algorithm.d.ts +24 -0
- package/dist/transformation/number.algorithm.d.ts +30 -0
- package/dist/transformation/string.algorithm.d.ts +18 -0
- package/dist/transformation/transform-registry.store.d.ts +36 -0
- package/dist/transformation/transform.error.d.ts +8 -0
- package/dist/transformation/url.algorithm.d.ts +20 -0
- package/dist/web-steps/evaluate-script.use-case.d.ts +10 -0
- package/dist/web-steps/extract-from-page.use-case.d.ts +10 -0
- package/dist/web-steps/index.d.ts +2 -0
- package/dist/web-steps/interact.use-case.d.ts +36 -0
- package/dist/web-steps/navigate.use-case.d.ts +15 -0
- package/dist/web-steps/run-web-step.use-case.d.ts +37 -0
- package/dist/web-steps/snapshot-elements.use-case.d.ts +13 -0
- package/dist/workbook-document/csv-parser.algorithm.d.ts +26 -0
- package/dist/workbook-document/csv-workbook.mapper.d.ts +24 -0
- package/dist/workbook-document/grid-table.algorithm.d.ts +53 -0
- package/dist/workbook-document/html-tables.mapper.d.ts +14 -0
- package/dist/workbook-document/index.d.ts +9 -0
- package/dist/workbook-document/read-xlsx.client.d.ts +18 -0
- package/dist/workbook-document/workbook-document.model.d.ts +51 -0
- package/dist/xml-document/index.d.ts +6 -0
- package/dist/xml-document/xml-document.model.d.ts +13 -0
- package/dist/xml-document/xml-parser.client.d.ts +49 -0
- package/dist/xml-document/xpath.algorithm.d.ts +30 -0
- package/dist/yaml-document/index.d.ts +3 -0
- package/dist/yaml-document/read-yaml.client.d.ts +25 -0
- package/package.json +4 -2
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import type { CrawlMode, KeepKind } from './recipe-kind.enum.js';
|
|
3
|
+
import type { CaptchaCheck, ErrorPolicy, Step } from './step.contract.js';
|
|
4
|
+
import type { MappingRule } from './transform-rule.contract.js';
|
|
5
|
+
/** A URL the crawl starts from, with variables visible to its templates as `vars.*`. */
|
|
6
|
+
export interface StartPoint {
|
|
7
|
+
url: string;
|
|
8
|
+
vars?: Record<string, string | number | boolean>;
|
|
9
|
+
}
|
|
10
|
+
/** A cookie in Playwright's shape. */
|
|
11
|
+
export interface RecipeCookie {
|
|
12
|
+
name: string;
|
|
13
|
+
value: string;
|
|
14
|
+
domain: string;
|
|
15
|
+
path?: string;
|
|
16
|
+
expires?: number;
|
|
17
|
+
httpOnly?: boolean;
|
|
18
|
+
secure?: boolean;
|
|
19
|
+
sameSite?: 'Strict' | 'Lax' | 'None';
|
|
20
|
+
}
|
|
21
|
+
/** Runs before the crawl, always in a browser, to obtain cookies or storage (a login, a consent wall). */
|
|
22
|
+
export interface SessionBootstrap {
|
|
23
|
+
steps: Step[];
|
|
24
|
+
keep: KeepKind[];
|
|
25
|
+
/** Persist the resulting storage state for the next run. */
|
|
26
|
+
saveTo?: string;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* What the site needs from the network, never how to get it: the runner's
|
|
30
|
+
* access config maps a profile name to a provider and credentials.
|
|
31
|
+
*/
|
|
32
|
+
export interface SessionAccess {
|
|
33
|
+
/** An access profile of the runner's config; its default when omitted. */
|
|
34
|
+
profile?: string;
|
|
35
|
+
/** ISO 3166 country the traffic should come from, for profiles that target by country. */
|
|
36
|
+
country?: string;
|
|
37
|
+
/** `false` lets the provider rotate IPs per request; the profile decides when omitted. */
|
|
38
|
+
sticky?: boolean;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* What counts as the site refusing the crawl, checked on every navigation and
|
|
42
|
+
* request. Any condition that matches is a block. When omitted: status 403 or
|
|
43
|
+
* 429, or an AWS WAF challenge (`x-amzn-waf-action: challenge`).
|
|
44
|
+
*/
|
|
45
|
+
export interface BlockRule {
|
|
46
|
+
status?: number[];
|
|
47
|
+
/** Header name to a regular expression its value must match (case-insensitive). */
|
|
48
|
+
header?: Record<string, string>;
|
|
49
|
+
/** A regular expression the response body must match (case-insensitive). */
|
|
50
|
+
text?: string;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* What to do when blocked. `solve`: when the block page shows a captcha, the
|
|
54
|
+
* `session.captcha` solver solves it on the spot. `rotate`: take a new access
|
|
55
|
+
* lease (a new IP), reopen the session, and retry the step; with `solve`, only
|
|
56
|
+
* once solving failed.
|
|
57
|
+
*/
|
|
58
|
+
export interface BlockRotation {
|
|
59
|
+
rotate?: boolean;
|
|
60
|
+
solve?: boolean;
|
|
61
|
+
/** How many rotations a recipe run may use. Default 2. */
|
|
62
|
+
attempts?: number;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* How a web recipe gets past captchas: a solver the runner registered (a
|
|
66
|
+
* plugin's `captchaSolvers`), where to look for challenges, how to confirm
|
|
67
|
+
* one is solved, and what the run may spend. With it, the engine checks for a
|
|
68
|
+
* challenge after each navigation, click and key press, and solves it before
|
|
69
|
+
* the next step runs.
|
|
70
|
+
*/
|
|
71
|
+
export interface CaptchaSettings {
|
|
72
|
+
/** The name of a registered captcha solver. */
|
|
73
|
+
solver: string;
|
|
74
|
+
/** Where challenges are (a Playwright selector); the common widgets when omitted. */
|
|
75
|
+
detect?: {
|
|
76
|
+
selector: string;
|
|
77
|
+
};
|
|
78
|
+
/** How a solve is confirmed. Default: the challenge is gone. */
|
|
79
|
+
verify?: CaptchaCheck;
|
|
80
|
+
/** Solves tried per challenge before it counts as a block. Default 3. */
|
|
81
|
+
attempts?: number;
|
|
82
|
+
/** How long one solve may take. Default 120000. */
|
|
83
|
+
timeoutMs?: number;
|
|
84
|
+
/** Solves the whole run may spend (a paid solver bills each). Default 10; 0 detects without solving. */
|
|
85
|
+
maxSolves?: number;
|
|
86
|
+
}
|
|
87
|
+
export interface SessionSpec {
|
|
88
|
+
headers?: Record<string, string>;
|
|
89
|
+
cookies?: RecipeCookie[];
|
|
90
|
+
userAgent?: string;
|
|
91
|
+
viewport?: {
|
|
92
|
+
width: number;
|
|
93
|
+
height: number;
|
|
94
|
+
};
|
|
95
|
+
/** Reuse a storage state saved by a previous bootstrap. */
|
|
96
|
+
storageStatePath?: string;
|
|
97
|
+
bootstrap?: SessionBootstrap;
|
|
98
|
+
access?: SessionAccess;
|
|
99
|
+
blockedWhen?: BlockRule;
|
|
100
|
+
onBlock?: BlockRotation;
|
|
101
|
+
captcha?: CaptchaSettings;
|
|
102
|
+
/**
|
|
103
|
+
* A browser profile of the runner that persists between runs (cookies,
|
|
104
|
+
* storage, cache): web recipes and bootstraps run in it. A name; the runner
|
|
105
|
+
* decides where profiles live.
|
|
106
|
+
*/
|
|
107
|
+
browserProfile?: string;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* How a request that fails in passing (a dropped connection, a timeout, a
|
|
111
|
+
* 503, a 429) is sent again. On by default: three tries in all.
|
|
112
|
+
*/
|
|
113
|
+
export interface RetryRule {
|
|
114
|
+
/** Tries per request, the first included; `1` turns retrying off. Default 3. */
|
|
115
|
+
attempts?: number;
|
|
116
|
+
/** The first pause; it doubles on every retry. Default 1000. */
|
|
117
|
+
backoffMs?: number;
|
|
118
|
+
/** The longest pause, `Retry-After` included; a server asking for longer is not retried. Default 30000. */
|
|
119
|
+
maxDelayMs?: number;
|
|
120
|
+
/** The statuses retried. Default `[408, 425, 429, 500, 502, 503, 504]`. */
|
|
121
|
+
statuses?: number[];
|
|
122
|
+
}
|
|
123
|
+
export interface CrawlLimits {
|
|
124
|
+
maxRecords?: number;
|
|
125
|
+
/** Minimum interval between two request starts across the recipe, whatever runs in parallel. */
|
|
126
|
+
delayMs?: number;
|
|
127
|
+
timeoutMs?: number;
|
|
128
|
+
/** How many `forEach` iterations over a list may run at once: requests in api mode, tabs in web mode. Default 1. */
|
|
129
|
+
concurrency?: number;
|
|
130
|
+
/** How requests that fail in passing are sent again; the crawler's `retry`, else three tries, when omitted. */
|
|
131
|
+
retry?: RetryRule;
|
|
132
|
+
}
|
|
133
|
+
/** Where to start, how to navigate, what to extract, and how it maps to one output recipe. */
|
|
134
|
+
export interface InputRecipe {
|
|
135
|
+
$schema?: string;
|
|
136
|
+
kind: 'input';
|
|
137
|
+
id: string;
|
|
138
|
+
/** The `OutputRecipe.id` this recipe feeds. */
|
|
139
|
+
output: string;
|
|
140
|
+
mode: CrawlMode;
|
|
141
|
+
description?: string;
|
|
142
|
+
start: StartPoint[];
|
|
143
|
+
vars?: Record<string, string | number | boolean>;
|
|
144
|
+
session?: SessionSpec;
|
|
145
|
+
limits?: CrawlLimits;
|
|
146
|
+
/** Default policy for every step. */
|
|
147
|
+
onError?: ErrorPolicy;
|
|
148
|
+
steps: Step[];
|
|
149
|
+
/** Keyed by output field path, dotted for nested fields. */
|
|
150
|
+
mapping: Record<string, MappingRule>;
|
|
151
|
+
}
|
|
152
|
+
export declare const startPointSchema: z.ZodType<StartPoint>;
|
|
153
|
+
export declare const sessionSpecSchema: z.ZodType<SessionSpec>;
|
|
154
|
+
export declare const retryRuleSchema: z.ZodType<RetryRule>;
|
|
155
|
+
export declare const inputRecipeSchema: z.ZodType<InputRecipe>;
|
|
156
|
+
//# sourceMappingURL=input-recipe.contract.d.ts.map
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/** A JSON Schema document, as `z.toJSONSchema` produces it. */
|
|
2
|
+
export type JsonSchemaDocument = Record<string, unknown>;
|
|
3
|
+
/**
|
|
4
|
+
* The JSON Schema for output recipe files, for editors and other tools.
|
|
5
|
+
*
|
|
6
|
+
* @returns A draft 2020-12 document.
|
|
7
|
+
*/
|
|
8
|
+
export declare function outputRecipeJsonSchema(): JsonSchemaDocument;
|
|
9
|
+
/**
|
|
10
|
+
* The JSON Schema for input recipe files, for editors and other tools.
|
|
11
|
+
*
|
|
12
|
+
* @returns A draft 2020-12 document.
|
|
13
|
+
*/
|
|
14
|
+
export declare function inputRecipeJsonSchema(): JsonSchemaDocument;
|
|
15
|
+
//# sourceMappingURL=json-schema.mapper.d.ts.map
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import type { FieldType, GeneratedValue, MissingPolicy, RecipeMissingPolicy } from './recipe-kind.enum.js';
|
|
3
|
+
/** One field of the records a crawl produces: its type, quality rules and what to do when it is missing. */
|
|
4
|
+
export interface FieldSpec {
|
|
5
|
+
type: FieldType;
|
|
6
|
+
description?: string;
|
|
7
|
+
/** Default `false`. */
|
|
8
|
+
required?: boolean;
|
|
9
|
+
/** Default `false`. */
|
|
10
|
+
nullable?: boolean;
|
|
11
|
+
/** Used when the value is missing and the resolved policy is `default`. */
|
|
12
|
+
default?: unknown;
|
|
13
|
+
onMissing?: MissingPolicy;
|
|
14
|
+
/** Part of the record identity, used to drop duplicates. */
|
|
15
|
+
key?: boolean;
|
|
16
|
+
/** Supplied by the engine, never mapped. */
|
|
17
|
+
generated?: GeneratedValue;
|
|
18
|
+
/** Input format for `date` / `datetime`; output is always ISO 8601. */
|
|
19
|
+
format?: string;
|
|
20
|
+
/** ISO 4217 code for `currency`. */
|
|
21
|
+
currency?: string;
|
|
22
|
+
/** Allowed values for `enum`. */
|
|
23
|
+
values?: string[];
|
|
24
|
+
/** Element spec for `array`. */
|
|
25
|
+
items?: FieldSpec;
|
|
26
|
+
/** Member specs for `object`. A `json` field takes none: it keeps whatever it is given. */
|
|
27
|
+
fields?: Record<string, FieldSpec>;
|
|
28
|
+
min?: number;
|
|
29
|
+
max?: number;
|
|
30
|
+
pattern?: string;
|
|
31
|
+
minLength?: number;
|
|
32
|
+
maxLength?: number;
|
|
33
|
+
}
|
|
34
|
+
/** The schema of the records every input recipe bound to it must produce. */
|
|
35
|
+
export interface OutputRecipe {
|
|
36
|
+
$schema?: string;
|
|
37
|
+
kind: 'output';
|
|
38
|
+
id: string;
|
|
39
|
+
version: number;
|
|
40
|
+
description?: string;
|
|
41
|
+
fields: Record<string, FieldSpec>;
|
|
42
|
+
/** Recipe-wide default: `fail` for required fields, `null` otherwise. */
|
|
43
|
+
onMissing?: RecipeMissingPolicy;
|
|
44
|
+
}
|
|
45
|
+
export declare const fieldSpecSchema: z.ZodType<FieldSpec>;
|
|
46
|
+
export declare const outputRecipeSchema: z.ZodType<OutputRecipe>;
|
|
47
|
+
//# sourceMappingURL=output-recipe.contract.d.ts.map
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/** The closed vocabularies a recipe file can use. Each is a `readonly` tuple so zod and TypeScript share it. */
|
|
2
|
+
export declare const RECIPE_KINDS: readonly ['input', 'output'];
|
|
3
|
+
export declare const CRAWL_MODES: readonly ['web', 'api'];
|
|
4
|
+
export declare const SELECTOR_KINDS: readonly ['css', 'xpath', 'jsonpath', 'regex', 'table'];
|
|
5
|
+
/** `take` also accepts `attr:<name>`, which is validated by pattern rather than listed. */
|
|
6
|
+
export declare const TAKE_KINDS: readonly ['text', 'html', 'value', 'json'];
|
|
7
|
+
export declare const BODY_KINDS: readonly ['json', 'jsonl', 'html', 'text', 'pdf', 'csv', 'xlsx', 'pptx', 'yaml', 'markdown', 'xml', 'docx'];
|
|
8
|
+
export declare const YAML_SCALARS: readonly ['typed', 'text'];
|
|
9
|
+
/** How a PDF table aligns a row's values against a cell wrapped over several lines. */
|
|
10
|
+
export declare const TABLE_ALIGNS: readonly ['auto', 'top', 'center', 'bottom'];
|
|
11
|
+
export declare const FIELD_TYPES: readonly ['string', 'number', 'integer', 'boolean', 'date', 'datetime', 'currency', 'url', 'enum', 'array', 'object', 'json'];
|
|
12
|
+
export declare const MISSING_POLICIES: readonly ['fail', 'skip-record', 'null', 'default'];
|
|
13
|
+
export declare const RECIPE_MISSING_POLICIES: readonly ['fail', 'skip-record', 'null'];
|
|
14
|
+
export declare const ERROR_POLICIES: readonly ['fail', 'skip', 'retry'];
|
|
15
|
+
export declare const GENERATED_VALUES: readonly ['now', 'uuid', 'sourceUrl', 'recipeId'];
|
|
16
|
+
export declare const KEEP_KINDS: readonly ['cookies', 'localStorage'];
|
|
17
|
+
export declare const WAIT_UNTIL: readonly ['load', 'domcontentloaded', 'networkidle', 'commit'];
|
|
18
|
+
export declare const HTTP_METHODS: readonly ['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD'];
|
|
19
|
+
/** Steps that only make sense with a live browser page. */
|
|
20
|
+
export declare const WEB_ONLY_STEPS: readonly ['goto', 'click', 'fill', 'press', 'select', 'scroll', 'wait', 'evaluate', 'screenshot', 'captcha'];
|
|
21
|
+
/** Steps that only make sense against an HTTP request context. */
|
|
22
|
+
export declare const API_ONLY_STEPS: readonly ['request'];
|
|
23
|
+
export type RecipeKind = typeof RECIPE_KINDS[number];
|
|
24
|
+
export type CrawlMode = typeof CRAWL_MODES[number];
|
|
25
|
+
export type SelectorKind = typeof SELECTOR_KINDS[number];
|
|
26
|
+
export type BodyKind = typeof BODY_KINDS[number];
|
|
27
|
+
export type TableAlign = typeof TABLE_ALIGNS[number];
|
|
28
|
+
export type FieldType = typeof FIELD_TYPES[number];
|
|
29
|
+
export type MissingPolicy = typeof MISSING_POLICIES[number];
|
|
30
|
+
export type RecipeMissingPolicy = typeof RECIPE_MISSING_POLICIES[number];
|
|
31
|
+
export type GeneratedValue = typeof GENERATED_VALUES[number];
|
|
32
|
+
export type KeepKind = typeof KEEP_KINDS[number];
|
|
33
|
+
export type WaitUntil = typeof WAIT_UNTIL[number];
|
|
34
|
+
export type HttpMethod = typeof HTTP_METHODS[number];
|
|
35
|
+
//# sourceMappingURL=recipe-kind.enum.d.ts.map
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/** One problem found in a recipe file, located by its JSON path. */
|
|
2
|
+
export interface RecipeIssue {
|
|
3
|
+
/** Dotted JSON path, `''` for the root. */
|
|
4
|
+
path: string;
|
|
5
|
+
message: string;
|
|
6
|
+
}
|
|
7
|
+
/** A recipe that does not match its contract. `issues` lists every problem, not just the first. */
|
|
8
|
+
export declare class RecipeValidationError extends Error {
|
|
9
|
+
/** Which recipe: a file path or an id when known. */
|
|
10
|
+
readonly source: string;
|
|
11
|
+
readonly issues: readonly RecipeIssue[];
|
|
12
|
+
readonly name = "RecipeValidationError";
|
|
13
|
+
constructor(
|
|
14
|
+
/** Which recipe: a file path or an id when known. */
|
|
15
|
+
source: string, issues: readonly RecipeIssue[]);
|
|
16
|
+
}
|
|
17
|
+
//# sourceMappingURL=recipe-validation.error.d.ts.map
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { InputRecipe } from './input-recipe.contract.js';
|
|
2
|
+
import type { OutputRecipe } from './output-recipe.contract.js';
|
|
3
|
+
/**
|
|
4
|
+
* Parses an unknown value (a decoded JSON file) as an output recipe.
|
|
5
|
+
*
|
|
6
|
+
* @param value - The decoded JSON.
|
|
7
|
+
* @param source - A name for error messages, usually the file path.
|
|
8
|
+
* @returns The typed recipe.
|
|
9
|
+
* @throws RecipeValidationError when it does not match the contract.
|
|
10
|
+
*/
|
|
11
|
+
export declare function parseOutputRecipe(value: unknown, source?: string): OutputRecipe;
|
|
12
|
+
/**
|
|
13
|
+
* Parses an unknown value (a decoded JSON file) as an input recipe.
|
|
14
|
+
*
|
|
15
|
+
* @param value - The decoded JSON.
|
|
16
|
+
* @param source - A name for error messages, usually the file path.
|
|
17
|
+
* @returns The typed recipe.
|
|
18
|
+
* @throws RecipeValidationError when it does not match the contract.
|
|
19
|
+
*/
|
|
20
|
+
export declare function parseInputRecipe(value: unknown, source?: string): InputRecipe;
|
|
21
|
+
/**
|
|
22
|
+
* Reads the `kind` of a decoded recipe without validating the rest.
|
|
23
|
+
*
|
|
24
|
+
* @param value - The decoded JSON.
|
|
25
|
+
* @returns `'input'`, `'output'`, or `undefined` when it is neither.
|
|
26
|
+
*/
|
|
27
|
+
export declare function recipeKindOf(value: unknown): 'input' | 'output' | undefined;
|
|
28
|
+
//# sourceMappingURL=recipe.validator.d.ts.map
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { TAKE_KINDS } from './recipe-kind.enum.js';
|
|
3
|
+
import type { BodyKind, HttpMethod, SelectorKind, TableAlign, WaitUntil } from './recipe-kind.enum.js';
|
|
4
|
+
/** What to do when a step fails. Resolved step -> recipe -> `fail`. */
|
|
5
|
+
export type ErrorPolicy = {
|
|
6
|
+
policy: 'fail';
|
|
7
|
+
} | {
|
|
8
|
+
policy: 'skip';
|
|
9
|
+
} | {
|
|
10
|
+
policy: 'retry';
|
|
11
|
+
attempts: number;
|
|
12
|
+
backoffMs?: number;
|
|
13
|
+
};
|
|
14
|
+
/** What a value is taken from an element or JSON node: text, inner HTML, an attribute, an input value, or the node itself. */
|
|
15
|
+
export type TakeKind = typeof TAKE_KINDS[number] | `attr:${string}`;
|
|
16
|
+
/** How `paginate` finds the next page once the page body has run. */
|
|
17
|
+
export type PaginateNext = {
|
|
18
|
+
selector: string;
|
|
19
|
+
} | {
|
|
20
|
+
url: string;
|
|
21
|
+
} | {
|
|
22
|
+
jsonpath: string;
|
|
23
|
+
as?: string;
|
|
24
|
+
};
|
|
25
|
+
export interface StepBaseFields {
|
|
26
|
+
/** Names the value this step produces. */
|
|
27
|
+
id?: string;
|
|
28
|
+
onError?: ErrorPolicy;
|
|
29
|
+
/** A template; the step runs only when it renders truthy. */
|
|
30
|
+
when?: string;
|
|
31
|
+
}
|
|
32
|
+
export interface GotoStep extends StepBaseFields {
|
|
33
|
+
type: 'goto';
|
|
34
|
+
url: string;
|
|
35
|
+
waitUntil?: WaitUntil;
|
|
36
|
+
}
|
|
37
|
+
/** Where an interaction lands: a selector, or a `target` template that renders to a live element (a `forEach` over `selector`) or to a selector string. */
|
|
38
|
+
export interface TargetFields {
|
|
39
|
+
selector?: string;
|
|
40
|
+
target?: string;
|
|
41
|
+
}
|
|
42
|
+
export interface ClickStep extends StepBaseFields, TargetFields {
|
|
43
|
+
type: 'click';
|
|
44
|
+
optional?: boolean;
|
|
45
|
+
}
|
|
46
|
+
export interface FillStep extends StepBaseFields, TargetFields {
|
|
47
|
+
type: 'fill';
|
|
48
|
+
value: string;
|
|
49
|
+
}
|
|
50
|
+
export interface PressStep extends StepBaseFields, TargetFields {
|
|
51
|
+
type: 'press';
|
|
52
|
+
key: string;
|
|
53
|
+
}
|
|
54
|
+
/** Picks an option of a `<select>` by value, label or index. */
|
|
55
|
+
export interface SelectStep extends StepBaseFields, TargetFields {
|
|
56
|
+
type: 'select';
|
|
57
|
+
value?: string;
|
|
58
|
+
label?: string;
|
|
59
|
+
index?: number;
|
|
60
|
+
}
|
|
61
|
+
export interface ScrollStep extends StepBaseFields {
|
|
62
|
+
type: 'scroll';
|
|
63
|
+
to: string;
|
|
64
|
+
times?: number;
|
|
65
|
+
untilStable?: boolean;
|
|
66
|
+
}
|
|
67
|
+
export interface WaitStep extends StepBaseFields {
|
|
68
|
+
type: 'wait';
|
|
69
|
+
selector?: string;
|
|
70
|
+
ms?: number;
|
|
71
|
+
state?: 'networkidle';
|
|
72
|
+
}
|
|
73
|
+
export interface EvaluateStep extends StepBaseFields {
|
|
74
|
+
type: 'evaluate';
|
|
75
|
+
script: string;
|
|
76
|
+
}
|
|
77
|
+
export interface ScreenshotStep extends StepBaseFields {
|
|
78
|
+
type: 'screenshot';
|
|
79
|
+
path: string;
|
|
80
|
+
}
|
|
81
|
+
export interface RequestStep extends StepBaseFields {
|
|
82
|
+
type: 'request';
|
|
83
|
+
method?: HttpMethod;
|
|
84
|
+
url: string;
|
|
85
|
+
query?: Record<string, string>;
|
|
86
|
+
headers?: Record<string, string>;
|
|
87
|
+
body?: unknown;
|
|
88
|
+
as?: BodyKind;
|
|
89
|
+
/** The body's text encoding (a WHATWG label, `windows-1252`); default: the BOM, the declared charset, UTF-8, else Windows-1252. */
|
|
90
|
+
encoding?: string;
|
|
91
|
+
/** A CSV body's delimiter (one character); default: detected among `,` `;` tab `|`. */
|
|
92
|
+
delimiter?: string;
|
|
93
|
+
/** A YAML body's scalars: `typed` (default, YAML 1.2) or `text`, every scalar as written (`0123` stays `"0123"`). */
|
|
94
|
+
scalars?: 'typed' | 'text';
|
|
95
|
+
}
|
|
96
|
+
export interface ExtractStep extends StepBaseFields {
|
|
97
|
+
type: 'extract';
|
|
98
|
+
selector: string;
|
|
99
|
+
kind: SelectorKind;
|
|
100
|
+
take?: TakeKind;
|
|
101
|
+
many?: boolean;
|
|
102
|
+
/** Id of a document or fragment to read instead of the current document. */
|
|
103
|
+
from?: string;
|
|
104
|
+
/** `table` only: output key -> a pattern (case-insensitive) for that column's header cell. */
|
|
105
|
+
columns?: Record<string, string>;
|
|
106
|
+
/** `table` only: a pattern (case-insensitive) for the row that ends a table. */
|
|
107
|
+
until?: string;
|
|
108
|
+
/** `table` only: how a row's values sit against a cell wrapped over several lines (PDF); default `auto`. */
|
|
109
|
+
align?: TableAlign;
|
|
110
|
+
/** `table` only: a pattern (case-insensitive) for the names of the sheets to read (workbook); default every sheet. */
|
|
111
|
+
sheet?: string;
|
|
112
|
+
/** `table` only: how many rows the header spans (workbook); a column's key joins its header texts. Default 1. */
|
|
113
|
+
headerRows?: number;
|
|
114
|
+
/** `table` only: output keys whose empty cells take the value of the row above. */
|
|
115
|
+
fillDown?: string[];
|
|
116
|
+
/** `table` only: read hidden sheets and rows (workbook) or hidden slides (deck) too. */
|
|
117
|
+
includeHidden?: boolean;
|
|
118
|
+
/** `table` only: a pattern (case-insensitive) for the titles of the slides to read (deck); default every slide. */
|
|
119
|
+
slide?: string;
|
|
120
|
+
/** `table` only: read text boxes laid out as a table instead of native tables (deck). */
|
|
121
|
+
shapes?: boolean;
|
|
122
|
+
/** `xpath` on XML only: prefix to namespace URI (`{ "atom": "http://www.w3.org/2005/Atom" }`). Prefixes the root declares are known already. */
|
|
123
|
+
namespaces?: Record<string, string>;
|
|
124
|
+
/** `xpath` on XML only: drop the document's namespaces, so `//entry/title` matches `<entry xmlns="…">`. */
|
|
125
|
+
ignoreNamespaces?: boolean;
|
|
126
|
+
}
|
|
127
|
+
export interface SetStep extends StepBaseFields {
|
|
128
|
+
type: 'set';
|
|
129
|
+
value: unknown;
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Appends to a list bound in an enclosing scope (a `set` to `[]` before the
|
|
133
|
+
* loop), so values gathered page by page or item by item outlive the child
|
|
134
|
+
* scope that found them. A list value is appended item by item.
|
|
135
|
+
*/
|
|
136
|
+
export interface CollectStep extends StepBaseFields {
|
|
137
|
+
type: 'collect';
|
|
138
|
+
into: string;
|
|
139
|
+
value: unknown;
|
|
140
|
+
}
|
|
141
|
+
/** Runs a body per item of a list (`over`) or per live element matching `selector` (web mode; the elements are re-resolved on every use). */
|
|
142
|
+
export interface ForEachStep extends StepBaseFields {
|
|
143
|
+
type: 'forEach';
|
|
144
|
+
over?: string;
|
|
145
|
+
selector?: string;
|
|
146
|
+
as: string;
|
|
147
|
+
steps: Step[];
|
|
148
|
+
emit?: true | {
|
|
149
|
+
output: string;
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
/** Runs `steps` when `test` renders truthy, else `else`; both in the current scope. */
|
|
153
|
+
export interface IfStep extends StepBaseFields {
|
|
154
|
+
type: 'if';
|
|
155
|
+
test: string;
|
|
156
|
+
steps: Step[];
|
|
157
|
+
else?: Step[];
|
|
158
|
+
}
|
|
159
|
+
export interface PaginateStep extends StepBaseFields {
|
|
160
|
+
type: 'paginate';
|
|
161
|
+
next: PaginateNext;
|
|
162
|
+
until?: string;
|
|
163
|
+
maxPages?: number;
|
|
164
|
+
steps: Step[];
|
|
165
|
+
}
|
|
166
|
+
export interface EmitStep extends StepBaseFields {
|
|
167
|
+
type: 'emit';
|
|
168
|
+
output?: string;
|
|
169
|
+
}
|
|
170
|
+
export interface HookStep extends StepBaseFields {
|
|
171
|
+
type: 'hook';
|
|
172
|
+
name: string;
|
|
173
|
+
args?: Record<string, unknown>;
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* How the engine confirms a captcha was solved (it never takes the solver's
|
|
177
|
+
* word): the challenge is gone, and/or an element appears.
|
|
178
|
+
*/
|
|
179
|
+
export interface CaptchaCheck {
|
|
180
|
+
/** The detected challenge must be off the page. Default `true`. */
|
|
181
|
+
gone?: boolean;
|
|
182
|
+
/** An element that must appear once solved. */
|
|
183
|
+
selector?: string;
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Solves the captcha on the live page, if there is one (none is not an error):
|
|
187
|
+
* a challenge known to sit at one point of the crawl, a login form. The solver
|
|
188
|
+
* and the defaults come from `session.captcha`, unless the step names its own.
|
|
189
|
+
*/
|
|
190
|
+
export interface CaptchaStep extends StepBaseFields {
|
|
191
|
+
type: 'captcha';
|
|
192
|
+
/** A captcha solver the runner registered; `session.captcha.solver` when omitted. */
|
|
193
|
+
solver?: string;
|
|
194
|
+
/** Where the challenge is (a Playwright selector); the common widgets when omitted. */
|
|
195
|
+
selector?: string;
|
|
196
|
+
verify?: CaptchaCheck;
|
|
197
|
+
attempts?: number;
|
|
198
|
+
timeoutMs?: number;
|
|
199
|
+
}
|
|
200
|
+
export type Step = GotoStep | ClickStep | FillStep | PressStep | SelectStep | ScrollStep | WaitStep | EvaluateStep | ScreenshotStep | RequestStep | ExtractStep | SetStep | CollectStep | ForEachStep | IfStep | PaginateStep | EmitStep | HookStep | CaptchaStep;
|
|
201
|
+
export type StepType = Step['type'];
|
|
202
|
+
export declare const errorPolicySchema: z.ZodType<ErrorPolicy>;
|
|
203
|
+
export declare const paginateNextSchema: z.ZodType<PaginateNext>;
|
|
204
|
+
export declare const captchaCheckSchema: z.ZodType<CaptchaCheck>;
|
|
205
|
+
export declare const stepSchema: z.ZodType<Step>;
|
|
206
|
+
//# sourceMappingURL=step.contract.d.ts.map
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import type { MissingPolicy } from './recipe-kind.enum.js';
|
|
3
|
+
/** One operation in a mapping chain. The set is closed: extension happens through `hook`. */
|
|
4
|
+
export type TransformRule = {
|
|
5
|
+
op: 'trim';
|
|
6
|
+
} | {
|
|
7
|
+
op: 'lowercase';
|
|
8
|
+
} | {
|
|
9
|
+
op: 'uppercase';
|
|
10
|
+
} | {
|
|
11
|
+
op: 'replace';
|
|
12
|
+
pattern: string;
|
|
13
|
+
replacement: string;
|
|
14
|
+
flags?: string;
|
|
15
|
+
} | {
|
|
16
|
+
op: 'regex';
|
|
17
|
+
pattern: string;
|
|
18
|
+
group?: number;
|
|
19
|
+
flags?: string;
|
|
20
|
+
} | {
|
|
21
|
+
op: 'split';
|
|
22
|
+
separator: string;
|
|
23
|
+
} | {
|
|
24
|
+
op: 'join';
|
|
25
|
+
separator: string;
|
|
26
|
+
} | {
|
|
27
|
+
op: 'first';
|
|
28
|
+
} | {
|
|
29
|
+
op: 'last';
|
|
30
|
+
} | {
|
|
31
|
+
op: 'nth';
|
|
32
|
+
index: number;
|
|
33
|
+
} | {
|
|
34
|
+
op: 'slice';
|
|
35
|
+
start: number;
|
|
36
|
+
end?: number;
|
|
37
|
+
} | {
|
|
38
|
+
op: 'concat';
|
|
39
|
+
separator?: string;
|
|
40
|
+
} | {
|
|
41
|
+
op: 'coalesce';
|
|
42
|
+
} | {
|
|
43
|
+
op: 'default';
|
|
44
|
+
value: unknown;
|
|
45
|
+
} | {
|
|
46
|
+
op: 'number';
|
|
47
|
+
locale?: string;
|
|
48
|
+
} | {
|
|
49
|
+
op: 'integer';
|
|
50
|
+
} | {
|
|
51
|
+
op: 'boolean';
|
|
52
|
+
truthy?: string[];
|
|
53
|
+
} | {
|
|
54
|
+
op: 'currency';
|
|
55
|
+
locale?: string;
|
|
56
|
+
currency?: string;
|
|
57
|
+
} | {
|
|
58
|
+
op: 'date';
|
|
59
|
+
format?: string;
|
|
60
|
+
timezone?: string;
|
|
61
|
+
} | {
|
|
62
|
+
op: 'absoluteUrl';
|
|
63
|
+
base?: string;
|
|
64
|
+
} |
|
|
65
|
+
/** Percent-encodes text for one URL component (`encodeURIComponent`). */
|
|
66
|
+
{
|
|
67
|
+
op: 'urlEncode';
|
|
68
|
+
} | {
|
|
69
|
+
op: 'flatten';
|
|
70
|
+
} | {
|
|
71
|
+
op: 'unique';
|
|
72
|
+
} | {
|
|
73
|
+
op: 'sum';
|
|
74
|
+
} | {
|
|
75
|
+
op: 'count';
|
|
76
|
+
} | {
|
|
77
|
+
op: 'template';
|
|
78
|
+
value: string;
|
|
79
|
+
} | {
|
|
80
|
+
op: 'jsonpath';
|
|
81
|
+
path: string;
|
|
82
|
+
} |
|
|
83
|
+
/** Finds the item of a table (an id or path in scope) whose `key` path equals the input; yields `pick` from it, or the item. */
|
|
84
|
+
{
|
|
85
|
+
op: 'lookup';
|
|
86
|
+
in: string;
|
|
87
|
+
key: string;
|
|
88
|
+
pick?: string;
|
|
89
|
+
} |
|
|
90
|
+
/** Groups a list by a path into `[{ key, items }]`, first-seen order. */
|
|
91
|
+
{
|
|
92
|
+
op: 'group';
|
|
93
|
+
by: string;
|
|
94
|
+
} | {
|
|
95
|
+
op: 'hook';
|
|
96
|
+
name: string;
|
|
97
|
+
args?: Record<string, unknown>;
|
|
98
|
+
};
|
|
99
|
+
export type TransformOp = TransformRule['op'];
|
|
100
|
+
/** Binds one output field to extracted values. */
|
|
101
|
+
export type MappingRule = {
|
|
102
|
+
from: string | string[];
|
|
103
|
+
transform?: TransformRule[];
|
|
104
|
+
onMissing?: MissingPolicy;
|
|
105
|
+
} |
|
|
106
|
+
/** Builds an array of objects from a list: `fields` paths are relative to each list item. */
|
|
107
|
+
{
|
|
108
|
+
each: string;
|
|
109
|
+
fields: Record<string, MappingRule>;
|
|
110
|
+
onMissing?: MissingPolicy;
|
|
111
|
+
};
|
|
112
|
+
export type FromRule = Extract<MappingRule, {
|
|
113
|
+
from: unknown;
|
|
114
|
+
}>;
|
|
115
|
+
export type EachRule = Extract<MappingRule, {
|
|
116
|
+
each: unknown;
|
|
117
|
+
}>;
|
|
118
|
+
export declare const transformRuleSchema: z.ZodType<TransformRule>;
|
|
119
|
+
export declare const mappingRuleSchema: z.ZodType<MappingRule>;
|
|
120
|
+
//# sourceMappingURL=transform-rule.contract.d.ts.map
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export { diffRecords, diffOptionsFor, readableKey } from './record-diff.algorithm.js';
|
|
2
|
+
export type { RecordDiff, RecordChange, FieldChange, DiffOptions, StoredRecord } from './record-diff.algorithm.js';
|
|
3
|
+
export { readRecordsFile } from './records-file.repository.js';
|
|
4
|
+
//# sourceMappingURL=index.d.ts.map
|