@opencraw/core 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +67 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.esm.js +5302 -0
- package/dist/src/access/access-broker.use-case.d.ts +43 -0
- package/dist/src/access/access-config.error.d.ts +5 -0
- package/dist/src/access/access-config.repository.d.ts +11 -0
- package/dist/src/access/access-json-schema.mapper.d.ts +8 -0
- package/dist/src/access/access-plugin.contract.d.ts +48 -0
- package/dist/src/access/access-preset.store.d.ts +19 -0
- package/dist/src/access/access-profile.contract.d.ts +91 -0
- package/dist/src/access/index.d.ts +11 -0
- package/dist/src/access/render-access.mapper.d.ts +41 -0
- package/dist/src/access/session-id.algorithm.d.ts +8 -0
- package/dist/src/api-steps/extract-from-document.use-case.d.ts +27 -0
- package/dist/src/api-steps/index.d.ts +4 -0
- package/dist/src/api-steps/run-api-step.use-case.d.ts +20 -0
- package/dist/src/api-steps/send-request.use-case.d.ts +22 -0
- package/dist/src/browser-session/browser-session.config.d.ts +21 -0
- package/dist/src/browser-session/browser.client.d.ts +67 -0
- package/dist/src/browser-session/index.d.ts +5 -0
- package/dist/src/crawl-events/crawl-event.contract.d.ts +120 -0
- package/dist/src/crawl-events/event-bus.store.d.ts +10 -0
- package/dist/src/crawl-events/index.d.ts +4 -0
- package/dist/src/crawl-events/trace-line.mapper.d.ts +14 -0
- package/dist/src/crawl-execution/bootstrap-session.use-case.d.ts +55 -0
- package/dist/src/crawl-execution/crawl-options.config.d.ts +37 -0
- package/dist/src/crawl-execution/crawl-report.model.d.ts +25 -0
- package/dist/src/crawl-execution/create-crawler.use-case.d.ts +19 -0
- package/dist/src/crawl-execution/index.d.ts +8 -0
- package/dist/src/crawl-execution/rotating-runner.use-case.d.ts +57 -0
- package/dist/src/crawl-execution/run-crawl.use-case.d.ts +13 -0
- package/dist/src/crawl-execution/run-input-recipe.use-case.d.ts +40 -0
- package/dist/src/extraction-scope/extraction-scope.model.d.ts +73 -0
- package/dist/src/extraction-scope/index.d.ts +5 -0
- package/dist/src/extraction-scope/live-element.model.d.ts +28 -0
- package/dist/src/hooks/hook-registry.store.d.ts +16 -0
- package/dist/src/hooks/hook.contract.d.ts +17 -0
- package/dist/src/hooks/hook.error.d.ts +7 -0
- package/dist/src/hooks/index.d.ts +4 -0
- package/dist/src/http-session/http-response.contract.d.ts +44 -0
- package/dist/src/http-session/http.client.d.ts +42 -0
- package/dist/src/http-session/index.d.ts +5 -0
- package/dist/src/index.d.ts +24 -0
- package/dist/src/output-mapping/coerce-field.mapper.d.ts +19 -0
- package/dist/src/output-mapping/generated-field.mapper.d.ts +16 -0
- package/dist/src/output-mapping/index.d.ts +9 -0
- package/dist/src/output-mapping/map-record.use-case.d.ts +27 -0
- package/dist/src/output-mapping/mapping.error.d.ts +17 -0
- package/dist/src/output-mapping/missing-value.policy.d.ts +14 -0
- package/dist/src/output-mapping/output-field.validator.d.ts +10 -0
- package/dist/src/output-mapping/output-record.model.d.ts +20 -0
- package/dist/src/recipe-loading/index.d.ts +9 -0
- package/dist/src/recipe-loading/load-recipe-set.use-case.d.ts +45 -0
- package/dist/src/recipe-loading/read-recipe-source.use-case.d.ts +14 -0
- package/dist/src/recipe-loading/recipe-binding.error.d.ts +14 -0
- package/dist/src/recipe-loading/recipe-binding.validator.d.ts +32 -0
- package/dist/src/recipe-loading/recipe-file.repository.d.ts +11 -0
- package/dist/src/recipe-loading/recipe-set.model.d.ts +12 -0
- package/dist/src/recipe-loading/recipe-source.contract.d.ts +28 -0
- package/dist/src/recipe-loading/recipe-text.mapper.d.ts +14 -0
- package/dist/src/recipe-schema/index.d.ts +15 -0
- package/dist/src/recipe-schema/input-recipe.contract.d.ts +103 -0
- package/dist/src/recipe-schema/json-schema.mapper.d.ts +15 -0
- package/dist/src/recipe-schema/output-recipe.contract.d.ts +47 -0
- package/dist/src/recipe-schema/recipe-kind.enum.d.ts +31 -0
- package/dist/src/recipe-schema/recipe-validation.error.d.ts +17 -0
- package/dist/src/recipe-schema/recipe.validator.d.ts +28 -0
- package/dist/src/recipe-schema/step.contract.d.ts +152 -0
- package/dist/src/recipe-schema/transform-rule.contract.d.ts +120 -0
- package/dist/src/record-sink/dedupe.policy.d.ts +17 -0
- package/dist/src/record-sink/index.d.ts +8 -0
- package/dist/src/record-sink/json-lines-sink.repository.d.ts +19 -0
- package/dist/src/record-sink/memory-sink.repository.d.ts +9 -0
- package/dist/src/record-sink/record-sink.contract.d.ts +17 -0
- package/dist/src/selection/html-selector.algorithm.d.ts +18 -0
- package/dist/src/selection/index.d.ts +8 -0
- package/dist/src/selection/json-path.algorithm.d.ts +9 -0
- package/dist/src/selection/json-text.algorithm.d.ts +31 -0
- package/dist/src/selection/regex.algorithm.d.ts +12 -0
- package/dist/src/selection/take-value.mapper.d.ts +27 -0
- package/dist/src/step-flow/block-rule.policy.d.ts +21 -0
- package/dist/src/step-flow/blocked.error.d.ts +9 -0
- package/dist/src/step-flow/for-each.use-case.d.ts +18 -0
- package/dist/src/step-flow/index.d.ts +10 -0
- package/dist/src/step-flow/paginate.use-case.d.ts +17 -0
- package/dist/src/step-flow/retry.policy.d.ts +21 -0
- package/dist/src/step-flow/run-gate.policy.d.ts +39 -0
- package/dist/src/step-flow/run-steps.use-case.d.ts +39 -0
- package/dist/src/step-flow/step-failure.error.d.ts +14 -0
- package/dist/src/step-flow/step-runner.contract.d.ts +39 -0
- package/dist/src/template/expression.algorithm.d.ts +53 -0
- package/dist/src/template/index.d.ts +7 -0
- package/dist/src/template/template.algorithm.d.ts +53 -0
- package/dist/src/template/value-path.algorithm.d.ts +29 -0
- package/dist/src/template/value-text.algorithm.d.ts +18 -0
- package/dist/src/transformation/apply-transform-chain.use-case.d.ts +17 -0
- package/dist/src/transformation/collection.algorithm.d.ts +15 -0
- package/dist/src/transformation/currency.algorithm.d.ts +15 -0
- package/dist/src/transformation/date.algorithm.d.ts +13 -0
- package/dist/src/transformation/index.d.ts +10 -0
- package/dist/src/transformation/lookup.algorithm.d.ts +24 -0
- package/dist/src/transformation/number.algorithm.d.ts +30 -0
- package/dist/src/transformation/string.algorithm.d.ts +18 -0
- package/dist/src/transformation/transform-registry.store.d.ts +36 -0
- package/dist/src/transformation/transform.error.d.ts +8 -0
- package/dist/src/transformation/url.algorithm.d.ts +20 -0
- package/dist/src/web-steps/evaluate-script.use-case.d.ts +10 -0
- package/dist/src/web-steps/extract-from-page.use-case.d.ts +10 -0
- package/dist/src/web-steps/index.d.ts +2 -0
- package/dist/src/web-steps/interact.use-case.d.ts +36 -0
- package/dist/src/web-steps/navigate.use-case.d.ts +14 -0
- package/dist/src/web-steps/run-web-step.use-case.d.ts +22 -0
- package/dist/src/web-steps/snapshot-elements.use-case.d.ts +13 -0
- package/package.json +75 -0
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import type { MissingPolicy } from './recipe-kind.enum.js';
|
|
3
|
+
/** One operation in a mapping chain. The set is closed: extension happens through `hook`. */
|
|
4
|
+
export type TransformRule = {
|
|
5
|
+
op: 'trim';
|
|
6
|
+
} | {
|
|
7
|
+
op: 'lowercase';
|
|
8
|
+
} | {
|
|
9
|
+
op: 'uppercase';
|
|
10
|
+
} | {
|
|
11
|
+
op: 'replace';
|
|
12
|
+
pattern: string;
|
|
13
|
+
replacement: string;
|
|
14
|
+
flags?: string;
|
|
15
|
+
} | {
|
|
16
|
+
op: 'regex';
|
|
17
|
+
pattern: string;
|
|
18
|
+
group?: number;
|
|
19
|
+
flags?: string;
|
|
20
|
+
} | {
|
|
21
|
+
op: 'split';
|
|
22
|
+
separator: string;
|
|
23
|
+
} | {
|
|
24
|
+
op: 'join';
|
|
25
|
+
separator: string;
|
|
26
|
+
} | {
|
|
27
|
+
op: 'first';
|
|
28
|
+
} | {
|
|
29
|
+
op: 'last';
|
|
30
|
+
} | {
|
|
31
|
+
op: 'nth';
|
|
32
|
+
index: number;
|
|
33
|
+
} | {
|
|
34
|
+
op: 'slice';
|
|
35
|
+
start: number;
|
|
36
|
+
end?: number;
|
|
37
|
+
} | {
|
|
38
|
+
op: 'concat';
|
|
39
|
+
separator?: string;
|
|
40
|
+
} | {
|
|
41
|
+
op: 'coalesce';
|
|
42
|
+
} | {
|
|
43
|
+
op: 'default';
|
|
44
|
+
value: unknown;
|
|
45
|
+
} | {
|
|
46
|
+
op: 'number';
|
|
47
|
+
locale?: string;
|
|
48
|
+
} | {
|
|
49
|
+
op: 'integer';
|
|
50
|
+
} | {
|
|
51
|
+
op: 'boolean';
|
|
52
|
+
truthy?: string[];
|
|
53
|
+
} | {
|
|
54
|
+
op: 'currency';
|
|
55
|
+
locale?: string;
|
|
56
|
+
currency?: string;
|
|
57
|
+
} | {
|
|
58
|
+
op: 'date';
|
|
59
|
+
format?: string;
|
|
60
|
+
timezone?: string;
|
|
61
|
+
} | {
|
|
62
|
+
op: 'absoluteUrl';
|
|
63
|
+
base?: string;
|
|
64
|
+
} |
|
|
65
|
+
/** Percent-encodes text for one URL component (`encodeURIComponent`). */
|
|
66
|
+
{
|
|
67
|
+
op: 'urlEncode';
|
|
68
|
+
} | {
|
|
69
|
+
op: 'flatten';
|
|
70
|
+
} | {
|
|
71
|
+
op: 'unique';
|
|
72
|
+
} | {
|
|
73
|
+
op: 'sum';
|
|
74
|
+
} | {
|
|
75
|
+
op: 'count';
|
|
76
|
+
} | {
|
|
77
|
+
op: 'template';
|
|
78
|
+
value: string;
|
|
79
|
+
} | {
|
|
80
|
+
op: 'jsonpath';
|
|
81
|
+
path: string;
|
|
82
|
+
} |
|
|
83
|
+
/** Finds the item of a table (an id or path in scope) whose `key` path equals the input; yields `pick` from it, or the item. */
|
|
84
|
+
{
|
|
85
|
+
op: 'lookup';
|
|
86
|
+
in: string;
|
|
87
|
+
key: string;
|
|
88
|
+
pick?: string;
|
|
89
|
+
} |
|
|
90
|
+
/** Groups a list by a path into `[{ key, items }]`, first-seen order. */
|
|
91
|
+
{
|
|
92
|
+
op: 'group';
|
|
93
|
+
by: string;
|
|
94
|
+
} | {
|
|
95
|
+
op: 'hook';
|
|
96
|
+
name: string;
|
|
97
|
+
args?: Record<string, unknown>;
|
|
98
|
+
};
|
|
99
|
+
export type TransformOp = TransformRule['op'];
|
|
100
|
+
/** Binds one output field to extracted values. */
|
|
101
|
+
export type MappingRule = {
|
|
102
|
+
from: string | string[];
|
|
103
|
+
transform?: TransformRule[];
|
|
104
|
+
onMissing?: MissingPolicy;
|
|
105
|
+
} |
|
|
106
|
+
/** Builds an array of objects from a list: `fields` paths are relative to each list item. */
|
|
107
|
+
{
|
|
108
|
+
each: string;
|
|
109
|
+
fields: Record<string, MappingRule>;
|
|
110
|
+
onMissing?: MissingPolicy;
|
|
111
|
+
};
|
|
112
|
+
export type FromRule = Extract<MappingRule, {
|
|
113
|
+
from: unknown;
|
|
114
|
+
}>;
|
|
115
|
+
export type EachRule = Extract<MappingRule, {
|
|
116
|
+
each: unknown;
|
|
117
|
+
}>;
|
|
118
|
+
export declare const transformRuleSchema: z.ZodType<TransformRule>;
|
|
119
|
+
export declare const mappingRuleSchema: z.ZodType<MappingRule>;
|
|
120
|
+
//# sourceMappingURL=transform-rule.contract.d.ts.map
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { OutputRecord } from '../output-mapping/index.js';
|
|
2
|
+
/** How far de-duplication reaches: the whole run, one input recipe, or not at all. */
|
|
3
|
+
export type DedupeScope = 'run' | 'recipe' | 'off';
|
|
4
|
+
/** Drops records whose key was already seen. First record wins; keyless records always pass. */
|
|
5
|
+
export declare class DedupePolicy {
|
|
6
|
+
readonly scope: DedupeScope;
|
|
7
|
+
private seen;
|
|
8
|
+
constructor(scope?: DedupeScope);
|
|
9
|
+
/** Called when an input recipe starts; forgets keys under `recipe` scope. */
|
|
10
|
+
startRecipe(): void;
|
|
11
|
+
/**
|
|
12
|
+
* @param record - A validated record.
|
|
13
|
+
* @returns `true` when the record repeats an earlier key and must be dropped.
|
|
14
|
+
*/
|
|
15
|
+
isDuplicate(record: OutputRecord): boolean;
|
|
16
|
+
}
|
|
17
|
+
//# sourceMappingURL=dedupe.policy.d.ts.map
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export type { RecordSink, SinkSummary } from './record-sink.contract.js';
|
|
2
|
+
export { memorySink } from './memory-sink.repository.js';
|
|
3
|
+
export type { MemorySink } from './memory-sink.repository.js';
|
|
4
|
+
export { jsonLinesSink } from './json-lines-sink.repository.js';
|
|
5
|
+
export type { JsonLinesSinkOptions } from './json-lines-sink.repository.js';
|
|
6
|
+
export { DedupePolicy } from './dedupe.policy.js';
|
|
7
|
+
export type { DedupeScope } from './dedupe.policy.js';
|
|
8
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { RecordSink } from './record-sink.contract.js';
|
|
2
|
+
export interface JsonLinesSinkOptions {
|
|
3
|
+
/**
|
|
4
|
+
* Keep what the file holds and add to it. Each line then carries the record
|
|
5
|
+
* key as `_key`, and `open` reads the keys already there so a resumed run
|
|
6
|
+
* (`CrawlOptions.resume`) can skip them.
|
|
7
|
+
*/
|
|
8
|
+
append?: boolean;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* A sink that writes one JSON object per line to a file (JSON Lines). Each line
|
|
12
|
+
* is the record's `data` plus a `_source` member (and `_key` in append mode).
|
|
13
|
+
*
|
|
14
|
+
* @param path - The file to write; created with its directories, truncated on open unless `append`.
|
|
15
|
+
* @param options - Append mode.
|
|
16
|
+
* @returns The sink.
|
|
17
|
+
*/
|
|
18
|
+
export declare function jsonLinesSink(path: string, options?: JsonLinesSinkOptions): RecordSink;
|
|
19
|
+
//# sourceMappingURL=json-lines-sink.repository.d.ts.map
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { OutputRecord } from '../output-mapping/index.js';
|
|
2
|
+
import type { RecordSink } from './record-sink.contract.js';
|
|
3
|
+
/** A sink that keeps records in memory; `records` is readable at any time. */
|
|
4
|
+
export interface MemorySink extends RecordSink {
|
|
5
|
+
readonly records: OutputRecord[];
|
|
6
|
+
}
|
|
7
|
+
/** @returns A new in-memory sink. */
|
|
8
|
+
export declare function memorySink(): MemorySink;
|
|
9
|
+
//# sourceMappingURL=memory-sink.repository.d.ts.map
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { OutputRecord } from '../output-mapping/index.js';
|
|
2
|
+
import type { OutputRecipe } from '../recipe-schema/index.js';
|
|
3
|
+
/** What a sink reports when closed. */
|
|
4
|
+
export interface SinkSummary {
|
|
5
|
+
written: number;
|
|
6
|
+
/** Where the records went, when the sink has a location. */
|
|
7
|
+
location?: string;
|
|
8
|
+
}
|
|
9
|
+
/** Where validated records go. Opened once per run, written per record, closed once. */
|
|
10
|
+
export interface RecordSink {
|
|
11
|
+
open: (output: OutputRecipe) => Promise<void>;
|
|
12
|
+
write: (record: OutputRecord) => Promise<void>;
|
|
13
|
+
close: () => Promise<SinkSummary>;
|
|
14
|
+
/** Whether a record with this key is already there; what a resumed run asks before writing. */
|
|
15
|
+
has?: (key: string) => Promise<boolean>;
|
|
16
|
+
}
|
|
17
|
+
//# sourceMappingURL=record-sink.contract.d.ts.map
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { Cheerio, CheerioAPI } from 'cheerio';
|
|
2
|
+
import type { AnyNode } from 'domhandler';
|
|
3
|
+
/** A matched HTML element together with the document it belongs to. */
|
|
4
|
+
export interface HtmlMatch {
|
|
5
|
+
api: CheerioAPI;
|
|
6
|
+
element: Cheerio<AnyNode>;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Runs a CSS selector on static HTML. A whole document is parsed as one; anything
|
|
10
|
+
* else (a table row, a list item taken with `take: "html"`) is parsed as a
|
|
11
|
+
* fragment, so cells and rows outside a table survive instead of being dropped.
|
|
12
|
+
*
|
|
13
|
+
* @param html - The markup (a whole document or a fragment).
|
|
14
|
+
* @param selector - A CSS selector.
|
|
15
|
+
* @returns Every match, in document order.
|
|
16
|
+
*/
|
|
17
|
+
export declare function selectHtml(html: string, selector: string): HtmlMatch[];
|
|
18
|
+
//# sourceMappingURL=html-selector.algorithm.d.ts.map
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export { selectJson } from './json-path.algorithm.js';
|
|
2
|
+
export { selectRegex } from './regex.algorithm.js';
|
|
3
|
+
export { selectHtml } from './html-selector.algorithm.js';
|
|
4
|
+
export type { HtmlMatch } from './html-selector.algorithm.js';
|
|
5
|
+
export { takeFromHtml, takeFromJson, collapse } from './take-value.mapper.js';
|
|
6
|
+
export type { Take } from './take-value.mapper.js';
|
|
7
|
+
export { tryParseJson, parseJsonText, dataItemsOf } from './json-text.algorithm.js';
|
|
8
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Evaluates a JSONPath expression on a decoded JSON document.
|
|
3
|
+
*
|
|
4
|
+
* @param document - The JSON value.
|
|
5
|
+
* @param path - A JSONPath such as `$.items[*].url`.
|
|
6
|
+
* @returns Every match, in document order.
|
|
7
|
+
*/
|
|
8
|
+
export declare function selectJson(document: unknown, path: string): unknown[];
|
|
9
|
+
//# sourceMappingURL=json-path.algorithm.d.ts.map
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JSON that arrives as text: a `<script type="application/ld+json">` body, a
|
|
3
|
+
* `data-*` attribute, a fetched document read as text. Sites wrap inline JSON
|
|
4
|
+
* in comment guards, which are stripped before parsing.
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Parses text as JSON, guards stripped.
|
|
8
|
+
*
|
|
9
|
+
* @param text - The text.
|
|
10
|
+
* @returns The value, or `undefined` when it is not JSON.
|
|
11
|
+
*/
|
|
12
|
+
export declare function tryParseJson(text: string): unknown;
|
|
13
|
+
/**
|
|
14
|
+
* Parses text that must be JSON.
|
|
15
|
+
*
|
|
16
|
+
* @param text - The text.
|
|
17
|
+
* @param id - What the text is, for the error.
|
|
18
|
+
* @returns The value.
|
|
19
|
+
* @throws Error when it is not JSON.
|
|
20
|
+
*/
|
|
21
|
+
export declare function parseJsonText(text: string, id: string): unknown;
|
|
22
|
+
/**
|
|
23
|
+
* The data a value holds, whatever shape it arrived in: JSON text is parsed, a
|
|
24
|
+
* list of texts becomes the list of its parsable entries, and an entry that
|
|
25
|
+
* parses to a list is spliced in. Data that is not text is kept as is.
|
|
26
|
+
*
|
|
27
|
+
* @param value - A bound value: data, text, or a list of either.
|
|
28
|
+
* @returns A list of items.
|
|
29
|
+
*/
|
|
30
|
+
export declare function dataItemsOf(value: unknown): unknown[];
|
|
31
|
+
//# sourceMappingURL=json-text.algorithm.d.ts.map
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Evaluates a regular expression on text: the extract kind for values that
|
|
3
|
+
* live in inline scripts, attributes or prose rather than in elements or JSON.
|
|
4
|
+
*
|
|
5
|
+
* @param text - The document text.
|
|
6
|
+
* @param pattern - A regular expression source; group 1 is returned when the
|
|
7
|
+
* pattern has a capturing group, else the whole match.
|
|
8
|
+
* @returns Every match, in document order.
|
|
9
|
+
* @throws When the pattern is not a valid regular expression.
|
|
10
|
+
*/
|
|
11
|
+
export declare function selectRegex(text: string, pattern: string): string[];
|
|
12
|
+
//# sourceMappingURL=regex.algorithm.d.ts.map
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { HtmlMatch } from './html-selector.algorithm.js';
|
|
2
|
+
/** What `extract` takes from a match: text, inner HTML, an attribute, an input's value, or the node as data. */
|
|
3
|
+
export type Take = 'text' | 'html' | 'value' | 'json' | `attr:${string}`;
|
|
4
|
+
/**
|
|
5
|
+
* The value of an HTML match.
|
|
6
|
+
*
|
|
7
|
+
* @param match - A selected element.
|
|
8
|
+
* @param take - What to take; `text` collapses whitespace.
|
|
9
|
+
* @returns The value; `undefined` for a missing attribute.
|
|
10
|
+
*/
|
|
11
|
+
export declare function takeFromHtml(match: HtmlMatch, take: Take): unknown;
|
|
12
|
+
/**
|
|
13
|
+
* The value of a JSON match.
|
|
14
|
+
*
|
|
15
|
+
* @param node - A JSONPath result.
|
|
16
|
+
* @param take - `json` keeps the node; `text` stringifies scalars.
|
|
17
|
+
* @returns The value.
|
|
18
|
+
*/
|
|
19
|
+
export declare function takeFromJson(node: unknown, take: Take): unknown;
|
|
20
|
+
/**
|
|
21
|
+
* Text as a human reads it: runs of whitespace collapsed, ends trimmed.
|
|
22
|
+
*
|
|
23
|
+
* @param text - Raw text content.
|
|
24
|
+
* @returns The collapsed text.
|
|
25
|
+
*/
|
|
26
|
+
export declare function collapse(text: string): string;
|
|
27
|
+
//# sourceMappingURL=take-value.mapper.d.ts.map
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { BlockRule } from '../recipe-schema/index.js';
|
|
2
|
+
import { BlockedError } from './blocked.error.js';
|
|
3
|
+
/** A block unless a recipe says otherwise: forbidden, rate limited, or an AWS WAF challenge (IMDb answers 202 with it). */
|
|
4
|
+
export declare const DEFAULT_BLOCK_RULE: BlockRule;
|
|
5
|
+
/** The parts of a response a block rule reads. The body is read only when the rule has a `text` condition. */
|
|
6
|
+
export interface ObservedResponse {
|
|
7
|
+
url: string;
|
|
8
|
+
status: number;
|
|
9
|
+
/** Header names in lower case, as Playwright reports them. */
|
|
10
|
+
headers: Record<string, string>;
|
|
11
|
+
text?: () => Promise<string>;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Whether a response is a block.
|
|
15
|
+
*
|
|
16
|
+
* @param response - What came back.
|
|
17
|
+
* @param rule - The recipe's `session.blockedWhen`; `DEFAULT_BLOCK_RULE` when omitted.
|
|
18
|
+
* @returns The error to throw, or `undefined` when the response is not a block.
|
|
19
|
+
*/
|
|
20
|
+
export declare function detectBlock(response: ObservedResponse, rule?: BlockRule): Promise<BlockedError | undefined>;
|
|
21
|
+
//# sourceMappingURL=block-rule.policy.d.ts.map
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** A response the recipe's `session.blockedWhen` rule (or the default one) says is the site refusing the crawl. */
|
|
2
|
+
export declare class BlockedError extends Error {
|
|
3
|
+
readonly url: string;
|
|
4
|
+
readonly status: number;
|
|
5
|
+
readonly reason: string;
|
|
6
|
+
readonly name = "BlockedError";
|
|
7
|
+
constructor(url: string, status: number, reason: string);
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=blocked.error.d.ts.map
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { ExtractionScope } from '../extraction-scope/index.js';
|
|
2
|
+
import type { ForEachStep } from '../recipe-schema/index.js';
|
|
3
|
+
import type { EmitOutcome, StepWalk } from './run-steps.use-case.js';
|
|
4
|
+
/**
|
|
5
|
+
* Runs a body once per item of a list (`over`), or once per live element
|
|
6
|
+
* matching `selector`, each in a fresh child scope with the item bound under
|
|
7
|
+
* `as`; emits a record per iteration when asked.
|
|
8
|
+
*
|
|
9
|
+
* With a concurrent gate, iterations run as permits allow and records come
|
|
10
|
+
* out in completion order; without one, in list order.
|
|
11
|
+
*
|
|
12
|
+
* @param step - The forEach step.
|
|
13
|
+
* @param scope - The scope the list lives in.
|
|
14
|
+
* @param walk - Runs a step list; also carries the emit callback.
|
|
15
|
+
* @returns `stop` when the crawl reached its record limit.
|
|
16
|
+
*/
|
|
17
|
+
export declare function runForEach(step: ForEachStep, scope: ExtractionScope, walk: StepWalk): Promise<EmitOutcome>;
|
|
18
|
+
//# sourceMappingURL=for-each.use-case.d.ts.map
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export { runSteps } from './run-steps.use-case.js';
|
|
2
|
+
export type { StepWalkOptions, EmitOutcome } from './run-steps.use-case.js';
|
|
3
|
+
export type { StepRunner, NextPageResult } from './step-runner.contract.js';
|
|
4
|
+
export { StepFailure, NoMatchError } from './step-failure.error.js';
|
|
5
|
+
export { resolveErrorPolicy, backoffFor, sleep } from './retry.policy.js';
|
|
6
|
+
export { RunGate } from './run-gate.policy.js';
|
|
7
|
+
export { BlockedError } from './blocked.error.js';
|
|
8
|
+
export { detectBlock, DEFAULT_BLOCK_RULE } from './block-rule.policy.js';
|
|
9
|
+
export type { ObservedResponse } from './block-rule.policy.js';
|
|
10
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { ExtractionScope } from '../extraction-scope/index.js';
|
|
2
|
+
import type { PaginateStep } from '../recipe-schema/index.js';
|
|
3
|
+
import type { EmitOutcome, StepWalk } from './run-steps.use-case.js';
|
|
4
|
+
/**
|
|
5
|
+
* Runs a body once per page, each in a fresh child scope, then asks the runner
|
|
6
|
+
* for the next page until there is none, `until` renders truthy, or `maxPages`
|
|
7
|
+
* is reached.
|
|
8
|
+
*
|
|
9
|
+
* The runner reports the visit of each new page; this only steers.
|
|
10
|
+
*
|
|
11
|
+
* @param step - The paginate step.
|
|
12
|
+
* @param scope - The scope to page in; its page URL advances with each page.
|
|
13
|
+
* @param walk - Runs a step list; carries the runner and the emit callback.
|
|
14
|
+
* @returns `stop` when the crawl reached its record limit.
|
|
15
|
+
*/
|
|
16
|
+
export declare function runPaginate(step: PaginateStep, scope: ExtractionScope, walk: StepWalk): Promise<EmitOutcome>;
|
|
17
|
+
//# sourceMappingURL=paginate.use-case.d.ts.map
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { ErrorPolicy, InputRecipe, Step } from '../recipe-schema/index.js';
|
|
2
|
+
/**
|
|
3
|
+
* The policy for a failing step: the step's own, else the recipe's, else `fail`.
|
|
4
|
+
*
|
|
5
|
+
* @param step - The step that failed.
|
|
6
|
+
* @param recipe - Its recipe.
|
|
7
|
+
* @returns The policy to apply.
|
|
8
|
+
*/
|
|
9
|
+
export declare function resolveErrorPolicy(step: Step, recipe: Pick<InputRecipe, 'onError'>): ErrorPolicy;
|
|
10
|
+
/**
|
|
11
|
+
* How long to wait before a retry: linear backoff.
|
|
12
|
+
*
|
|
13
|
+
* @param policy - A retry policy.
|
|
14
|
+
* @param attempt - The attempt about to be made, starting at 2.
|
|
15
|
+
* @returns Milliseconds.
|
|
16
|
+
*/
|
|
17
|
+
export declare function backoffFor(policy: Extract<ErrorPolicy, {
|
|
18
|
+
policy: 'retry';
|
|
19
|
+
}>, attempt: number): number;
|
|
20
|
+
export declare function sleep(ms: number): Promise<void>;
|
|
21
|
+
//# sourceMappingURL=retry.policy.d.ts.map
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What bounds a recipe run: how many `forEach` iterations may be in flight and
|
|
3
|
+
* how close together requests may start. One gate per recipe run, shared by
|
|
4
|
+
* every loop in it, so nested loops never multiply the limit.
|
|
5
|
+
*
|
|
6
|
+
* Permits go to the outermost concurrent loop: a loop that runs inside an
|
|
7
|
+
* iteration already holding a permit runs its body sequentially (see `nested`),
|
|
8
|
+
* which keeps the total at `permits` and cannot deadlock.
|
|
9
|
+
*/
|
|
10
|
+
export declare class RunGate {
|
|
11
|
+
readonly permits: number;
|
|
12
|
+
readonly minIntervalMs: number;
|
|
13
|
+
private readonly shared?;
|
|
14
|
+
private inFlight;
|
|
15
|
+
private readonly waiting;
|
|
16
|
+
private lastStart;
|
|
17
|
+
/**
|
|
18
|
+
* @param permits - Iterations allowed in flight; 1 is sequential.
|
|
19
|
+
* @param minIntervalMs - Minimum time between two request starts across the run.
|
|
20
|
+
* @param shared - The throttle state to share (internal: `nested` gates keep their parent's).
|
|
21
|
+
*/
|
|
22
|
+
constructor(permits: number, minIntervalMs: number, shared?: RunGate | undefined);
|
|
23
|
+
/** Whether this gate lets more than one iteration run at once. */
|
|
24
|
+
get concurrent(): boolean;
|
|
25
|
+
/**
|
|
26
|
+
* Takes a permit, waiting for one when all are in flight.
|
|
27
|
+
*
|
|
28
|
+
* @returns The release; call it exactly once, when the iteration ends.
|
|
29
|
+
*/
|
|
30
|
+
acquire(): Promise<() => void>;
|
|
31
|
+
/**
|
|
32
|
+
* Waits until a request may start: `minIntervalMs` after the previous start,
|
|
33
|
+
* whichever loop started it. Returns at once when the interval has passed.
|
|
34
|
+
*/
|
|
35
|
+
throttle(): Promise<void>;
|
|
36
|
+
/** The gate for a body running inside an iteration that holds a permit: sequential, same throttle. */
|
|
37
|
+
nested(): RunGate;
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=run-gate.policy.d.ts.map
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { EventBus } from '../crawl-events/index.js';
|
|
2
|
+
import type { ExtractionScope } from '../extraction-scope/index.js';
|
|
3
|
+
import type { HookRegistry } from '../hooks/index.js';
|
|
4
|
+
import type { InputRecipe, Step } from '../recipe-schema/index.js';
|
|
5
|
+
import type { RunGate } from './run-gate.policy.js';
|
|
6
|
+
import type { StepRunner } from './step-runner.contract.js';
|
|
7
|
+
/** Whether the walk goes on after a record was emitted. */
|
|
8
|
+
export type EmitOutcome = 'continue' | 'stop';
|
|
9
|
+
/** Everything a step walk needs. */
|
|
10
|
+
export interface StepWalkOptions {
|
|
11
|
+
recipe: InputRecipe;
|
|
12
|
+
runner: StepRunner;
|
|
13
|
+
hooks: HookRegistry;
|
|
14
|
+
events: EventBus;
|
|
15
|
+
/** Called with the scope to snapshot for each record; `stop` ends the walk. */
|
|
16
|
+
onEmit: (scope: ExtractionScope, output?: string) => Promise<EmitOutcome>;
|
|
17
|
+
/** Bounds concurrency and request rate; absent means sequential and unthrottled. */
|
|
18
|
+
gate?: RunGate;
|
|
19
|
+
}
|
|
20
|
+
/** The walk as the control-flow steps see it: options plus the current path. */
|
|
21
|
+
export interface StepWalk extends StepWalkOptions {
|
|
22
|
+
path: string;
|
|
23
|
+
/** Runs a nested step list; `overrides` replace walk options for that list (a sequential gate inside a concurrent iteration). */
|
|
24
|
+
runSteps: (steps: readonly Step[], scope: ExtractionScope, path: string, overrides?: Partial<StepWalkOptions>) => Promise<EmitOutcome>;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Walks a step list in order. Control flow (`forEach`, `if`, `paginate`, `emit`,
|
|
28
|
+
* `set`, `hook`, `when`, error policies) is handled here; leaf steps go to the
|
|
29
|
+
* runner. Mode-agnostic: the same walk drives a browser page or an HTTP context.
|
|
30
|
+
*
|
|
31
|
+
* @param steps - The steps.
|
|
32
|
+
* @param scope - The scope to run in.
|
|
33
|
+
* @param options - Recipe, runner, hooks, events and the emit callback.
|
|
34
|
+
* @param path - Where these steps are, for messages and events.
|
|
35
|
+
* @returns `stop` when the emit callback asked to stop.
|
|
36
|
+
* @throws StepFailure when a step fails under the `fail` policy.
|
|
37
|
+
*/
|
|
38
|
+
export declare function runSteps(steps: readonly Step[], scope: ExtractionScope, options: StepWalkOptions, path?: string): Promise<EmitOutcome>;
|
|
39
|
+
//# sourceMappingURL=run-steps.use-case.d.ts.map
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/** A step that failed under the `fail` policy: the recipe stops here. */
|
|
2
|
+
export declare class StepFailure extends Error {
|
|
3
|
+
readonly stepPath: string;
|
|
4
|
+
readonly stepType: string;
|
|
5
|
+
readonly name = "StepFailure";
|
|
6
|
+
constructor(stepPath: string, stepType: string, cause: unknown);
|
|
7
|
+
}
|
|
8
|
+
/** A single `extract` that matched nothing. */
|
|
9
|
+
export declare class NoMatchError extends Error {
|
|
10
|
+
readonly selector: string;
|
|
11
|
+
readonly name = "NoMatchError";
|
|
12
|
+
constructor(selector: string);
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=step-failure.error.d.ts.map
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { ExtractionScope, LiveElement } from '../extraction-scope/index.js';
|
|
2
|
+
import type { PaginateNext, Step } from '../recipe-schema/index.js';
|
|
3
|
+
import type { BlockedError } from './blocked.error.js';
|
|
4
|
+
/** What `paginate` learns from the runner after a page body ran. */
|
|
5
|
+
export type NextPageResult =
|
|
6
|
+
/** The next page is at this URL (the runner already navigated in web mode). */
|
|
7
|
+
{
|
|
8
|
+
kind: 'url';
|
|
9
|
+
url: string;
|
|
10
|
+
} |
|
|
11
|
+
/** A value to bind in the next page's scope under `name` (a cursor, a token). */
|
|
12
|
+
{
|
|
13
|
+
kind: 'value';
|
|
14
|
+
name: string;
|
|
15
|
+
value: unknown;
|
|
16
|
+
} |
|
|
17
|
+
/** There is no next page. */
|
|
18
|
+
null;
|
|
19
|
+
/**
|
|
20
|
+
* The mode-specific half of step execution. `step-flow` walks the tree and
|
|
21
|
+
* owns control flow; a runner executes leaf steps against a page or a request
|
|
22
|
+
* context. Runners never mutate scope bindings other than through this contract.
|
|
23
|
+
*/
|
|
24
|
+
export interface StepRunner {
|
|
25
|
+
/** Runs one leaf step (not forEach, paginate, emit, set or hook). */
|
|
26
|
+
runLeaf: (step: Step, scope: ExtractionScope) => Promise<void>;
|
|
27
|
+
/** Finds (and in web mode reaches) the next page. */
|
|
28
|
+
nextPage: (next: PaginateNext, scope: ExtractionScope) => Promise<NextPageResult>;
|
|
29
|
+
/** Snapshots every element matching a rendered selector, for `forEach` over `selector`. Web mode only. */
|
|
30
|
+
elements?: (selector: string, scope: ExtractionScope) => Promise<LiveElement[]>;
|
|
31
|
+
/**
|
|
32
|
+
* Called when a step is blocked. `true` means the runner now reaches the
|
|
33
|
+
* network another way (a new access lease) and the step should run again;
|
|
34
|
+
* `false` means it cannot, and the block fails the step like any error.
|
|
35
|
+
*/
|
|
36
|
+
rotate?: (error: BlockedError) => Promise<boolean>;
|
|
37
|
+
dispose: () => Promise<void>;
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=step-runner.contract.d.ts.map
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A small expression language for `{{ }}` placeholders: literals, paths,
|
|
3
|
+
* arithmetic, comparison, logic, `??`, `a ? b : c` and a fixed set of
|
|
4
|
+
* functions. It is parsed into a tree and walked; nothing is ever compiled or
|
|
5
|
+
* evaluated as JavaScript, and a path can only read own properties of plain
|
|
6
|
+
* data, so `constructor`, `__proto__` and friends resolve to nothing.
|
|
7
|
+
*/
|
|
8
|
+
import type { Lookup } from './value-text.algorithm.js';
|
|
9
|
+
export type Expression = {
|
|
10
|
+
kind: 'literal';
|
|
11
|
+
value: unknown;
|
|
12
|
+
} | {
|
|
13
|
+
kind: 'path';
|
|
14
|
+
path: string;
|
|
15
|
+
} | {
|
|
16
|
+
kind: 'unary';
|
|
17
|
+
op: '-' | '!';
|
|
18
|
+
operand: Expression;
|
|
19
|
+
} | {
|
|
20
|
+
kind: 'binary';
|
|
21
|
+
op: string;
|
|
22
|
+
left: Expression;
|
|
23
|
+
right: Expression;
|
|
24
|
+
} | {
|
|
25
|
+
kind: 'ternary';
|
|
26
|
+
test: Expression;
|
|
27
|
+
consequent: Expression;
|
|
28
|
+
alternate: Expression;
|
|
29
|
+
} | {
|
|
30
|
+
kind: 'call';
|
|
31
|
+
name: string;
|
|
32
|
+
args: Expression[];
|
|
33
|
+
};
|
|
34
|
+
/** The names an expression may call. */
|
|
35
|
+
export declare const EXPRESSION_FUNCTIONS: readonly string[];
|
|
36
|
+
/**
|
|
37
|
+
* Parses an expression, with a small cache since templates render per iteration.
|
|
38
|
+
*
|
|
39
|
+
* @param source - The text inside the placeholder.
|
|
40
|
+
* @returns The tree.
|
|
41
|
+
* @throws Error naming the position of the first problem.
|
|
42
|
+
*/
|
|
43
|
+
export declare function parseExpression(source: string): Expression;
|
|
44
|
+
/**
|
|
45
|
+
* Evaluates a tree against a lookup. A function found where data is expected
|
|
46
|
+
* counts as `undefined`, so nothing callable ever leaks out of the scope.
|
|
47
|
+
*
|
|
48
|
+
* @param expression - The tree.
|
|
49
|
+
* @param lookup - Resolves a path.
|
|
50
|
+
* @returns The value.
|
|
51
|
+
*/
|
|
52
|
+
export declare function evaluateExpression(expression: Expression, lookup: Lookup): unknown;
|
|
53
|
+
//# sourceMappingURL=expression.algorithm.d.ts.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export { getPath, setPath, segmentsOf } from './value-path.algorithm.js';
|
|
2
|
+
export { render, renderDeep, renderText, resolve, hasPlaceholder } from './template.algorithm.js';
|
|
3
|
+
export { isTruthy, stringify } from './value-text.algorithm.js';
|
|
4
|
+
export { parseExpression, evaluateExpression, EXPRESSION_FUNCTIONS } from './expression.algorithm.js';
|
|
5
|
+
export type { Expression } from './expression.algorithm.js';
|
|
6
|
+
export type { Lookup } from './value-text.algorithm.js';
|
|
7
|
+
//# sourceMappingURL=index.d.ts.map
|