@opencraw/core 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access/access-broker.use-case.d.ts +43 -0
- package/dist/access/access-config.error.d.ts +5 -0
- package/dist/access/access-config.repository.d.ts +11 -0
- package/dist/access/access-json-schema.mapper.d.ts +8 -0
- package/dist/access/access-plugin.contract.d.ts +48 -0
- package/dist/access/access-preset.store.d.ts +19 -0
- package/dist/access/access-profile.contract.d.ts +95 -0
- package/dist/access/index.d.ts +11 -0
- package/dist/access/render-access.mapper.d.ts +41 -0
- package/dist/access/session-id.algorithm.d.ts +8 -0
- package/dist/api-steps/extract-from-document.use-case.d.ts +41 -0
- package/dist/api-steps/index.d.ts +4 -0
- package/dist/api-steps/run-api-step.use-case.d.ts +20 -0
- package/dist/api-steps/send-request.use-case.d.ts +23 -0
- package/dist/browser-session/browser-profile.store.d.ts +52 -0
- package/dist/browser-session/browser-session.config.d.ts +21 -0
- package/dist/browser-session/browser.client.d.ts +75 -0
- package/dist/browser-session/index.d.ts +6 -0
- package/dist/captcha/captcha-budget.model.d.ts +21 -0
- package/dist/captcha/captcha-detection.client.d.ts +28 -0
- package/dist/captcha/captcha-guard.use-case.d.ts +64 -0
- package/dist/captcha/captcha-solver-registry.store.d.ts +19 -0
- package/dist/captcha/captcha-solver.contract.d.ts +47 -0
- package/dist/captcha/captcha.error.d.ts +13 -0
- package/dist/captcha/index.d.ts +10 -0
- package/dist/captcha/resolve-captcha.use-case.d.ts +40 -0
- package/dist/crawl-events/crawl-event.contract.d.ts +169 -0
- package/dist/crawl-events/event-bus.store.d.ts +10 -0
- package/dist/crawl-events/index.d.ts +4 -0
- package/dist/crawl-events/trace-line.mapper.d.ts +14 -0
- package/dist/crawl-execution/bootstrap-session.use-case.d.ts +79 -0
- package/dist/crawl-execution/crawl-options.config.d.ts +67 -0
- package/dist/crawl-execution/crawl-report.model.d.ts +33 -0
- package/dist/crawl-execution/create-crawler.use-case.d.ts +20 -0
- package/dist/crawl-execution/index.d.ts +8 -0
- package/dist/crawl-execution/rotating-runner.use-case.d.ts +66 -0
- package/dist/crawl-execution/run-crawl.use-case.d.ts +17 -0
- package/dist/crawl-execution/run-input-recipe.use-case.d.ts +51 -0
- package/dist/deck-document/deck-document.model.d.ts +58 -0
- package/dist/deck-document/deck-table.algorithm.d.ts +35 -0
- package/dist/deck-document/index.d.ts +6 -0
- package/dist/deck-document/read-pptx.client.d.ts +16 -0
- package/dist/docx-document/index.d.ts +2 -0
- package/dist/docx-document/read-docx-html.client.d.ts +23 -0
- package/dist/extraction-scope/extraction-scope.model.d.ts +77 -0
- package/dist/extraction-scope/index.d.ts +5 -0
- package/dist/extraction-scope/live-element.model.d.ts +28 -0
- package/dist/hooks/hook-registry.store.d.ts +16 -0
- package/dist/hooks/hook.contract.d.ts +17 -0
- package/dist/hooks/hook.error.d.ts +7 -0
- package/dist/hooks/index.d.ts +4 -0
- package/dist/http-session/http-response.contract.d.ts +58 -0
- package/dist/http-session/http.client.d.ts +42 -0
- package/dist/http-session/index.d.ts +5 -0
- package/dist/http-session/text-decoding.algorithm.d.ts +35 -0
- package/dist/index.esm.js +649 -25
- package/dist/markdown-document/index.d.ts +3 -0
- package/dist/markdown-document/read-markdown.client.d.ts +39 -0
- package/dist/output-mapping/coerce-field.mapper.d.ts +19 -0
- package/dist/output-mapping/generated-field.mapper.d.ts +16 -0
- package/dist/output-mapping/index.d.ts +9 -0
- package/dist/output-mapping/map-record.use-case.d.ts +27 -0
- package/dist/output-mapping/mapping.error.d.ts +17 -0
- package/dist/output-mapping/missing-value.policy.d.ts +14 -0
- package/dist/output-mapping/output-field.validator.d.ts +10 -0
- package/dist/output-mapping/output-record.model.d.ts +20 -0
- package/dist/pdf-document/index.d.ts +7 -0
- package/dist/pdf-document/pdf-document.model.d.ts +51 -0
- package/dist/pdf-document/pdf-table.algorithm.d.ts +43 -0
- package/dist/pdf-document/read-pdf.client.d.ts +18 -0
- package/dist/pdf-document/row-assembly.algorithm.d.ts +26 -0
- package/dist/recipe-loading/index.d.ts +9 -0
- package/dist/recipe-loading/load-recipe-set.use-case.d.ts +45 -0
- package/dist/recipe-loading/read-recipe-source.use-case.d.ts +14 -0
- package/dist/recipe-loading/recipe-binding.error.d.ts +14 -0
- package/dist/recipe-loading/recipe-binding.validator.d.ts +34 -0
- package/dist/recipe-loading/recipe-file.repository.d.ts +11 -0
- package/dist/recipe-loading/recipe-set.model.d.ts +12 -0
- package/dist/recipe-loading/recipe-source.contract.d.ts +28 -0
- package/dist/recipe-loading/recipe-text.mapper.d.ts +14 -0
- package/dist/recipe-schema/index.d.ts +15 -0
- package/dist/recipe-schema/input-recipe.contract.d.ts +156 -0
- package/dist/recipe-schema/json-schema.mapper.d.ts +15 -0
- package/dist/recipe-schema/output-recipe.contract.d.ts +47 -0
- package/dist/recipe-schema/recipe-kind.enum.d.ts +35 -0
- package/dist/recipe-schema/recipe-validation.error.d.ts +17 -0
- package/dist/recipe-schema/recipe.validator.d.ts +28 -0
- package/dist/recipe-schema/step.contract.d.ts +206 -0
- package/dist/recipe-schema/transform-rule.contract.d.ts +120 -0
- package/dist/record-diff/index.d.ts +4 -0
- package/dist/record-diff/record-diff.algorithm.d.ts +82 -0
- package/dist/record-diff/records-file.repository.d.ts +10 -0
- package/dist/record-sink/dedupe.policy.d.ts +29 -0
- package/dist/record-sink/index.d.ts +9 -0
- package/dist/record-sink/json-lines-sink.repository.d.ts +19 -0
- package/dist/record-sink/memory-sink.repository.d.ts +9 -0
- package/dist/record-sink/record-sink.contract.d.ts +17 -0
- package/dist/selection/html-selector.algorithm.d.ts +19 -0
- package/dist/selection/index.d.ts +8 -0
- package/dist/selection/json-path.algorithm.d.ts +9 -0
- package/dist/selection/json-text.algorithm.d.ts +58 -0
- package/dist/selection/regex.algorithm.d.ts +12 -0
- package/dist/selection/take-value.mapper.d.ts +27 -0
- package/dist/src/api-steps/extract-from-document.use-case.d.ts +5 -5
- package/dist/src/docx-document/index.d.ts +2 -0
- package/dist/src/docx-document/read-docx-html.client.d.ts +23 -0
- package/dist/src/extraction-scope/extraction-scope.model.d.ts +2 -1
- package/dist/src/http-session/http-response.contract.d.ts +2 -1
- package/dist/src/index.d.ts +4 -0
- package/dist/src/markdown-document/index.d.ts +1 -1
- package/dist/src/markdown-document/read-markdown.client.d.ts +10 -0
- package/dist/src/recipe-schema/recipe-kind.enum.d.ts +1 -1
- package/dist/src/recipe-schema/step.contract.d.ts +4 -0
- package/dist/src/record-diff/index.d.ts +4 -0
- package/dist/src/record-diff/record-diff.algorithm.d.ts +82 -0
- package/dist/src/record-diff/records-file.repository.d.ts +10 -0
- package/dist/src/selection/html-selector.algorithm.d.ts +2 -1
- package/dist/src/xml-document/index.d.ts +6 -0
- package/dist/src/xml-document/xml-document.model.d.ts +13 -0
- package/dist/src/xml-document/xml-parser.client.d.ts +49 -0
- package/dist/src/xml-document/xpath.algorithm.d.ts +30 -0
- package/dist/step-flow/block-rule.policy.d.ts +21 -0
- package/dist/step-flow/blocked.error.d.ts +9 -0
- package/dist/step-flow/for-each.use-case.d.ts +20 -0
- package/dist/step-flow/host-throttle.policy.d.ts +49 -0
- package/dist/step-flow/index.d.ts +15 -0
- package/dist/step-flow/paginate.use-case.d.ts +17 -0
- package/dist/step-flow/retry.policy.d.ts +21 -0
- package/dist/step-flow/run-gate.policy.d.ts +50 -0
- package/dist/step-flow/run-steps.use-case.d.ts +39 -0
- package/dist/step-flow/step-failure.error.d.ts +14 -0
- package/dist/step-flow/step-runner.contract.d.ts +52 -0
- package/dist/step-flow/transport-retry.policy.d.ts +76 -0
- package/dist/template/expression.algorithm.d.ts +53 -0
- package/dist/template/index.d.ts +7 -0
- package/dist/template/template.algorithm.d.ts +53 -0
- package/dist/template/value-path.algorithm.d.ts +29 -0
- package/dist/template/value-text.algorithm.d.ts +18 -0
- package/dist/transformation/apply-transform-chain.use-case.d.ts +17 -0
- package/dist/transformation/collection.algorithm.d.ts +15 -0
- package/dist/transformation/currency.algorithm.d.ts +15 -0
- package/dist/transformation/date.algorithm.d.ts +13 -0
- package/dist/transformation/index.d.ts +10 -0
- package/dist/transformation/lookup.algorithm.d.ts +24 -0
- package/dist/transformation/number.algorithm.d.ts +30 -0
- package/dist/transformation/string.algorithm.d.ts +18 -0
- package/dist/transformation/transform-registry.store.d.ts +36 -0
- package/dist/transformation/transform.error.d.ts +8 -0
- package/dist/transformation/url.algorithm.d.ts +20 -0
- package/dist/web-steps/evaluate-script.use-case.d.ts +10 -0
- package/dist/web-steps/extract-from-page.use-case.d.ts +10 -0
- package/dist/web-steps/index.d.ts +2 -0
- package/dist/web-steps/interact.use-case.d.ts +36 -0
- package/dist/web-steps/navigate.use-case.d.ts +15 -0
- package/dist/web-steps/run-web-step.use-case.d.ts +37 -0
- package/dist/web-steps/snapshot-elements.use-case.d.ts +13 -0
- package/dist/workbook-document/csv-parser.algorithm.d.ts +26 -0
- package/dist/workbook-document/csv-workbook.mapper.d.ts +24 -0
- package/dist/workbook-document/grid-table.algorithm.d.ts +53 -0
- package/dist/workbook-document/html-tables.mapper.d.ts +14 -0
- package/dist/workbook-document/index.d.ts +9 -0
- package/dist/workbook-document/read-xlsx.client.d.ts +18 -0
- package/dist/workbook-document/workbook-document.model.d.ts +51 -0
- package/dist/xml-document/index.d.ts +6 -0
- package/dist/xml-document/xml-document.model.d.ts +13 -0
- package/dist/xml-document/xml-parser.client.d.ts +49 -0
- package/dist/xml-document/xpath.algorithm.d.ts +30 -0
- package/dist/yaml-document/index.d.ts +3 -0
- package/dist/yaml-document/read-yaml.client.d.ts +25 -0
- package/package.json +4 -2
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/** A parsed document, as the XPath engine sees it. */
|
|
2
|
+
export type XmlNode = Node;
|
|
3
|
+
/**
|
|
4
|
+
* Parses XML strictly but safely: a malformed document is an error naming
|
|
5
|
+
* the URL and the line; entities a DOCTYPE declares are never expanded (no
|
|
6
|
+
* "billion laughs") and nothing external is ever fetched (no XXE), they stay
|
|
7
|
+
* as they are.
|
|
8
|
+
*
|
|
9
|
+
* @param xml - The document text.
|
|
10
|
+
* @param where - The URL or id, for messages.
|
|
11
|
+
* @returns The document.
|
|
12
|
+
* @throws Error when the text is not well-formed XML.
|
|
13
|
+
*/
|
|
14
|
+
export declare function parseXml(xml: string, where: string): Document;
|
|
15
|
+
/**
|
|
16
|
+
* HTML as an XML document, for XPath on fetched pages: parsed the way a
|
|
17
|
+
* browser parses it (a forgiving HTML5 parser that inserts `<tbody>` and
|
|
18
|
+
* closes what the page left open), without namespaces, so `//table/tbody/tr`
|
|
19
|
+
* finds what it finds on the live page.
|
|
20
|
+
*
|
|
21
|
+
* @param html - A page or a fragment.
|
|
22
|
+
* @returns The document.
|
|
23
|
+
*/
|
|
24
|
+
export declare function htmlAsXml(html: string): Document;
|
|
25
|
+
/**
|
|
26
|
+
* A copy of a document with every namespace dropped: elements and attributes
|
|
27
|
+
* keep their local names, `xmlns` declarations go. What `ignoreNamespaces`
|
|
28
|
+
* queries run on: `//entry/title` instead of `//atom:entry/atom:title`.
|
|
29
|
+
*
|
|
30
|
+
* @param document - A parsed document.
|
|
31
|
+
* @returns The copy.
|
|
32
|
+
*/
|
|
33
|
+
export declare function withoutNamespaces(document: Document): Document;
|
|
34
|
+
/**
|
|
35
|
+
* A node as markup: the element with its children (`outer`), or its children only.
|
|
36
|
+
*
|
|
37
|
+
* @param node - A node.
|
|
38
|
+
* @param outer - Whether to include the node itself.
|
|
39
|
+
* @returns The markup.
|
|
40
|
+
*/
|
|
41
|
+
export declare function serializeXml(node: Node, outer: boolean): string;
|
|
42
|
+
/**
|
|
43
|
+
* The text of an XML body, gunzipped first when it is gzip (a `sitemap.xml.gz`).
|
|
44
|
+
*
|
|
45
|
+
* @param bytes - The body.
|
|
46
|
+
* @returns The bytes to decode.
|
|
47
|
+
*/
|
|
48
|
+
export declare function gunzipIfNeeded(bytes: Uint8Array): Uint8Array;
|
|
49
|
+
//# sourceMappingURL=xml-parser.client.d.ts.map
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { Take } from '../selection/index.js';
|
|
2
|
+
/** An XPath result: a node, or what a function such as `count()` or `string()` gives. */
|
|
3
|
+
export type XpathValue = Node | string | number | boolean;
|
|
4
|
+
export interface XpathOptions {
|
|
5
|
+
/** Prefix to URI, for queries on namespaced documents (`atom: http://www.w3.org/2005/Atom`). */
|
|
6
|
+
namespaces?: Record<string, string>;
|
|
7
|
+
/** Drop the document's namespaces first, so plain names match (`//entry/title`). */
|
|
8
|
+
ignoreNamespaces?: boolean;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Runs an XPath 1.0 query on a parsed document. Prefixes the root element
|
|
12
|
+
* declares are known without being listed; a default namespace has no
|
|
13
|
+
* prefix, so name one in `namespaces` or use `ignoreNamespaces`.
|
|
14
|
+
*
|
|
15
|
+
* @param document - The document.
|
|
16
|
+
* @param expression - The query.
|
|
17
|
+
* @param options - Namespaces.
|
|
18
|
+
* @returns Every node selected, or the one value a function returned.
|
|
19
|
+
* @throws Error naming the query when it does not parse or uses an unknown prefix.
|
|
20
|
+
*/
|
|
21
|
+
export declare function selectXpath(document: Document, expression: string, options?: XpathOptions): XpathValue[];
|
|
22
|
+
/**
|
|
23
|
+
* The value of an XPath result.
|
|
24
|
+
*
|
|
25
|
+
* @param value - A node or a function result.
|
|
26
|
+
* @param take - `text` (whitespace collapsed), `html` (the node's inner markup), `json` (its outer markup), `value` (its text as is), `attr:<name>`.
|
|
27
|
+
* @returns The value; `undefined` for a missing attribute.
|
|
28
|
+
*/
|
|
29
|
+
export declare function takeFromXml(value: XpathValue, take: Take): unknown;
|
|
30
|
+
//# sourceMappingURL=xpath.algorithm.d.ts.map
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { BlockRule } from '../recipe-schema/index.js';
|
|
2
|
+
import { BlockedError } from './blocked.error.js';
|
|
3
|
+
/** A block unless a recipe says otherwise: forbidden, rate limited, or an AWS WAF challenge (IMDb answers 202 with it). */
|
|
4
|
+
export declare const DEFAULT_BLOCK_RULE: BlockRule;
|
|
5
|
+
/** The parts of a response a block rule reads. The body is read only when the rule has a `text` condition. */
|
|
6
|
+
export interface ObservedResponse {
|
|
7
|
+
url: string;
|
|
8
|
+
status: number;
|
|
9
|
+
/** Header names in lower case, as Playwright reports them. */
|
|
10
|
+
headers: Record<string, string>;
|
|
11
|
+
text?: () => Promise<string>;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Whether a response is a block.
|
|
15
|
+
*
|
|
16
|
+
* @param response - What came back.
|
|
17
|
+
* @param rule - The recipe's `session.blockedWhen`; `DEFAULT_BLOCK_RULE` when omitted.
|
|
18
|
+
* @returns The error to throw, or `undefined` when the response is not a block.
|
|
19
|
+
*/
|
|
20
|
+
export declare function detectBlock(response: ObservedResponse, rule?: BlockRule): Promise<BlockedError | undefined>;
|
|
21
|
+
//# sourceMappingURL=block-rule.policy.d.ts.map
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** A response the recipe's `session.blockedWhen` rule (or the default one) says is the site refusing the crawl. */
|
|
2
|
+
export declare class BlockedError extends Error {
|
|
3
|
+
readonly url: string;
|
|
4
|
+
readonly status: number;
|
|
5
|
+
readonly reason: string;
|
|
6
|
+
readonly name = "BlockedError";
|
|
7
|
+
constructor(url: string, status: number, reason: string);
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=blocked.error.d.ts.map
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { ExtractionScope } from '../extraction-scope/index.js';
|
|
2
|
+
import type { ForEachStep } from '../recipe-schema/index.js';
|
|
3
|
+
import type { EmitOutcome, StepWalk } from './run-steps.use-case.js';
|
|
4
|
+
/**
|
|
5
|
+
* Runs a body once per item of a list (`over`), or once per live element
|
|
6
|
+
* matching `selector`, each in a fresh child scope with the item bound under
|
|
7
|
+
* `as`; emits a record per iteration when asked.
|
|
8
|
+
*
|
|
9
|
+
* With a concurrent gate, iterations of a list run as permits allow and
|
|
10
|
+
* records come out in completion order; without one, in list order. In web
|
|
11
|
+
* mode each parallel iteration runs in a tab of its own (`runner.fork`); a
|
|
12
|
+
* loop over live elements stays sequential, since its elements live on one page.
|
|
13
|
+
*
|
|
14
|
+
* @param step - The forEach step.
|
|
15
|
+
* @param scope - The scope the list lives in.
|
|
16
|
+
* @param walk - Runs a step list; also carries the emit callback.
|
|
17
|
+
* @returns `stop` when the crawl reached its record limit.
|
|
18
|
+
*/
|
|
19
|
+
export declare function runForEach(step: ForEachStep, scope: ExtractionScope, walk: StepWalk): Promise<EmitOutcome>;
|
|
20
|
+
//# sourceMappingURL=for-each.use-case.d.ts.map
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/** How gently one site is crawled. */
|
|
2
|
+
export interface HostRule {
|
|
3
|
+
/** Minimum time between two request starts to the site, whatever recipe sends them. */
|
|
4
|
+
delayMs?: number;
|
|
5
|
+
/** Requests to the site in flight at once. */
|
|
6
|
+
concurrency?: number;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* The crawler's politeness towards each site, across every recipe and run it
|
|
10
|
+
* executes: a default rule for any host, and rules by domain (`example.com`
|
|
11
|
+
* also covers `www.example.com`; the longest match wins).
|
|
12
|
+
*/
|
|
13
|
+
export interface ThrottleConfig extends HostRule {
|
|
14
|
+
domains?: Record<string, HostRule>;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Spaces and bounds requests per site, shared by every recipe of a crawler,
|
|
18
|
+
* so two recipes (or two parallel iterations) that hit one site add up to one
|
|
19
|
+
* polite client rather than two. A recipe's own `limits.delayMs` still applies
|
|
20
|
+
* on top, per recipe.
|
|
21
|
+
*
|
|
22
|
+
* Like a single-lane bridge with a traffic light: whoever arrives waits for
|
|
23
|
+
* the car ahead to be far enough, and for a free lane.
|
|
24
|
+
*/
|
|
25
|
+
export declare class HostThrottle {
|
|
26
|
+
private readonly config;
|
|
27
|
+
private readonly buckets;
|
|
28
|
+
private readonly domains;
|
|
29
|
+
constructor(config?: ThrottleConfig);
|
|
30
|
+
private bucketFor;
|
|
31
|
+
/** Whether any rule can hold a request back. */
|
|
32
|
+
get active(): boolean;
|
|
33
|
+
/**
|
|
34
|
+
* Waits until a request to `url` may start, then holds one of its site's
|
|
35
|
+
* lanes until the returned release is called.
|
|
36
|
+
*
|
|
37
|
+
* @param url - Where the request goes; anything but `http(s):` passes at once.
|
|
38
|
+
* @returns The release: call it once, when the response arrived or the request failed.
|
|
39
|
+
*/
|
|
40
|
+
slot(url: string): Promise<() => void>;
|
|
41
|
+
/**
|
|
42
|
+
* Holds every request to the site of `url` back until `untilMs` (a `Retry-After`).
|
|
43
|
+
*
|
|
44
|
+
* @param url - A URL of the site.
|
|
45
|
+
* @param untilMs - An epoch time.
|
|
46
|
+
*/
|
|
47
|
+
pause(url: string, untilMs: number): void;
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=host-throttle.policy.d.ts.map
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export { runSteps } from './run-steps.use-case.js';
|
|
2
|
+
export type { StepWalkOptions, EmitOutcome } from './run-steps.use-case.js';
|
|
3
|
+
export type { StepRunner, NextPageResult } from './step-runner.contract.js';
|
|
4
|
+
export { disposeQuietly } from './step-runner.contract.js';
|
|
5
|
+
export { StepFailure, NoMatchError } from './step-failure.error.js';
|
|
6
|
+
export { resolveErrorPolicy, backoffFor, sleep } from './retry.policy.js';
|
|
7
|
+
export { RunGate } from './run-gate.policy.js';
|
|
8
|
+
export { HostThrottle } from './host-throttle.policy.js';
|
|
9
|
+
export { withTransportRetry, resolveRetryRule, transientError, retryDelay, DEFAULT_RETRY_RULE, RETRY_STATUSES } from './transport-retry.policy.js';
|
|
10
|
+
export type { ResolvedRetryRule, Transient, TransportAttempt, RetryContext } from './transport-retry.policy.js';
|
|
11
|
+
export type { HostRule, ThrottleConfig } from './host-throttle.policy.js';
|
|
12
|
+
export { BlockedError } from './blocked.error.js';
|
|
13
|
+
export { detectBlock, DEFAULT_BLOCK_RULE } from './block-rule.policy.js';
|
|
14
|
+
export type { ObservedResponse } from './block-rule.policy.js';
|
|
15
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { ExtractionScope } from '../extraction-scope/index.js';
|
|
2
|
+
import type { PaginateStep } from '../recipe-schema/index.js';
|
|
3
|
+
import type { EmitOutcome, StepWalk } from './run-steps.use-case.js';
|
|
4
|
+
/**
|
|
5
|
+
* Runs a body once per page, each in a fresh child scope, then asks the runner
|
|
6
|
+
* for the next page until there is none, `until` renders truthy, or `maxPages`
|
|
7
|
+
* is reached.
|
|
8
|
+
*
|
|
9
|
+
* The runner reports the visit of each new page; this only steers.
|
|
10
|
+
*
|
|
11
|
+
* @param step - The paginate step.
|
|
12
|
+
* @param scope - The scope to page in; its page URL advances with each page.
|
|
13
|
+
* @param walk - Runs a step list; carries the runner and the emit callback.
|
|
14
|
+
* @returns `stop` when the crawl reached its record limit.
|
|
15
|
+
*/
|
|
16
|
+
export declare function runPaginate(step: PaginateStep, scope: ExtractionScope, walk: StepWalk): Promise<EmitOutcome>;
|
|
17
|
+
//# sourceMappingURL=paginate.use-case.d.ts.map
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { ErrorPolicy, InputRecipe, Step } from '../recipe-schema/index.js';
|
|
2
|
+
/**
|
|
3
|
+
* The policy for a failing step: the step's own, else the recipe's, else `fail`.
|
|
4
|
+
*
|
|
5
|
+
* @param step - The step that failed.
|
|
6
|
+
* @param recipe - Its recipe.
|
|
7
|
+
* @returns The policy to apply.
|
|
8
|
+
*/
|
|
9
|
+
export declare function resolveErrorPolicy(step: Step, recipe: Pick<InputRecipe, 'onError'>): ErrorPolicy;
|
|
10
|
+
/**
|
|
11
|
+
* How long to wait before a retry: linear backoff.
|
|
12
|
+
*
|
|
13
|
+
* @param policy - A retry policy.
|
|
14
|
+
* @param attempt - The attempt about to be made, starting at 2.
|
|
15
|
+
* @returns Milliseconds.
|
|
16
|
+
*/
|
|
17
|
+
export declare function backoffFor(policy: Extract<ErrorPolicy, {
|
|
18
|
+
policy: 'retry';
|
|
19
|
+
}>, attempt: number): number;
|
|
20
|
+
export declare function sleep(ms: number): Promise<void>;
|
|
21
|
+
//# sourceMappingURL=retry.policy.d.ts.map
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type { HostThrottle } from './host-throttle.policy.js';
|
|
2
|
+
/**
|
|
3
|
+
* What bounds a recipe run: how many `forEach` iterations may be in flight and
|
|
4
|
+
* how close together requests may start. One gate per recipe run, shared by
|
|
5
|
+
* every loop in it, so nested loops never multiply the limit.
|
|
6
|
+
*
|
|
7
|
+
* Permits go to the outermost concurrent loop: a loop that runs inside an
|
|
8
|
+
* iteration already holding a permit runs its body sequentially (see `nested`),
|
|
9
|
+
* which keeps the total at `permits` and cannot deadlock.
|
|
10
|
+
*/
|
|
11
|
+
export declare class RunGate {
|
|
12
|
+
readonly permits: number;
|
|
13
|
+
readonly minIntervalMs: number;
|
|
14
|
+
readonly hosts?: HostThrottle | undefined;
|
|
15
|
+
private readonly shared?;
|
|
16
|
+
private inFlight;
|
|
17
|
+
private readonly waiting;
|
|
18
|
+
private lastStart;
|
|
19
|
+
/**
|
|
20
|
+
* @param permits - Iterations allowed in flight; 1 is sequential.
|
|
21
|
+
* @param minIntervalMs - Minimum time between two request starts across the run.
|
|
22
|
+
* @param hosts - The crawler's per-site throttle, shared with every other recipe.
|
|
23
|
+
* @param shared - The throttle state to share (internal: `nested` gates keep their parent's).
|
|
24
|
+
*/
|
|
25
|
+
constructor(permits: number, minIntervalMs: number, hosts?: HostThrottle | undefined, shared?: RunGate | undefined);
|
|
26
|
+
/** Whether this gate lets more than one iteration run at once. */
|
|
27
|
+
get concurrent(): boolean;
|
|
28
|
+
/**
|
|
29
|
+
* Takes a permit, waiting for one when all are in flight.
|
|
30
|
+
*
|
|
31
|
+
* @returns The release; call it exactly once, when the iteration ends.
|
|
32
|
+
*/
|
|
33
|
+
acquire(): Promise<() => void>;
|
|
34
|
+
/**
|
|
35
|
+
* Waits until a request may start: `minIntervalMs` after the previous start,
|
|
36
|
+
* whichever loop started it. Returns at once when the interval has passed.
|
|
37
|
+
*/
|
|
38
|
+
throttle(): Promise<void>;
|
|
39
|
+
/**
|
|
40
|
+
* Waits until a request to `url` may start: the recipe's interval, then its
|
|
41
|
+
* site's turn in the crawler's per-site throttle.
|
|
42
|
+
*
|
|
43
|
+
* @param url - Where the request goes.
|
|
44
|
+
* @returns The release of the site's lane: call it once the response arrived or the request failed.
|
|
45
|
+
*/
|
|
46
|
+
request(url: string): Promise<() => void>;
|
|
47
|
+
/** The gate for a body running inside an iteration that holds a permit: sequential, same throttle. */
|
|
48
|
+
nested(): RunGate;
|
|
49
|
+
}
|
|
50
|
+
//# sourceMappingURL=run-gate.policy.d.ts.map
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { EventBus } from '../crawl-events/index.js';
|
|
2
|
+
import type { ExtractionScope } from '../extraction-scope/index.js';
|
|
3
|
+
import type { HookRegistry } from '../hooks/index.js';
|
|
4
|
+
import type { InputRecipe, Step } from '../recipe-schema/index.js';
|
|
5
|
+
import type { RunGate } from './run-gate.policy.js';
|
|
6
|
+
import type { StepRunner } from './step-runner.contract.js';
|
|
7
|
+
/** Whether the walk goes on after a record was emitted. */
|
|
8
|
+
export type EmitOutcome = 'continue' | 'stop';
|
|
9
|
+
/** Everything a step walk needs. */
|
|
10
|
+
export interface StepWalkOptions {
|
|
11
|
+
recipe: InputRecipe;
|
|
12
|
+
runner: StepRunner;
|
|
13
|
+
hooks: HookRegistry;
|
|
14
|
+
events: EventBus;
|
|
15
|
+
/** Called with the scope to snapshot for each record; `stop` ends the walk. */
|
|
16
|
+
onEmit: (scope: ExtractionScope, output?: string) => Promise<EmitOutcome>;
|
|
17
|
+
/** Bounds concurrency and request rate; absent means sequential and unthrottled. */
|
|
18
|
+
gate?: RunGate;
|
|
19
|
+
}
|
|
20
|
+
/** The walk as the control-flow steps see it: options plus the current path. */
|
|
21
|
+
export interface StepWalk extends StepWalkOptions {
|
|
22
|
+
path: string;
|
|
23
|
+
/** Runs a nested step list; `overrides` replace walk options for that list (a sequential gate inside a concurrent iteration). */
|
|
24
|
+
runSteps: (steps: readonly Step[], scope: ExtractionScope, path: string, overrides?: Partial<StepWalkOptions>) => Promise<EmitOutcome>;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Walks a step list in order. Control flow (`forEach`, `if`, `paginate`, `emit`,
|
|
28
|
+
* `set`, `hook`, `when`, error policies) is handled here; leaf steps go to the
|
|
29
|
+
* runner. Mode-agnostic: the same walk drives a browser page or an HTTP context.
|
|
30
|
+
*
|
|
31
|
+
* @param steps - The steps.
|
|
32
|
+
* @param scope - The scope to run in.
|
|
33
|
+
* @param options - Recipe, runner, hooks, events and the emit callback.
|
|
34
|
+
* @param path - Where these steps are, for messages and events.
|
|
35
|
+
* @returns `stop` when the emit callback asked to stop.
|
|
36
|
+
* @throws StepFailure when a step fails under the `fail` policy.
|
|
37
|
+
*/
|
|
38
|
+
export declare function runSteps(steps: readonly Step[], scope: ExtractionScope, options: StepWalkOptions, path?: string): Promise<EmitOutcome>;
|
|
39
|
+
//# sourceMappingURL=run-steps.use-case.d.ts.map
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/** A step that failed under the `fail` policy: the recipe stops here. */
|
|
2
|
+
export declare class StepFailure extends Error {
|
|
3
|
+
readonly stepPath: string;
|
|
4
|
+
readonly stepType: string;
|
|
5
|
+
readonly name = "StepFailure";
|
|
6
|
+
constructor(stepPath: string, stepType: string, cause: unknown);
|
|
7
|
+
}
|
|
8
|
+
/** A single `extract` that matched nothing. */
|
|
9
|
+
export declare class NoMatchError extends Error {
|
|
10
|
+
readonly selector: string;
|
|
11
|
+
readonly name = "NoMatchError";
|
|
12
|
+
constructor(selector: string);
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=step-failure.error.d.ts.map
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type { ExtractionScope, LiveElement } from '../extraction-scope/index.js';
|
|
2
|
+
import type { PaginateNext, Step } from '../recipe-schema/index.js';
|
|
3
|
+
import type { BlockedError } from './blocked.error.js';
|
|
4
|
+
/**
|
|
5
|
+
* Disposes a runner, ignoring a failure: a tab whose browser already went
|
|
6
|
+
* away (a rotation, a crash) has nothing left to close.
|
|
7
|
+
*
|
|
8
|
+
* @param runner - The runner.
|
|
9
|
+
*/
|
|
10
|
+
export declare function disposeQuietly(runner: StepRunner): Promise<void>;
|
|
11
|
+
/** What `paginate` learns from the runner after a page body ran. */
|
|
12
|
+
export type NextPageResult =
|
|
13
|
+
/** The next page is at this URL (the runner already navigated in web mode). */
|
|
14
|
+
{
|
|
15
|
+
kind: 'url';
|
|
16
|
+
url: string;
|
|
17
|
+
} |
|
|
18
|
+
/** A value to bind in the next page's scope under `name` (a cursor, a token). */
|
|
19
|
+
{
|
|
20
|
+
kind: 'value';
|
|
21
|
+
name: string;
|
|
22
|
+
value: unknown;
|
|
23
|
+
} |
|
|
24
|
+
/** There is no next page. */
|
|
25
|
+
null;
|
|
26
|
+
/**
|
|
27
|
+
* The mode-specific half of step execution. `step-flow` walks the tree and
|
|
28
|
+
* owns control flow; a runner executes leaf steps against a page or a request
|
|
29
|
+
* context. Runners never mutate scope bindings other than through this contract.
|
|
30
|
+
*/
|
|
31
|
+
export interface StepRunner {
|
|
32
|
+
/** Runs one leaf step (not forEach, paginate, emit, set or hook). */
|
|
33
|
+
runLeaf: (step: Step, scope: ExtractionScope) => Promise<void>;
|
|
34
|
+
/** Finds (and in web mode reaches) the next page. */
|
|
35
|
+
nextPage: (next: PaginateNext, scope: ExtractionScope) => Promise<NextPageResult>;
|
|
36
|
+
/** Snapshots every element matching a rendered selector, for `forEach` over `selector`. Web mode only. */
|
|
37
|
+
elements?: (selector: string, scope: ExtractionScope) => Promise<LiveElement[]>;
|
|
38
|
+
/**
|
|
39
|
+
* Called when a step is blocked. `true` means the runner now reaches the
|
|
40
|
+
* network another way (a new access lease) and the step should run again;
|
|
41
|
+
* `false` means it cannot, and the block fails the step like any error.
|
|
42
|
+
*/
|
|
43
|
+
rotate?: (error: BlockedError) => Promise<boolean>;
|
|
44
|
+
/**
|
|
45
|
+
* A runner of its own for one parallel `forEach` iteration: a new tab in the
|
|
46
|
+
* same browser context (same cookies, its own page), disposed when the
|
|
47
|
+
* iteration ends. Web mode; an api runner is shared as it is.
|
|
48
|
+
*/
|
|
49
|
+
fork?: () => Promise<StepRunner>;
|
|
50
|
+
dispose: () => Promise<void>;
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=step-runner.contract.d.ts.map
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import type { EventBus } from '../crawl-events/index.js';
|
|
2
|
+
import type { RetryRule } from '../recipe-schema/index.js';
|
|
3
|
+
import type { RunGate } from './run-gate.policy.js';
|
|
4
|
+
/** A retry rule with every default filled in. */
|
|
5
|
+
export type ResolvedRetryRule = Required<RetryRule>;
|
|
6
|
+
/** Statuses a server uses for "not now": timeout, too early, too many requests, and the 5xx that pass. */
|
|
7
|
+
export declare const RETRY_STATUSES: readonly number[];
|
|
8
|
+
/** Three tries, one second then two apart, never a wait over 30 seconds. */
|
|
9
|
+
export declare const DEFAULT_RETRY_RULE: ResolvedRetryRule;
|
|
10
|
+
/** Why an attempt may be worth repeating, and the server's own `Retry-After`, if it sent one. */
|
|
11
|
+
export interface Transient {
|
|
12
|
+
reason: string;
|
|
13
|
+
retryAfter?: string;
|
|
14
|
+
}
|
|
15
|
+
/** One try of a request, and how to tell a passing failure in what it gave. */
|
|
16
|
+
export interface TransportAttempt<T> {
|
|
17
|
+
run: () => Promise<T>;
|
|
18
|
+
/** A transient problem in the outcome (a retry status, a connection error), or `undefined` when the outcome stands. */
|
|
19
|
+
problem: (outcome: {
|
|
20
|
+
value: T;
|
|
21
|
+
} | {
|
|
22
|
+
error: unknown;
|
|
23
|
+
}) => Transient | undefined;
|
|
24
|
+
}
|
|
25
|
+
export interface RetryContext {
|
|
26
|
+
recipeId: string;
|
|
27
|
+
gate: RunGate;
|
|
28
|
+
events: EventBus;
|
|
29
|
+
rule: ResolvedRetryRule;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* The retry rule a recipe runs with: its `limits.retry` over the crawler's
|
|
33
|
+
* default over `DEFAULT_RETRY_RULE`.
|
|
34
|
+
*
|
|
35
|
+
* @param own - The recipe's `limits.retry`.
|
|
36
|
+
* @param crawler - The crawler's `retry` option.
|
|
37
|
+
* @returns The rule.
|
|
38
|
+
*/
|
|
39
|
+
export declare function resolveRetryRule(own?: RetryRule, crawler?: RetryRule): ResolvedRetryRule;
|
|
40
|
+
/**
|
|
41
|
+
* Whether an error is a connection failure worth another try.
|
|
42
|
+
*
|
|
43
|
+
* @param error - What the request threw.
|
|
44
|
+
* @returns The reason, or `undefined`.
|
|
45
|
+
*/
|
|
46
|
+
export declare function transientError(error: unknown): Transient | undefined;
|
|
47
|
+
/**
|
|
48
|
+
* How long to wait before attempt `attempt + 1`: the server's `Retry-After`
|
|
49
|
+
* when it gave one, else `backoffMs` doubling per attempt with a little
|
|
50
|
+
* jitter; `undefined` when the server asks for longer than `maxDelayMs` (it
|
|
51
|
+
* means "come back much later", which a crawl cannot wait for).
|
|
52
|
+
*
|
|
53
|
+
* @param rule - The retry rule.
|
|
54
|
+
* @param attempt - The attempt that just failed, from 1.
|
|
55
|
+
* @param retryAfter - The `Retry-After` header: seconds, or an HTTP date.
|
|
56
|
+
* @param now - The current time, for dates.
|
|
57
|
+
* @returns Milliseconds, or `undefined` for no retry.
|
|
58
|
+
*/
|
|
59
|
+
export declare function retryDelay(rule: ResolvedRetryRule, attempt: number, retryAfter?: string, now?: number): number | undefined;
|
|
60
|
+
/**
|
|
61
|
+
* Sends a request through the gate (the recipe's rate, the site's lane), and
|
|
62
|
+
* sends it again after a pause while it fails in a passing way, up to
|
|
63
|
+
* `rule.attempts` tries in all. A `Retry-After` holds back every request to
|
|
64
|
+
* that site, not only this one. Each retry is reported as `request:retry`.
|
|
65
|
+
*
|
|
66
|
+
* Like redialling a busy number: wait a moment, dial again, give up after a
|
|
67
|
+
* few tries; and if the other end said "call back in a minute", wait that minute.
|
|
68
|
+
*
|
|
69
|
+
* @param url - Where the request goes.
|
|
70
|
+
* @param attempt - How to send it and how to judge the outcome.
|
|
71
|
+
* @param context - The recipe, gate, events and rule.
|
|
72
|
+
* @returns What the last try gave.
|
|
73
|
+
* @throws What the last try threw.
|
|
74
|
+
*/
|
|
75
|
+
export declare function withTransportRetry<T>(url: string, attempt: TransportAttempt<T>, context: RetryContext): Promise<T>;
|
|
76
|
+
//# sourceMappingURL=transport-retry.policy.d.ts.map
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A small expression language for `{{ }}` placeholders: literals, paths,
|
|
3
|
+
* arithmetic, comparison, logic, `??`, `a ? b : c` and a fixed set of
|
|
4
|
+
* functions. It is parsed into a tree and walked; nothing is ever compiled or
|
|
5
|
+
* evaluated as JavaScript, and a path can only read own properties of plain
|
|
6
|
+
* data, so `constructor`, `__proto__` and friends resolve to nothing.
|
|
7
|
+
*/
|
|
8
|
+
import type { Lookup } from './value-text.algorithm.js';
|
|
9
|
+
export type Expression = {
|
|
10
|
+
kind: 'literal';
|
|
11
|
+
value: unknown;
|
|
12
|
+
} | {
|
|
13
|
+
kind: 'path';
|
|
14
|
+
path: string;
|
|
15
|
+
} | {
|
|
16
|
+
kind: 'unary';
|
|
17
|
+
op: '-' | '!';
|
|
18
|
+
operand: Expression;
|
|
19
|
+
} | {
|
|
20
|
+
kind: 'binary';
|
|
21
|
+
op: string;
|
|
22
|
+
left: Expression;
|
|
23
|
+
right: Expression;
|
|
24
|
+
} | {
|
|
25
|
+
kind: 'ternary';
|
|
26
|
+
test: Expression;
|
|
27
|
+
consequent: Expression;
|
|
28
|
+
alternate: Expression;
|
|
29
|
+
} | {
|
|
30
|
+
kind: 'call';
|
|
31
|
+
name: string;
|
|
32
|
+
args: Expression[];
|
|
33
|
+
};
|
|
34
|
+
/** The names an expression may call. */
|
|
35
|
+
export declare const EXPRESSION_FUNCTIONS: readonly string[];
|
|
36
|
+
/**
|
|
37
|
+
* Parses an expression, with a small cache since templates render per iteration.
|
|
38
|
+
*
|
|
39
|
+
* @param source - The text inside the placeholder.
|
|
40
|
+
* @returns The tree.
|
|
41
|
+
* @throws Error naming the position of the first problem.
|
|
42
|
+
*/
|
|
43
|
+
export declare function parseExpression(source: string): Expression;
|
|
44
|
+
/**
|
|
45
|
+
* Evaluates a tree against a lookup. A function found where data is expected
|
|
46
|
+
* counts as `undefined`, so nothing callable ever leaks out of the scope.
|
|
47
|
+
*
|
|
48
|
+
* @param expression - The tree.
|
|
49
|
+
* @param lookup - Resolves a path.
|
|
50
|
+
* @returns The value.
|
|
51
|
+
*/
|
|
52
|
+
export declare function evaluateExpression(expression: Expression, lookup: Lookup): unknown;
|
|
53
|
+
//# sourceMappingURL=expression.algorithm.d.ts.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export { getPath, setPath, segmentsOf } from './value-path.algorithm.js';
|
|
2
|
+
export { render, renderDeep, renderText, resolve, hasPlaceholder } from './template.algorithm.js';
|
|
3
|
+
export { isTruthy, stringify } from './value-text.algorithm.js';
|
|
4
|
+
export { parseExpression, evaluateExpression, EXPRESSION_FUNCTIONS } from './expression.algorithm.js';
|
|
5
|
+
export type { Expression } from './expression.algorithm.js';
|
|
6
|
+
export type { Lookup } from './value-text.algorithm.js';
|
|
7
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `{{ }}` interpolation for recipe strings. A placeholder is a dotted path
|
|
3
|
+
* resolved through the lookup the caller provides, or an expression over such
|
|
4
|
+
* paths (see `expression.algorithm`). No code is ever executed.
|
|
5
|
+
*/
|
|
6
|
+
import type { Lookup } from './value-text.algorithm.js';
|
|
7
|
+
/**
|
|
8
|
+
* Whether a string contains at least one placeholder.
|
|
9
|
+
*
|
|
10
|
+
* @param text - The string.
|
|
11
|
+
* @returns `true` when it has a `{{ }}`.
|
|
12
|
+
*/
|
|
13
|
+
export declare function hasPlaceholder(text: string): boolean;
|
|
14
|
+
/**
|
|
15
|
+
* Renders a template. When the template is exactly one placeholder the resolved
|
|
16
|
+
* value is returned as is (so a list or a number survives); otherwise every
|
|
17
|
+
* placeholder is stringified into the text, `undefined`/`null` as `''`.
|
|
18
|
+
*
|
|
19
|
+
* @param template - The string to render.
|
|
20
|
+
* @param lookup - Resolves a path.
|
|
21
|
+
* @returns The rendered value.
|
|
22
|
+
*/
|
|
23
|
+
export declare function render(template: string, lookup: Lookup): unknown;
|
|
24
|
+
/**
|
|
25
|
+
* The value of one placeholder: a plain path is looked up as is; anything else
|
|
26
|
+
* is parsed and evaluated as an expression.
|
|
27
|
+
*
|
|
28
|
+
* @param inner - The trimmed text between the braces.
|
|
29
|
+
* @param lookup - Resolves a path.
|
|
30
|
+
* @returns The value.
|
|
31
|
+
* @throws Error when an expression does not parse.
|
|
32
|
+
*/
|
|
33
|
+
export declare function resolve(inner: string, lookup: Lookup): unknown;
|
|
34
|
+
/**
|
|
35
|
+
* Renders every string inside a JSON-shaped value (an object, an array, or a
|
|
36
|
+
* string) with {@link render}, so a lone placeholder keeps its type
|
|
37
|
+
* (`"{{vars.limit}}"` becomes the number) and anything else becomes text.
|
|
38
|
+
* Numbers, booleans and `null` pass through untouched.
|
|
39
|
+
*
|
|
40
|
+
* @param value - The value.
|
|
41
|
+
* @param lookup - Resolves a path.
|
|
42
|
+
* @returns A new value with every string rendered.
|
|
43
|
+
*/
|
|
44
|
+
export declare function renderDeep(value: unknown, lookup: Lookup): unknown;
|
|
45
|
+
/**
|
|
46
|
+
* Renders a template that must produce text.
|
|
47
|
+
*
|
|
48
|
+
* @param template - The string to render.
|
|
49
|
+
* @param lookup - Resolves a path.
|
|
50
|
+
* @returns The rendered text; a non-string single value is stringified.
|
|
51
|
+
*/
|
|
52
|
+
export declare function renderText(template: string, lookup: Lookup): string;
|
|
53
|
+
//# sourceMappingURL=template.algorithm.d.ts.map
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reads and writes values by dotted path (`seller.name`, `items.0.url`, `items[0].url`).
|
|
3
|
+
* Pure: the read never throws on a missing segment, the write creates what is missing.
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Splits a path into its segments; `a.b[0].c` and `a.b.0.c` are the same path.
|
|
7
|
+
*
|
|
8
|
+
* @param path - The dotted path.
|
|
9
|
+
* @returns Its segments, `[]` for an empty path or `.`.
|
|
10
|
+
*/
|
|
11
|
+
export declare function segmentsOf(path: string): string[];
|
|
12
|
+
/**
|
|
13
|
+
* Reads a value by path. Only own properties are read, so `constructor`,
|
|
14
|
+
* `__proto__`, `toString` and the like resolve to `undefined` whatever the data.
|
|
15
|
+
*
|
|
16
|
+
* @param source - Where to read from.
|
|
17
|
+
* @param path - The dotted path; `.` or `''` returns the source itself.
|
|
18
|
+
* @returns The value, or `undefined` when any segment is missing.
|
|
19
|
+
*/
|
|
20
|
+
export declare function getPath(source: unknown, path: string): unknown;
|
|
21
|
+
/**
|
|
22
|
+
* Writes a value by path, creating intermediate objects (or arrays for numeric segments).
|
|
23
|
+
*
|
|
24
|
+
* @param target - The object to write into. Mutated.
|
|
25
|
+
* @param path - The dotted path.
|
|
26
|
+
* @param value - The value to store.
|
|
27
|
+
*/
|
|
28
|
+
export declare function setPath(target: Record<string, unknown>, path: string, value: unknown): void;
|
|
29
|
+
//# sourceMappingURL=value-path.algorithm.d.ts.map
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/** How recipes read a value: as text for interpolation, as a truth for conditions. Shared by templates and expressions. */
|
|
2
|
+
/** Resolves a dotted path to a value; `undefined` when unknown. */
|
|
3
|
+
export type Lookup = (path: string) => unknown;
|
|
4
|
+
/**
|
|
5
|
+
* Truthiness as recipes mean it: `when` and `until` conditions.
|
|
6
|
+
*
|
|
7
|
+
* @param value - Any rendered value.
|
|
8
|
+
* @returns `false` for `undefined`, `null`, `''`, `0`, `false`, `'false'`, `'0'`, `'null'` and an empty list.
|
|
9
|
+
*/
|
|
10
|
+
export declare function isTruthy(value: unknown): boolean;
|
|
11
|
+
/**
|
|
12
|
+
* The text form of a value for interpolation.
|
|
13
|
+
*
|
|
14
|
+
* @param value - Any value.
|
|
15
|
+
* @returns A string; objects and arrays as JSON.
|
|
16
|
+
*/
|
|
17
|
+
export declare function stringify(value: unknown): string;
|
|
18
|
+
//# sourceMappingURL=value-text.algorithm.d.ts.map
|