@opencraw/core 0.1.1 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/README.md +4 -2
  2. package/dist/index.esm.js +3010 -638
  3. package/dist/src/access/access-profile.contract.d.ts +4 -0
  4. package/dist/src/access/index.d.ts +1 -1
  5. package/dist/src/api-steps/extract-from-document.use-case.d.ts +16 -3
  6. package/dist/src/api-steps/index.d.ts +1 -1
  7. package/dist/src/api-steps/send-request.use-case.d.ts +5 -4
  8. package/dist/src/browser-session/browser-profile.store.d.ts +52 -0
  9. package/dist/src/browser-session/browser.client.d.ts +8 -0
  10. package/dist/src/browser-session/index.d.ts +1 -0
  11. package/dist/src/captcha/captcha-budget.model.d.ts +21 -0
  12. package/dist/src/captcha/captcha-detection.client.d.ts +28 -0
  13. package/dist/src/captcha/captcha-guard.use-case.d.ts +64 -0
  14. package/dist/src/captcha/captcha-solver-registry.store.d.ts +19 -0
  15. package/dist/src/captcha/captcha-solver.contract.d.ts +47 -0
  16. package/dist/src/captcha/captcha.error.d.ts +13 -0
  17. package/dist/src/captcha/index.d.ts +10 -0
  18. package/dist/src/captcha/resolve-captcha.use-case.d.ts +40 -0
  19. package/dist/src/crawl-events/crawl-event.contract.d.ts +48 -0
  20. package/dist/src/crawl-execution/bootstrap-session.use-case.d.ts +27 -3
  21. package/dist/src/crawl-execution/crawl-options.config.d.ts +30 -0
  22. package/dist/src/crawl-execution/crawl-report.model.d.ts +6 -0
  23. package/dist/src/crawl-execution/create-crawler.use-case.d.ts +2 -1
  24. package/dist/src/crawl-execution/rotating-runner.use-case.d.ts +9 -0
  25. package/dist/src/crawl-execution/run-crawl.use-case.d.ts +6 -2
  26. package/dist/src/crawl-execution/run-input-recipe.use-case.d.ts +14 -3
  27. package/dist/src/deck-document/deck-document.model.d.ts +58 -0
  28. package/dist/src/deck-document/deck-table.algorithm.d.ts +35 -0
  29. package/dist/src/deck-document/index.d.ts +6 -0
  30. package/dist/src/deck-document/read-pptx.client.d.ts +16 -0
  31. package/dist/src/extraction-scope/extraction-scope.model.d.ts +3 -1
  32. package/dist/src/http-session/http-response.contract.d.ts +13 -1
  33. package/dist/src/http-session/text-decoding.algorithm.d.ts +35 -0
  34. package/dist/src/index.d.ts +13 -4
  35. package/dist/src/markdown-document/index.d.ts +3 -0
  36. package/dist/src/markdown-document/read-markdown.client.d.ts +29 -0
  37. package/dist/src/pdf-document/index.d.ts +1 -1
  38. package/dist/src/pdf-document/row-assembly.algorithm.d.ts +10 -1
  39. package/dist/src/recipe-loading/recipe-binding.validator.d.ts +3 -1
  40. package/dist/src/recipe-schema/index.d.ts +3 -3
  41. package/dist/src/recipe-schema/input-recipe.contract.d.ts +57 -4
  42. package/dist/src/recipe-schema/recipe-kind.enum.d.ts +3 -2
  43. package/dist/src/recipe-schema/step.contract.d.ts +46 -2
  44. package/dist/src/record-sink/dedupe.policy.d.ts +19 -7
  45. package/dist/src/record-sink/index.d.ts +1 -0
  46. package/dist/src/selection/index.d.ts +1 -1
  47. package/dist/src/selection/json-text.algorithm.d.ts +30 -3
  48. package/dist/src/step-flow/for-each.use-case.d.ts +4 -2
  49. package/dist/src/step-flow/host-throttle.policy.d.ts +49 -0
  50. package/dist/src/step-flow/index.d.ts +5 -0
  51. package/dist/src/step-flow/run-gate.policy.d.ts +12 -1
  52. package/dist/src/step-flow/step-runner.contract.d.ts +13 -0
  53. package/dist/src/step-flow/transport-retry.policy.d.ts +76 -0
  54. package/dist/src/web-steps/navigate.use-case.d.ts +3 -2
  55. package/dist/src/web-steps/run-web-step.use-case.d.ts +18 -3
  56. package/dist/src/workbook-document/csv-parser.algorithm.d.ts +26 -0
  57. package/dist/src/workbook-document/csv-workbook.mapper.d.ts +24 -0
  58. package/dist/src/workbook-document/grid-table.algorithm.d.ts +53 -0
  59. package/dist/src/workbook-document/html-tables.mapper.d.ts +14 -0
  60. package/dist/src/workbook-document/index.d.ts +9 -0
  61. package/dist/src/workbook-document/read-xlsx.client.d.ts +18 -0
  62. package/dist/src/workbook-document/workbook-document.model.d.ts +51 -0
  63. package/dist/src/yaml-document/index.d.ts +3 -0
  64. package/dist/src/yaml-document/read-yaml.client.d.ts +25 -0
  65. package/package.json +16 -2
@@ -4,5 +4,5 @@ export { selectHtml } from './html-selector.algorithm.js';
4
4
  export type { HtmlMatch } from './html-selector.algorithm.js';
5
5
  export { takeFromHtml, takeFromJson, collapse } from './take-value.mapper.js';
6
6
  export type { Take } from './take-value.mapper.js';
7
- export { tryParseJson, parseJsonText, dataItemsOf } from './json-text.algorithm.js';
7
+ export { tryParseJson, parseJsonText, dataItemsOf, parseJsonLike, parseJsonLines } from './json-text.algorithm.js';
8
8
  //# sourceMappingURL=index.d.ts.map
@@ -1,10 +1,37 @@
1
1
  /**
2
2
  * JSON that arrives as text: a `<script type="application/ld+json">` body, a
3
- * `data-*` attribute, a fetched document read as text. Sites wrap inline JSON
4
- * in comment guards, which are stripped before parsing.
3
+ * `data-*` attribute, a fetched document read as text. Sites wrap JSON in
4
+ * things that are not JSON: comment guards around inline JSON-LD, prefixes
5
+ * that stop a page from loading an API as a script, a JSONP callback, an
6
+ * assignment in an inline script. Those wrappers are removed, but only after
7
+ * the text failed to parse as it is, and what is left must still be strict
8
+ * JSON: nothing is evaluated.
5
9
  */
6
10
  /**
7
- * Parses text as JSON, guards stripped.
11
+ * Parses text as JSON, or as JSON inside one of the wrappers sites put around
12
+ * it: comment guards, an anti-hijacking prefix, a JSONP call, an assignment.
13
+ * Valid JSON is always read as it is; a wrapper is only removed when that
14
+ * fails.
15
+ *
16
+ * @param text - The text.
17
+ * @returns The value, or the error the text as it is gave.
18
+ */
19
+ export declare function parseJsonLike(text: string): {
20
+ value: unknown;
21
+ } | {
22
+ error: Error;
23
+ };
24
+ /**
25
+ * Parses JSON Lines (NDJSON): one JSON value per non-blank line.
26
+ *
27
+ * @param text - The text.
28
+ * @param source - Where it came from, for the error.
29
+ * @returns The values, in order.
30
+ * @throws Error naming the source and the line that does not parse.
31
+ */
32
+ export declare function parseJsonLines(text: string, source: string): unknown[];
33
+ /**
34
+ * Parses text as JSON, wrappers removed (see {@link parseJsonLike}).
8
35
  *
9
36
  * @param text - The text.
10
37
  * @returns The value, or `undefined` when it is not JSON.
@@ -6,8 +6,10 @@ import type { EmitOutcome, StepWalk } from './run-steps.use-case.js';
6
6
  * matching `selector`, each in a fresh child scope with the item bound under
7
7
  * `as`; emits a record per iteration when asked.
8
8
  *
9
- * With a concurrent gate, iterations run as permits allow and records come
10
- * out in completion order; without one, in list order.
9
+ * With a concurrent gate, iterations of a list run as permits allow and
10
+ * records come out in completion order; without one, in list order. In web
11
+ * mode each parallel iteration runs in a tab of its own (`runner.fork`); a
12
+ * loop over live elements stays sequential, since its elements live on one page.
11
13
  *
12
14
  * @param step - The forEach step.
13
15
  * @param scope - The scope the list lives in.
@@ -0,0 +1,49 @@
1
+ /** How gently one site is crawled. */
2
+ export interface HostRule {
3
+ /** Minimum time between two request starts to the site, whatever recipe sends them. */
4
+ delayMs?: number;
5
+ /** Requests to the site in flight at once. */
6
+ concurrency?: number;
7
+ }
8
+ /**
9
+ * The crawler's politeness towards each site, across every recipe and run it
10
+ * executes: a default rule for any host, and rules by domain (`example.com`
11
+ * also covers `www.example.com`; the longest match wins).
12
+ */
13
+ export interface ThrottleConfig extends HostRule {
14
+ domains?: Record<string, HostRule>;
15
+ }
16
+ /**
17
+ * Spaces and bounds requests per site, shared by every recipe of a crawler,
18
+ * so two recipes (or two parallel iterations) that hit one site add up to one
19
+ * polite client rather than two. A recipe's own `limits.delayMs` still applies
20
+ * on top, per recipe.
21
+ *
22
+ * Like a single-lane bridge with a traffic light: whoever arrives waits for
23
+ * the car ahead to be far enough, and for a free lane.
24
+ */
25
+ export declare class HostThrottle {
26
+ private readonly config;
27
+ private readonly buckets;
28
+ private readonly domains;
29
+ constructor(config?: ThrottleConfig);
30
+ private bucketFor;
31
+ /** Whether any rule can hold a request back. */
32
+ get active(): boolean;
33
+ /**
34
+ * Waits until a request to `url` may start, then holds one of its site's
35
+ * lanes until the returned release is called.
36
+ *
37
+ * @param url - Where the request goes; anything but `http(s):` passes at once.
38
+ * @returns The release: call it once, when the response arrived or the request failed.
39
+ */
40
+ slot(url: string): Promise<() => void>;
41
+ /**
42
+ * Holds every request to the site of `url` back until `untilMs` (a `Retry-After`).
43
+ *
44
+ * @param url - A URL of the site.
45
+ * @param untilMs - An epoch time.
46
+ */
47
+ pause(url: string, untilMs: number): void;
48
+ }
49
+ //# sourceMappingURL=host-throttle.policy.d.ts.map
@@ -1,9 +1,14 @@
1
1
  export { runSteps } from './run-steps.use-case.js';
2
2
  export type { StepWalkOptions, EmitOutcome } from './run-steps.use-case.js';
3
3
  export type { StepRunner, NextPageResult } from './step-runner.contract.js';
4
+ export { disposeQuietly } from './step-runner.contract.js';
4
5
  export { StepFailure, NoMatchError } from './step-failure.error.js';
5
6
  export { resolveErrorPolicy, backoffFor, sleep } from './retry.policy.js';
6
7
  export { RunGate } from './run-gate.policy.js';
8
+ export { HostThrottle } from './host-throttle.policy.js';
9
+ export { withTransportRetry, resolveRetryRule, transientError, retryDelay, DEFAULT_RETRY_RULE, RETRY_STATUSES } from './transport-retry.policy.js';
10
+ export type { ResolvedRetryRule, Transient, TransportAttempt, RetryContext } from './transport-retry.policy.js';
11
+ export type { HostRule, ThrottleConfig } from './host-throttle.policy.js';
7
12
  export { BlockedError } from './blocked.error.js';
8
13
  export { detectBlock, DEFAULT_BLOCK_RULE } from './block-rule.policy.js';
9
14
  export type { ObservedResponse } from './block-rule.policy.js';
@@ -1,3 +1,4 @@
1
+ import type { HostThrottle } from './host-throttle.policy.js';
1
2
  /**
2
3
  * What bounds a recipe run: how many `forEach` iterations may be in flight and
3
4
  * how close together requests may start. One gate per recipe run, shared by
@@ -10,6 +11,7 @@
10
11
  export declare class RunGate {
11
12
  readonly permits: number;
12
13
  readonly minIntervalMs: number;
14
+ readonly hosts?: HostThrottle | undefined;
13
15
  private readonly shared?;
14
16
  private inFlight;
15
17
  private readonly waiting;
@@ -17,9 +19,10 @@ export declare class RunGate {
17
19
  /**
18
20
  * @param permits - Iterations allowed in flight; 1 is sequential.
19
21
  * @param minIntervalMs - Minimum time between two request starts across the run.
22
+ * @param hosts - The crawler's per-site throttle, shared with every other recipe.
20
23
  * @param shared - The throttle state to share (internal: `nested` gates keep their parent's).
21
24
  */
22
- constructor(permits: number, minIntervalMs: number, shared?: RunGate | undefined);
25
+ constructor(permits: number, minIntervalMs: number, hosts?: HostThrottle | undefined, shared?: RunGate | undefined);
23
26
  /** Whether this gate lets more than one iteration run at once. */
24
27
  get concurrent(): boolean;
25
28
  /**
@@ -33,6 +36,14 @@ export declare class RunGate {
33
36
  * whichever loop started it. Returns at once when the interval has passed.
34
37
  */
35
38
  throttle(): Promise<void>;
39
+ /**
40
+ * Waits until a request to `url` may start: the recipe's interval, then its
41
+ * site's turn in the crawler's per-site throttle.
42
+ *
43
+ * @param url - Where the request goes.
44
+ * @returns The release of the site's lane: call it once the response arrived or the request failed.
45
+ */
46
+ request(url: string): Promise<() => void>;
36
47
  /** The gate for a body running inside an iteration that holds a permit: sequential, same throttle. */
37
48
  nested(): RunGate;
38
49
  }
@@ -1,6 +1,13 @@
1
1
  import type { ExtractionScope, LiveElement } from '../extraction-scope/index.js';
2
2
  import type { PaginateNext, Step } from '../recipe-schema/index.js';
3
3
  import type { BlockedError } from './blocked.error.js';
4
+ /**
5
+ * Disposes a runner, ignoring a failure: a tab whose browser already went
6
+ * away (a rotation, a crash) has nothing left to close.
7
+ *
8
+ * @param runner - The runner.
9
+ */
10
+ export declare function disposeQuietly(runner: StepRunner): Promise<void>;
4
11
  /** What `paginate` learns from the runner after a page body ran. */
5
12
  export type NextPageResult =
6
13
  /** The next page is at this URL (the runner already navigated in web mode). */
@@ -34,6 +41,12 @@ export interface StepRunner {
34
41
  * `false` means it cannot, and the block fails the step like any error.
35
42
  */
36
43
  rotate?: (error: BlockedError) => Promise<boolean>;
44
+ /**
45
+ * A runner of its own for one parallel `forEach` iteration: a new tab in the
46
+ * same browser context (same cookies, its own page), disposed when the
47
+ * iteration ends. Web mode; an api runner is shared as it is.
48
+ */
49
+ fork?: () => Promise<StepRunner>;
37
50
  dispose: () => Promise<void>;
38
51
  }
39
52
  //# sourceMappingURL=step-runner.contract.d.ts.map
@@ -0,0 +1,76 @@
1
+ import type { EventBus } from '../crawl-events/index.js';
2
+ import type { RetryRule } from '../recipe-schema/index.js';
3
+ import type { RunGate } from './run-gate.policy.js';
4
+ /** A retry rule with every default filled in. */
5
+ export type ResolvedRetryRule = Required<RetryRule>;
6
+ /** Statuses a server uses for "not now": timeout, too early, too many requests, and the 5xx that pass. */
7
+ export declare const RETRY_STATUSES: readonly number[];
8
+ /** Three tries, one second then two apart, never a wait over 30 seconds. */
9
+ export declare const DEFAULT_RETRY_RULE: ResolvedRetryRule;
10
+ /** Why an attempt may be worth repeating, and the server's own `Retry-After`, if it sent one. */
11
+ export interface Transient {
12
+ reason: string;
13
+ retryAfter?: string;
14
+ }
15
+ /** One try of a request, and how to tell a passing failure in what it gave. */
16
+ export interface TransportAttempt<T> {
17
+ run: () => Promise<T>;
18
+ /** A transient problem in the outcome (a retry status, a connection error), or `undefined` when the outcome stands. */
19
+ problem: (outcome: {
20
+ value: T;
21
+ } | {
22
+ error: unknown;
23
+ }) => Transient | undefined;
24
+ }
25
+ export interface RetryContext {
26
+ recipeId: string;
27
+ gate: RunGate;
28
+ events: EventBus;
29
+ rule: ResolvedRetryRule;
30
+ }
31
+ /**
32
+ * The retry rule a recipe runs with: its `limits.retry` over the crawler's
33
+ * default over `DEFAULT_RETRY_RULE`.
34
+ *
35
+ * @param own - The recipe's `limits.retry`.
36
+ * @param crawler - The crawler's `retry` option.
37
+ * @returns The rule.
38
+ */
39
+ export declare function resolveRetryRule(own?: RetryRule, crawler?: RetryRule): ResolvedRetryRule;
40
+ /**
41
+ * Whether an error is a connection failure worth another try.
42
+ *
43
+ * @param error - What the request threw.
44
+ * @returns The reason, or `undefined`.
45
+ */
46
+ export declare function transientError(error: unknown): Transient | undefined;
47
+ /**
48
+ * How long to wait before attempt `attempt + 1`: the server's `Retry-After`
49
+ * when it gave one, else `backoffMs` doubling per attempt with a little
50
+ * jitter; `undefined` when the server asks for longer than `maxDelayMs` (it
51
+ * means "come back much later", which a crawl cannot wait for).
52
+ *
53
+ * @param rule - The retry rule.
54
+ * @param attempt - The attempt that just failed, from 1.
55
+ * @param retryAfter - The `Retry-After` header: seconds, or an HTTP date.
56
+ * @param now - The current time, for dates.
57
+ * @returns Milliseconds, or `undefined` for no retry.
58
+ */
59
+ export declare function retryDelay(rule: ResolvedRetryRule, attempt: number, retryAfter?: string, now?: number): number | undefined;
60
+ /**
61
+ * Sends a request through the gate (the recipe's rate, the site's lane), and
62
+ * sends it again after a pause while it fails in a passing way, up to
63
+ * `rule.attempts` tries in all. A `Retry-After` holds back every request to
64
+ * that site, not only this one. Each retry is reported as `request:retry`.
65
+ *
66
+ * Like redialling a busy number: wait a moment, dial again, give up after a
67
+ * few tries; and if the other end said "call back in a minute", wait that minute.
68
+ *
69
+ * @param url - Where the request goes.
70
+ * @param attempt - How to send it and how to judge the outcome.
71
+ * @param context - The recipe, gate, events and rule.
72
+ * @returns What the last try gave.
73
+ * @throws What the last try threw.
74
+ */
75
+ export declare function withTransportRetry<T>(url: string, attempt: TransportAttempt<T>, context: RetryContext): Promise<T>;
76
+ //# sourceMappingURL=transport-retry.policy.d.ts.map
@@ -5,8 +5,9 @@ import type { GotoStep, InputRecipe } from '../recipe-schema/index.js';
5
5
  import type { RunGate } from '../step-flow/index.js';
6
6
  /**
7
7
  * Runs a `goto` step: renders the URL (relative to the current page), waits for
8
- * the gate's throttle (`delayMs`), navigates, records the page's real URL in the
9
- * scope, and checks the response against the recipe's block rule.
8
+ * the gate's throttle (`delayMs`), navigates (again, after a pause, while it
9
+ * fails in passing: `limits.retry`), records the page's real URL in the scope,
10
+ * and checks the response against the recipe's block rule.
10
11
  *
11
12
  * @throws BlockedError when the response is a block.
12
13
  */
@@ -1,21 +1,36 @@
1
- import type { BrowserSession } from '../browser-session/index.js';
1
+ import { BrowserSession } from '../browser-session/index.js';
2
+ import type { CaptchaGuard } from '../captcha/index.js';
2
3
  import type { EventBus } from '../crawl-events/index.js';
3
4
  import type { ExtractionScope, LiveElement } from '../extraction-scope/index.js';
4
5
  import type { InputRecipe, PaginateNext, Step } from '../recipe-schema/index.js';
5
6
  import { RunGate } from '../step-flow/index.js';
6
7
  import type { NextPageResult, StepRunner } from '../step-flow/index.js';
7
- /** Runs web-mode leaf steps on a browser page. */
8
+ /**
9
+ * Runs web-mode leaf steps on a browser page. With a captcha guard, a page a
10
+ * navigation, click or key press leads to is checked for a challenge, solved
11
+ * before the next step runs.
12
+ */
8
13
  export declare class WebStepRunner implements StepRunner {
9
14
  private readonly session;
10
15
  private readonly recipe;
11
16
  private readonly events;
12
17
  private readonly gate;
18
+ private readonly captcha?;
13
19
  private readonly page;
14
- constructor(session: BrowserSession, recipe: InputRecipe, events: EventBus, gate?: RunGate);
20
+ constructor(session: BrowserSession, recipe: InputRecipe, events: EventBus, gate?: RunGate, captcha?: CaptchaGuard | undefined);
15
21
  /** Clicks and key presses can navigate; keep `page.url` honest after every leaf step. */
16
22
  private trackUrl;
23
+ /** Navigates; a block page showing a captcha is solved under `onBlock.solve`, and a page reached is checked for one. */
24
+ private visit;
17
25
  runLeaf(step: Step, scope: ExtractionScope): Promise<void>;
18
26
  nextPage(next: PaginateNext, scope: ExtractionScope): Promise<NextPageResult>;
27
+ /**
28
+ * A runner on a new tab of the same context, for one parallel iteration:
29
+ * it shares cookies, the gate and the captcha guard; disposing it closes the tab only.
30
+ *
31
+ * @returns The forked runner.
32
+ */
33
+ fork(): Promise<WebStepRunner>;
19
34
  elements(selector: string): Promise<LiveElement[]>;
20
35
  dispose(): Promise<void>;
21
36
  }
@@ -0,0 +1,26 @@
1
+ /** The delimiters detection chooses between, in order of preference on a tie. */
2
+ export declare const CSV_DELIMITERS: readonly [",", ";", "\t", "|"];
3
+ /**
4
+ * Parses CSV text (RFC 4180, tolerant): a field in double quotes may hold the
5
+ * delimiter, line breaks and `""` for a quote; a quote inside an unquoted
6
+ * field is taken literally (`1.0 Hybrid "Cross"`); CRLF, LF and CR all end a
7
+ * record. Rows are kept as read: ragged rows stay ragged, nothing is trimmed.
8
+ *
9
+ * @param text - The decoded file.
10
+ * @param delimiter - One character.
11
+ * @returns The rows; a trailing empty line adds no row.
12
+ */
13
+ export declare function parseCsv(text: string, delimiter: string): string[][];
14
+ /**
15
+ * Chooses the delimiter of a CSV: the candidate whose field count (above one)
16
+ * is the most consistent over the first lines, so a title line or two above
17
+ * the header does not mislead it. A file of one column gets `,`.
18
+ *
19
+ * `;` with decimal commas (`Panda;15.950,00`), the usual European export,
20
+ * scores `;`: a comma split gives rows of uneven width.
21
+ *
22
+ * @param text - The decoded file.
23
+ * @returns The delimiter.
24
+ */
25
+ export declare function detectDelimiter(text: string): string;
26
+ //# sourceMappingURL=csv-parser.algorithm.d.ts.map
@@ -0,0 +1,24 @@
1
+ import type { WorkbookDocument } from './workbook-document.model.js';
2
+ /**
3
+ * Reads decoded CSV text into a workbook of one sheet, named after the file.
4
+ *
5
+ * @param text - The decoded file.
6
+ * @param options - The sheet name, the encoding it was decoded from (for a
7
+ * probe to report) and a delimiter; without one it is detected.
8
+ * @returns The workbook.
9
+ * @throws Error when the delimiter given is not one character.
10
+ */
11
+ export declare function csvWorkbook(text: string, options: {
12
+ name: string;
13
+ encoding: string;
14
+ delimiter?: string;
15
+ }): WorkbookDocument;
16
+ /**
17
+ * The name a CSV's sheet takes: the file name without its extension
18
+ * (`…/prezzo_alle_8.csv` → `prezzo_alle_8`), else `csv`.
19
+ *
20
+ * @param url - Where the file came from.
21
+ * @returns The name.
22
+ */
23
+ export declare function sheetNameOf(url: string): string;
24
+ //# sourceMappingURL=csv-workbook.mapper.d.ts.map
@@ -0,0 +1,53 @@
1
+ import type { WorkbookCell, WorkbookDocument } from './workbook-document.model.js';
2
+ /** What a table extract looks for in a workbook. */
3
+ export interface GridTableQuery {
4
+ /** Matches a table's (first) header row: its non-empty cells joined by spaces, whitespace collapsed. */
5
+ header: RegExp;
6
+ /** Matches the row that ends a table; a table also ends at the next header or the sheet's end. `^$` ends it at the first empty row. */
7
+ until?: RegExp;
8
+ /** Output key -> a pattern for that column's header; unmatched columns are dropped. Without it, the headers are the keys. */
9
+ columns?: Record<string, RegExp>;
10
+ /** Matches the names of the sheets to read; default every sheet. */
11
+ sheet?: RegExp;
12
+ /** How many rows the header spans (default 1): a column's key joins its header texts. */
13
+ headerRows?: number;
14
+ /** Output keys whose empty cells take the value of the row above. */
15
+ fillDown?: string[];
16
+ /** Read hidden sheets and hidden rows too. */
17
+ includeHidden?: boolean;
18
+ }
19
+ /** One table found in a workbook. */
20
+ export interface GridTable {
21
+ sheet: string;
22
+ /** The first header cell: the table's name when it has one. */
23
+ title: string;
24
+ /** The column keys, left to right. */
25
+ header: string[];
26
+ /** One object per row, keyed by column; text trimmed, numbers and booleans as they are. */
27
+ rows: Record<string, WorkbookCell>[];
28
+ }
29
+ /**
30
+ * Finds every table whose header row matches, in every sheet the query
31
+ * selects, and reads its rows by column. Unlike a PDF, a grid needs no
32
+ * geometry: column *i* of a row belongs to header *i*.
33
+ *
34
+ * Merged ranges are filled first (the file stores their value in the top-left
35
+ * cell only), so a brand merged down its models' rows reads on every row, and
36
+ * a group header merged across its sub-columns names each of them. Empty rows
37
+ * are skipped.
38
+ *
39
+ * @param document - The workbook.
40
+ * @param query - Which tables, and how to name their columns.
41
+ * @returns The tables, sheet by sheet, top to bottom.
42
+ */
43
+ export declare function findGridTables(document: WorkbookDocument, query: GridTableQuery): GridTable[];
44
+ /**
45
+ * Fills blank cells in the given columns with the value of the row above,
46
+ * within one table: pivot exports write a group's name on its first row only.
47
+ *
48
+ * @param rows - The table's rows, in order.
49
+ * @param keys - The columns to fill.
50
+ * @returns The rows, filled (new objects; the input is not changed).
51
+ */
52
+ export declare function fillDown<Row extends Record<string, unknown>>(rows: readonly Row[], keys: readonly string[]): Row[];
53
+ //# sourceMappingURL=grid-table.algorithm.d.ts.map
@@ -0,0 +1,14 @@
1
+ import type { Sheet } from './workbook-document.model.js';
2
+ /**
3
+ * Every `<table>` of an HTML document as a sheet (`table 1`, `table 2`…, in
4
+ * document order), so the workbook table reader works on web pages and
5
+ * rendered Markdown: rows in order (`thead`, `tbody`, `tfoot` alike), `th` and
6
+ * `td` alike, cell text with whitespace collapsed, `colspan` and `rowspan` as
7
+ * merged ranges. A table inside a table is a sheet of its own, and its rows
8
+ * are not its parent's.
9
+ *
10
+ * @param html - The document.
11
+ * @returns The tables.
12
+ */
13
+ export declare function htmlTableSheets(html: string): Sheet[];
14
+ //# sourceMappingURL=html-tables.mapper.d.ts.map
@@ -0,0 +1,9 @@
1
+ export { workbookText, isWorkbookDocument } from './workbook-document.model.js';
2
+ export type { WorkbookDocument, Sheet, CsvFormat, WorkbookCell } from './workbook-document.model.js';
3
+ export { parseCsv, detectDelimiter, CSV_DELIMITERS } from './csv-parser.algorithm.js';
4
+ export { csvWorkbook, sheetNameOf } from './csv-workbook.mapper.js';
5
+ export { readXlsxWorkbook } from './read-xlsx.client.js';
6
+ export { findGridTables, fillDown } from './grid-table.algorithm.js';
7
+ export { htmlTableSheets } from './html-tables.mapper.js';
8
+ export type { GridTable, GridTableQuery } from './grid-table.algorithm.js';
9
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1,18 @@
1
+ import type { WorkbookDocument } from './workbook-document.model.js';
2
+ /**
3
+ * Reads an `.xlsx` workbook into a workbook document, through
4
+ * `@opencraw/office-reader`: every worksheet's cells, with hidden sheets,
5
+ * hidden rows and merged ranges. Numbers and booleans keep their type (a
6
+ * cell's `13955.625` is unambiguous; as text, a locale guess could read it as
7
+ * thirteen million), dates become ISO text, errors their text, empty cells
8
+ * `''`. Formulas give their cached value. The reader is imported on first use,
9
+ * so recipes that never read a spreadsheet never load it.
10
+ *
11
+ * @param bytes - The file.
12
+ * @param source - Where it came from, for messages.
13
+ * @returns The workbook.
14
+ * @throws Error naming the source, and saying what to do, for a file that is
15
+ * not a readable workbook (a legacy `.xls`, a password-protected file, an `.ods`…).
16
+ */
17
+ export declare function readXlsxWorkbook(bytes: Uint8Array, source: string): Promise<WorkbookDocument>;
18
+ //# sourceMappingURL=read-xlsx.client.d.ts.map
@@ -0,0 +1,51 @@
1
+ /**
2
+ * A cell: text, or, from a spreadsheet, the number or boolean it holds
3
+ * (`13955.625` stays a number, so no locale guess can misread it). Dates are
4
+ * ISO text (`2026-06-01`, `2026-06-01T09:30:00`), errors their text
5
+ * (`#DIV/0!`), an empty cell `''`. A CSV's cells are all text.
6
+ */
7
+ export type WorkbookCell = string | number | boolean;
8
+ /** One sheet of a workbook. A CSV is a workbook of one sheet. */
9
+ export interface Sheet {
10
+ name: string;
11
+ /** Top to bottom; a row holds as many cells as were read (rows can be ragged). */
12
+ rows: WorkbookCell[][];
13
+ /** A sheet hidden in the workbook: `table` skips it unless `includeHidden`. */
14
+ hidden?: boolean;
15
+ /** Rows hidden in the sheet (0-based): `table` skips them unless `includeHidden`. */
16
+ hiddenRows?: number[];
17
+ /**
18
+ * Merged ranges as A1 references (`B10:B13`). The value of a merged range
19
+ * sits in its top-left cell only, as the file stores it; `table` copies it
20
+ * into every cell the range covers.
21
+ */
22
+ merges?: string[];
23
+ }
24
+ /** How a CSV was read, for a probe to report. */
25
+ export interface CsvFormat {
26
+ encoding: string;
27
+ delimiter: string;
28
+ }
29
+ /** A spreadsheet or a CSV read into sheets of cells: what `extract` works on. */
30
+ export interface WorkbookDocument {
31
+ kind: 'workbook';
32
+ sheets: Sheet[];
33
+ /** Present when the workbook came from a CSV. */
34
+ csv?: CsvFormat;
35
+ }
36
+ /**
37
+ * The text a `regex` extract reads: the visible rows of the visible sheets,
38
+ * cells separated by a tab, sheets separated by a blank line.
39
+ *
40
+ * @param document - The workbook.
41
+ * @returns The text.
42
+ */
43
+ export declare function workbookText(document: WorkbookDocument): string;
44
+ /**
45
+ * Whether a value bound in scope is a read workbook (so `extract … from` can take it).
46
+ *
47
+ * @param value - Anything.
48
+ * @returns Whether it is a {@link WorkbookDocument}.
49
+ */
50
+ export declare function isWorkbookDocument(value: unknown): value is WorkbookDocument;
51
+ //# sourceMappingURL=workbook-document.model.d.ts.map
@@ -0,0 +1,3 @@
1
+ export { readYaml } from './read-yaml.client.js';
2
+ export type { YamlRead, YamlScalars } from './read-yaml.client.js';
3
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1,25 @@
1
+ /** How YAML scalars are read: typed as YAML 1.2 types them, or as the text written. */
2
+ export type YamlScalars = 'typed' | 'text';
3
+ /** A YAML text, read. */
4
+ export interface YamlRead {
5
+ /** The document's value; several documents (`---`) give an array of their values. */
6
+ data: unknown;
7
+ documents: number;
8
+ /** What the parser noticed but read anyway: an unknown tag (`!!js/function`) is kept as its plain value. */
9
+ warnings: string[];
10
+ }
11
+ /**
12
+ * Parses YAML with the `yaml` package, imported on first use. The version is
13
+ * pinned to YAML 1.2 (core schema) whatever the document declares: under a
14
+ * `%YAML 1.1` directive, `NO` would read as `false` and `0123` as octal `83`.
15
+ * Merge keys (`<<: *base`) are applied, duplicate keys are an error, aliases
16
+ * are capped, and custom tags never build values: nothing in the text runs.
17
+ *
18
+ * @param text - The YAML.
19
+ * @param source - Where it came from, for messages.
20
+ * @param scalars - `typed` (default), or `text` to keep every scalar as written (`0123` stays `"0123"`).
21
+ * @returns The data, the number of documents and the warnings.
22
+ * @throws Error naming the source, with the line and column, for YAML that does not parse.
23
+ */
24
+ export declare function readYaml(text: string, source: string, scalars?: YamlScalars): Promise<YamlRead>;
25
+ //# sourceMappingURL=read-yaml.client.d.ts.map
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@opencraw/core",
3
- "version": "0.1.1",
3
+ "version": "0.1.3",
4
4
  "type": "module",
5
5
  "main": "./dist/index.esm.js",
6
6
  "module": "./dist/index.esm.js",
@@ -20,11 +20,14 @@
20
20
  "!**/*.js.map"
21
21
  ],
22
22
  "dependencies": {
23
+ "@opencraw/office-reader": "^0.0.2",
23
24
  "cheerio": "^1.2.0",
24
25
  "domhandler": "^6.0.1",
25
26
  "jsonpath-plus": "^10.4.0",
27
+ "marked": "^18.0.14",
26
28
  "pdfjs-dist": "^6.3.289",
27
29
  "playwright": "^1.63.0",
30
+ "yaml": "^2.9.1",
28
31
  "zod": "^4.6.5"
29
32
  },
30
33
  "publishConfig": {
@@ -54,7 +57,8 @@
54
57
  "e2e": {
55
58
  "executor": "nx:run-commands",
56
59
  "dependsOn": [
57
- "build"
60
+ "build",
61
+ "^build"
58
62
  ],
59
63
  "options": {
60
64
  "command": "tsc -p tsconfig.e2e.json && jest --config jest.e2e.config.cts",
@@ -70,6 +74,16 @@
70
74
  "command": "node tools/emit-json-schemas.mjs",
71
75
  "cwd": "packages/core"
72
76
  }
77
+ },
78
+ "typecheck": {
79
+ "dependsOn": [
80
+ "^build"
81
+ ]
82
+ },
83
+ "lint": {
84
+ "dependsOn": [
85
+ "^build"
86
+ ]
73
87
  }
74
88
  }
75
89
  }