@opencraw/core 0.1.0

package/dist/src/template/template.algorithm.d.ts

@@ -0,0 +1,53 @@
+/**
+ * `{{ }}` interpolation for recipe strings. A placeholder is a dotted path
+ * resolved through the lookup the caller provides, or an expression over such
+ * paths (see `expression.algorithm`). No code is ever executed.
+ */
+import type { Lookup } from './value-text.algorithm.js';
+/**
+ * Whether a string contains at least one placeholder.
+ *
+ * @param text - The string.
+ * @returns `true` when it has a `{{ }}`.
+ */
+export declare function hasPlaceholder(text: string): boolean;
+/**
+ * Renders a template. When the template is exactly one placeholder the resolved
+ * value is returned as is (so a list or a number survives); otherwise every
+ * placeholder is stringified into the text, `undefined`/`null` as `''`.
+ *
+ * @param template - The string to render.
+ * @param lookup - Resolves a path.
+ * @returns The rendered value.
+ */
+export declare function render(template: string, lookup: Lookup): unknown;
+/**
+ * The value of one placeholder: a plain path is looked up as is; anything else
+ * is parsed and evaluated as an expression.
+ *
+ * @param inner - The trimmed text between the braces.
+ * @param lookup - Resolves a path.
+ * @returns The value.
+ * @throws Error when an expression does not parse.
+ */
+export declare function resolve(inner: string, lookup: Lookup): unknown;
+/**
+ * Renders every string inside a JSON-shaped value (an object, an array, or a
+ * string) with {@link render}, so a lone placeholder keeps its type
+ * (`"{{vars.limit}}"` becomes the number) and anything else becomes text.
+ * Numbers, booleans and `null` pass through untouched.
+ *
+ * @param value - The value.
+ * @param lookup - Resolves a path.
+ * @returns A new value with every string rendered.
+ */
+export declare function renderDeep(value: unknown, lookup: Lookup): unknown;
+/**
+ * Renders a template that must produce text.
+ *
+ * @param template - The string to render.
+ * @param lookup - Resolves a path.
+ * @returns The rendered text; a non-string single value is stringified.
+ */
+export declare function renderText(template: string, lookup: Lookup): string;
+//# sourceMappingURL=template.algorithm.d.ts.map

package/dist/src/template/value-path.algorithm.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Reads and writes values by dotted path (`seller.name`, `items.0.url`, `items[0].url`).
+ * Pure: the read never throws on a missing segment, the write creates what is missing.
+ */
+/**
+ * Splits a path into its segments; `a.b[0].c` and `a.b.0.c` are the same path.
+ *
+ * @param path - The dotted path.
+ * @returns Its segments, `[]` for an empty path or `.`.
+ */
+export declare function segmentsOf(path: string): string[];
+/**
+ * Reads a value by path. Only own properties are read, so `constructor`,
+ * `__proto__`, `toString` and the like resolve to `undefined` whatever the data.
+ *
+ * @param source - Where to read from.
+ * @param path - The dotted path; `.` or `''` returns the source itself.
+ * @returns The value, or `undefined` when any segment is missing.
+ */
+export declare function getPath(source: unknown, path: string): unknown;
+/**
+ * Writes a value by path, creating intermediate objects (or arrays for numeric segments).
+ *
+ * @param target - The object to write into. Mutated.
+ * @param path - The dotted path.
+ * @param value - The value to store.
+ */
+export declare function setPath(target: Record<string, unknown>, path: string, value: unknown): void;
+//# sourceMappingURL=value-path.algorithm.d.ts.map

package/dist/src/template/value-text.algorithm.d.ts

@@ -0,0 +1,18 @@
+/** How recipes read a value: as text for interpolation, as a truth for conditions. Shared by templates and expressions. */
+/** Resolves a dotted path to a value; `undefined` when unknown. */
+export type Lookup = (path: string) => unknown;
+/**
+ * Truthiness as recipes mean it: `when` and `until` conditions.
+ *
+ * @param value - Any rendered value.
+ * @returns `false` for `undefined`, `null`, `''`, `0`, `false`, `'false'`, `'0'`, `'null'` and an empty list.
+ */
+export declare function isTruthy(value: unknown): boolean;
+/**
+ * The text form of a value for interpolation.
+ *
+ * @param value - Any value.
+ * @returns A string; objects and arrays as JSON.
+ */
+export declare function stringify(value: unknown): string;
+//# sourceMappingURL=value-text.algorithm.d.ts.map

package/dist/src/transformation/apply-transform-chain.use-case.d.ts

@@ -0,0 +1,17 @@
+import type { TransformRule } from '../recipe-schema/index.js';
+import type { TransformContext } from './transform-registry.store.js';
+/**
+ * Applies a chain of transforms in order. A scalar op applied to a list runs
+ * on every item; a list op runs on the list itself. A missing value (`undefined`
+ * or `null`) passes through every op except `default`, `template` and `hook`,
+ * so a missing source reaches the missing-value policy untouched. Async because
+ * a `hook` may be.
+ *
+ * @param value - The resolved source value(s).
+ * @param rules - The chain.
+ * @param context - Lookup, hooks and the base URL.
+ * @returns The transformed value.
+ * @throws TransformError (or whatever a hook throws) when a step cannot apply.
+ */
+export declare function applyTransformChain(value: unknown, rules: readonly TransformRule[], context: TransformContext): Promise<unknown>;
+//# sourceMappingURL=apply-transform-chain.use-case.d.ts.map

package/dist/src/transformation/collection.algorithm.d.ts

@@ -0,0 +1,15 @@
+/** The input as a list: a list as is, a missing value as empty; anything else fails the op. */
+export declare function asList(op: string, value: unknown): unknown[];
+export declare function first(value: unknown): unknown;
+export declare function last(value: unknown): unknown;
+export declare function nth(value: unknown, index: number): unknown;
+export declare function slice(value: unknown, start: number, end?: number): unknown[];
+export declare function join(value: unknown, separator: string): string;
+export declare function concat(value: unknown, separator?: string): string;
+/** The first value that is not `undefined`, `null` or blank text. */
+export declare function coalesce(value: unknown): unknown;
+export declare function flatten(value: unknown): unknown[];
+export declare function unique(value: unknown): unknown[];
+export declare function sum(value: unknown): number;
+export declare function count(value: unknown): number;
+//# sourceMappingURL=collection.algorithm.d.ts.map

package/dist/src/transformation/currency.algorithm.d.ts

@@ -0,0 +1,15 @@
+/** A money amount with its ISO 4217 code, when known. */
+export interface Money {
+    amount: number;
+    currency?: string;
+}
+/**
+ * Parses a price. The currency comes from the explicit code, else a code or symbol in the text.
+ *
+ * @param value - Text such as `1.299,00 €` or a number.
+ * @param locale - The locale the text is written in.
+ * @param currency - An explicit ISO 4217 code that wins over anything in the text.
+ * @returns The money value.
+ */
+export declare function parseCurrency(value: unknown, locale?: string, currency?: string): Money;
+//# sourceMappingURL=currency.algorithm.d.ts.map

package/dist/src/transformation/date.algorithm.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Parses a date or instant.
+ *
+ * @param value - Text, a number (epoch milliseconds) or a Date.
+ * @param format - Tokens `YYYY MM DD HH mm ss`, e.g. `DD/MM/YYYY`; without one the text must be ISO 8601 or otherwise `Date.parse`-able.
+ * @param timezone - An IANA zone the text is written in when it carries no offset; default UTC.
+ * @returns The instant.
+ * @throws TransformError when the text cannot be read.
+ */
+export declare function parseDate(value: unknown, format?: string, timezone?: string): Date;
+/** `YYYY-MM-DD` of an instant, in UTC. */
+export declare function toIsoDate(date: Date): string;
+//# sourceMappingURL=date.algorithm.d.ts.map

package/dist/src/transformation/index.d.ts

@@ -0,0 +1,10 @@
+export { applyTransformChain } from './apply-transform-chain.use-case.js';
+export { BUILT_IN_TRANSFORMS } from './transform-registry.store.js';
+export type { TransformContext } from './transform-registry.store.js';
+export { TransformError } from './transform.error.js';
+export { parseNumber, parseInteger, parseBoolean } from './number.algorithm.js';
+export { parseCurrency } from './currency.algorithm.js';
+export type { Money } from './currency.algorithm.js';
+export { parseDate, toIsoDate } from './date.algorithm.js';
+export { absoluteUrl } from './url.algorithm.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/transformation/lookup.algorithm.d.ts

@@ -0,0 +1,24 @@
+/**
+ * The first item of a table whose `key` path equals a value, compared as text
+ * so `"3"` finds `3`. The table is whatever a bound id holds: data, JSON text
+ * or a list of JSON texts (a `data-*` attribute per element).
+ *
+ * @param value - The value to find.
+ * @param table - The table, as bound in scope.
+ * @param key - A dotted path inside each item.
+ * @param pick - A dotted path to return from the item; the item itself when omitted.
+ * @returns The picked value, or `undefined` when nothing matches.
+ */
+export declare function lookup(value: unknown, table: unknown, key: string, pick?: string): unknown;
+/**
+ * Groups a list by a path, keeping first-seen order.
+ *
+ * @param items - The list.
+ * @param by - A dotted path inside each item; items without one group under `null`.
+ * @returns One `{ key, items }` per distinct key.
+ */
+export declare function group(items: unknown[], by: string): {
+    key: unknown;
+    items: unknown[];
+}[];
+//# sourceMappingURL=lookup.algorithm.d.ts.map

package/dist/src/transformation/number.algorithm.d.ts

@@ -0,0 +1,30 @@
+/**
+ * The decimal and group separators a locale uses.
+ *
+ * @param locale - A BCP 47 tag; `undefined` means "guess from the text".
+ * @returns The two separators.
+ */
+export declare function separatorsOf(locale: string): {
+    decimal: string;
+    group: string;
+};
+/**
+ * Parses a number from text, tolerating currency symbols, spaces and locale separators.
+ *
+ * @param value - Text or a number.
+ * @param locale - The locale the text is written in. Without one, the last separator is the decimal one when
+ * it is followed by 1 or 2 digits, otherwise it is a group separator.
+ * @returns The number.
+ * @throws TransformError when no number can be read.
+ */
+export declare function parseNumber(value: unknown, locale?: string): number;
+export declare function parseInteger(value: unknown, locale?: string): number;
+/**
+ * Reads a boolean the way a recipe means it.
+ *
+ * @param value - Any value.
+ * @param truthy - Phrases that mean `true` (case-insensitive, matched as a substring); default: yes/true/1/on/in stock/available.
+ * @returns The boolean.
+ */
+export declare function parseBoolean(value: unknown, truthy?: readonly string[]): boolean;
+//# sourceMappingURL=number.algorithm.d.ts.map

package/dist/src/transformation/string.algorithm.d.ts

@@ -0,0 +1,18 @@
+/** A short description of a value for error messages. */
+export declare function describe(value: unknown): string;
+export declare function trim(value: unknown): string;
+export declare function lowercase(value: unknown): string;
+export declare function uppercase(value: unknown): string;
+export declare function replace(value: unknown, pattern: string, replacement: string, flags?: string): string;
+/**
+ * The first match of a pattern.
+ *
+ * @param value - The text.
+ * @param pattern - A regular expression source.
+ * @param group - The capture group to return; default: group 1 when the pattern has one, else the whole match.
+ * @param flags - Regular expression flags.
+ * @returns The captured text, or `undefined` when nothing matches.
+ */
+export declare function regex(value: unknown, pattern: string, group?: number, flags?: string): string | undefined;
+export declare function split(value: unknown, separator: string): string[];
+//# sourceMappingURL=string.algorithm.d.ts.map

package/dist/src/transformation/transform-registry.store.d.ts

@@ -0,0 +1,36 @@
+import type { HookRegistry } from '../hooks/index.js';
+import type { TransformOp, TransformRule } from '../recipe-schema/index.js';
+import type { Lookup } from '../template/index.js';
+/** What a transform may need besides its input. */
+export interface TransformContext {
+    recipeId: string;
+    /** Resolves template paths against the current scope. */
+    lookup: Lookup;
+    /** The scope snapshot, for hooks. */
+    scope: Record<string, unknown>;
+    hooks: HookRegistry;
+    /** The current page URL; `absoluteUrl` resolves against it by default. */
+    baseUrl?: string;
+    log: (level: 'debug' | 'info' | 'warn' | 'error', message: string, meta?: Record<string, unknown>) => void;
+}
+type RuleOf<Op extends TransformOp> = Extract<TransformRule, {
+    op: Op;
+}>;
+/** One registered operation. `elementwise` ops apply to each item when the input is a list. */
+interface Registered<Op extends TransformOp = TransformOp> {
+    elementwise: boolean;
+    apply: (value: unknown, rule: RuleOf<Op>, context: TransformContext) => unknown;
+}
+/** The closed set of built-in transforms, one per `TransformRule` op. */
+export declare const BUILT_IN_TRANSFORMS: {
+    [Op in TransformOp]: Registered<Op>;
+};
+/**
+ * Looks up the implementation of a rule.
+ *
+ * @param rule - A transform rule from a recipe.
+ * @returns The registered operation.
+ */
+export declare function transformFor(rule: TransformRule): Registered;
+export {};
+//# sourceMappingURL=transform-registry.store.d.ts.map

package/dist/src/transformation/transform.error.d.ts

@@ -0,0 +1,8 @@
+/** A transform that could not be applied to its input. */
+export declare class TransformError extends Error {
+    readonly op: string;
+    readonly input?: unknown | undefined;
+    readonly name = "TransformError";
+    constructor(op: string, reason: string, input?: unknown | undefined);
+}
+//# sourceMappingURL=transform.error.d.ts.map

package/dist/src/transformation/url.algorithm.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Percent-encodes a value for one URL component (a path segment, a query
+ * value), with `encodeURIComponent`'s rules: everything but letters, digits
+ * and `-_.!~*'()` is encoded, non-ASCII as UTF-8.
+ *
+ * @param value - Text, a number or a boolean.
+ * @returns The encoded text.
+ * @throws TransformError for anything else.
+ */
+export declare function urlEncode(value: unknown): string;
+/**
+ * Resolves a possibly relative URL.
+ *
+ * @param value - The URL text.
+ * @param base - What relative URLs resolve against; usually the page URL.
+ * @returns The absolute URL.
+ * @throws TransformError when the result is not a valid URL.
+ */
+export declare function absoluteUrl(value: unknown, base?: string): string;
+//# sourceMappingURL=url.algorithm.d.ts.map

package/dist/src/web-steps/evaluate-script.use-case.d.ts

@@ -0,0 +1,10 @@
+import type { Page } from 'playwright';
+import type { ExtractionScope } from '../extraction-scope/index.js';
+import type { EvaluateStep } from '../recipe-schema/index.js';
+/**
+ * Runs an `evaluate` step: the script is evaluated in the page as an expression
+ * (a function expression is called), and its JSON-serialisable result is bound
+ * under the step id. Trusted recipes only: this is arbitrary code in the page.
+ */
+export declare function evaluateScript(step: EvaluateStep, page: Page, scope: ExtractionScope): Promise<void>;
+//# sourceMappingURL=evaluate-script.use-case.d.ts.map

package/dist/src/web-steps/extract-from-page.use-case.d.ts

@@ -0,0 +1,10 @@
+import type { Page } from 'playwright';
+import type { ExtractionScope } from '../extraction-scope/index.js';
+import type { ExtractStep } from '../recipe-schema/index.js';
+/**
+ * Runs an `extract` step on the live page (css or xpath through locators, one
+ * round trip for all matches). With `from`, it reads a fragment bound earlier
+ * instead, the same way api mode reads a fetched document.
+ */
+export declare function extractFromPage(step: ExtractStep, page: Page, scope: ExtractionScope): Promise<void>;
+//# sourceMappingURL=extract-from-page.use-case.d.ts.map

package/dist/src/web-steps/index.d.ts

@@ -0,0 +1,2 @@
+export { WebStepRunner } from './run-web-step.use-case.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/web-steps/interact.use-case.d.ts

@@ -0,0 +1,36 @@
+import type { Locator, Page } from 'playwright';
+import type { ExtractionScope } from '../extraction-scope/index.js';
+import type { ClickStep, FillStep, PressStep, ScreenshotStep, ScrollStep, SelectStep, TargetFields, WaitStep } from '../recipe-schema/index.js';
+/**
+ * The element an interaction lands on: the first match of `selector`, or what
+ * `target` renders to - a live element (re-resolved by selector and index, so a
+ * re-render since the loop started does not matter) or a selector string.
+ *
+ * @param step - A step with `selector` or `target`.
+ * @param page - The page.
+ * @param scope - Where `target` is resolved.
+ * @returns The locator.
+ * @throws Error when `target` renders to something that is neither.
+ */
+export declare function targetOf(step: TargetFields, page: Page, scope: ExtractionScope): Locator;
+export declare function click(step: ClickStep, page: Page, scope: ExtractionScope): Promise<void>;
+export declare function fill(step: FillStep, page: Page, scope: ExtractionScope): Promise<void>;
+export declare function press(step: PressStep, page: Page, scope: ExtractionScope): Promise<void>;
+/** Picks an option of a `<select>` by value, label or index; each is a template. */
+export declare function select(step: SelectStep, page: Page, scope: ExtractionScope): Promise<void>;
+/**
+ * Scrolls to the bottom (or to an element) `times` times; with `untilStable`
+ * it keeps going until the page stops growing, which is how infinite lists end.
+ */
+export declare function scroll(step: ScrollStep, page: Page): Promise<void>;
+export declare function wait(step: WaitStep, page: Page): Promise<void>;
+/**
+ * Whether a locator becomes visible within a timeout. Never throws.
+ *
+ * @param target - The locator.
+ * @param timeout - Milliseconds to wait.
+ * @returns `true` when visible in time.
+ */
+export declare function appears(target: Locator, timeout: number): Promise<boolean>;
+export declare function screenshot(step: ScreenshotStep, page: Page, scope: ExtractionScope): Promise<void>;
+//# sourceMappingURL=interact.use-case.d.ts.map

package/dist/src/web-steps/navigate.use-case.d.ts

@@ -0,0 +1,14 @@
+import type { Page } from 'playwright';
+import type { EventBus } from '../crawl-events/index.js';
+import type { ExtractionScope } from '../extraction-scope/index.js';
+import type { GotoStep, InputRecipe } from '../recipe-schema/index.js';
+import type { RunGate } from '../step-flow/index.js';
+/**
+ * Runs a `goto` step: renders the URL (relative to the current page), waits for
+ * the gate's throttle (`delayMs`), navigates, records the page's real URL in the
+ * scope, and checks the response against the recipe's block rule.
+ *
+ * @throws BlockedError when the response is a block.
+ */
+export declare function navigate(step: GotoStep, page: Page, scope: ExtractionScope, recipe: InputRecipe, gate: RunGate, events: EventBus): Promise<void>;
+//# sourceMappingURL=navigate.use-case.d.ts.map

package/dist/src/web-steps/run-web-step.use-case.d.ts

@@ -0,0 +1,22 @@
+import type { BrowserSession } from '../browser-session/index.js';
+import type { EventBus } from '../crawl-events/index.js';
+import type { ExtractionScope, LiveElement } from '../extraction-scope/index.js';
+import type { InputRecipe, PaginateNext, Step } from '../recipe-schema/index.js';
+import { RunGate } from '../step-flow/index.js';
+import type { NextPageResult, StepRunner } from '../step-flow/index.js';
+/** Runs web-mode leaf steps on a browser page. */
+export declare class WebStepRunner implements StepRunner {
+    private readonly session;
+    private readonly recipe;
+    private readonly events;
+    private readonly gate;
+    private readonly page;
+    constructor(session: BrowserSession, recipe: InputRecipe, events: EventBus, gate?: RunGate);
+    /** Clicks and key presses can navigate; keep `page.url` honest after every leaf step. */
+    private trackUrl;
+    runLeaf(step: Step, scope: ExtractionScope): Promise<void>;
+    nextPage(next: PaginateNext, scope: ExtractionScope): Promise<NextPageResult>;
+    elements(selector: string): Promise<LiveElement[]>;
+    dispose(): Promise<void>;
+}
+//# sourceMappingURL=run-web-step.use-case.d.ts.map

package/dist/src/web-steps/snapshot-elements.use-case.d.ts

@@ -0,0 +1,13 @@
+import type { Page } from 'playwright';
+import type { LiveElement } from '../extraction-scope/index.js';
+/**
+ * Snapshots every element a selector matches, in one round trip, for a
+ * `forEach` over `selector`. Each snapshot carries the selector and its index
+ * so a later `target` can find the element again, even after a re-render.
+ *
+ * @param selector - A css selector (or `xpath=...`), already rendered.
+ * @param page - The page.
+ * @returns One snapshot per match, in document order.
+ */
+export declare function snapshotElements(selector: string, page: Page): Promise<LiveElement[]>;
+//# sourceMappingURL=snapshot-elements.use-case.d.ts.map

package/package.json

@@ -0,0 +1,75 @@
+{
+  "name": "@opencraw/core",
+  "version": "0.1.0",
+  "type": "module",
+  "main": "./dist/index.esm.js",
+  "module": "./dist/index.esm.js",
+  "types": "./dist/src/index.d.ts",
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "types": "./dist/src/index.d.ts",
+      "import": "./dist/index.esm.js",
+      "default": "./dist/index.esm.js"
+    }
+  },
+  "files": [
+    "dist",
+    "!**/*.tsbuildinfo",
+    "!**/*.d.ts.map",
+    "!**/*.js.map"
+  ],
+  "dependencies": {
+    "cheerio": "^1.2.0",
+    "domhandler": "^6.0.1",
+    "jsonpath-plus": "^10.4.0",
+    "playwright": "^1.63.0",
+    "zod": "^4.6.5"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "description": "Recipe-driven crawler engine: JSON input recipes (Playwright browser or HTTP API steps) mapped through transforms to JSON output schemas.",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/russoedu/open.craw.git",
+    "directory": "packages/core"
+  },
+  "keywords": [
+    "crawler",
+    "scraper",
+    "playwright",
+    "recipe",
+    "data-driven",
+    "json"
+  ],
+  "engines": {
+    "node": ">=22"
+  },
+  "sideEffects": false,
+  "nx": {
+    "targets": {
+      "e2e": {
+        "executor": "nx:run-commands",
+        "dependsOn": [
+          "build"
+        ],
+        "options": {
+          "command": "tsc -p tsconfig.e2e.json && jest --config jest.e2e.config.cts",
+          "cwd": "packages/core"
+        }
+      },
+      "schemas": {
+        "executor": "nx:run-commands",
+        "dependsOn": [
+          "build"
+        ],
+        "options": {
+          "command": "node tools/emit-json-schemas.mjs",
+          "cwd": "packages/core"
+        }
+      }
+    }
+  }
+}