harnery 0.40.0 → 0.42.0

package/dist/lib/browser/collect/collector.d.ts

@@ -0,0 +1,99 @@
+import { type Rng, type ScrollPacing, type ScrollStepPlan, type WheelTick } from "./scroll-plan.js";
+/** One item as read from the page in a single harvest. */
+export interface HarvestedItem {
+    key: string;
+    fields: Record<string, unknown>;
+}
+export interface CollectedItem {
+    key: string;
+    /** Step index the item was first seen at (0 is the initial harvest). */
+    firstSeenStep: number;
+    [field: string]: unknown;
+}
+export interface PageMetrics {
+    scrollTop: number;
+    scrollHeight: number;
+    viewportHeight: number;
+    atBottom: boolean;
+}
+/** What the collector needs from a page. The Playwright driver implements it. */
+export interface CollectDriver {
+    harvest(): Promise<HarvestedItem[]>;
+    metrics(): Promise<PageMetrics>;
+    /** Scroll by these wheel notches. Returns true when a non-wheel fallback was used. */
+    wheel(ticks: WheelTick[]): Promise<boolean>;
+    /** Click up to `limit` unexpanded "show more" buttons; returns how many were clicked. */
+    expand(limit: number): Promise<number>;
+    /** A short reason when the page shows a login wall or rate limit, else null. */
+    blocked(): Promise<string | null>;
+    sleep(ms: number): Promise<void>;
+    now(): number;
+}
+export interface CollectLimits {
+    /** Stop once this many unique items are collected (null: no item cap). */
+    maxItems: number | null;
+    /** Stop after this many scroll steps. */
+    maxScrolls: number;
+    /** Stop once this much wall time has passed, in ms. */
+    maxMs: number;
+    /** Stop after this many consecutive steps with no new items and no page growth. */
+    idleSteps: number;
+    /** Expand clicks per step (0 disables expansion). */
+    expandPerStep: number;
+    /** Expand clicks per run. */
+    expandTotal: number;
+}
+export declare const DEFAULT_COLLECT_LIMITS: CollectLimits;
+export type StopReason = "max-items" | "max-scrolls" | "time-budget" | "idle" | "end-of-feed" | "blocked" | "interrupted";
+export interface CollectStats {
+    /** Scroll steps taken (the initial harvest is step 0 and not counted). */
+    steps: number;
+    harvests: number;
+    items: number;
+    /** Sightings of an already-collected key across all harvests. */
+    duplicates: number;
+    expanded: number;
+    readingPauses: number;
+    scrollBacks: number;
+    /** Steps where wheel scrolling did not move the page and a fallback ran. */
+    scrollFallbacks: number;
+    elapsedMs: number;
+    stopReason: StopReason;
+    stopDetail: string | null;
+}
+export interface CollectResult {
+    items: CollectedItem[];
+    stats: CollectStats;
+}
+export interface CollectRunOptions {
+    limits: CollectLimits;
+    pacing: ScrollPacing;
+    rng: Rng;
+    /** Polled between and during pauses; true stops the run with "interrupted". */
+    shouldStop?: () => boolean;
+    onStep?: (info: {
+        step: number;
+        items: number;
+        added: number;
+        plan: ScrollStepPlan;
+    }) => void;
+}
+/**
+ * Ordered, de-duplicated item store. The first sighting fixes an item's
+ * position and step; later sightings only fill in what was missing: empty
+ * fields, longer text (an expanded "show more"), and new array members.
+ */
+export declare class ItemStore {
+    private byKey;
+    duplicates: number;
+    get size(): number;
+    add(items: HarvestedItem[], step: number, maxItems?: number | null): number;
+    items(): CollectedItem[];
+}
+/**
+ * Scroll a feed like a reader and collect every item that passes through the
+ * DOM. Each step harvests right after scrolling and again after the pause, so
+ * a virtualized list cannot drop an item between two harvests.
+ */
+export declare function runCollect(driver: CollectDriver, options: CollectRunOptions): Promise<CollectResult>;
+//# sourceMappingURL=collector.d.ts.map

package/dist/lib/browser/collect/collector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"collector.d.ts","sourceRoot":"","sources":["../../../../src/lib/browser/collect/collector.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,GAAG,EACR,KAAK,YAAY,EACjB,KAAK,cAAc,EAEnB,KAAK,SAAS,EACf,MAAM,kBAAkB,CAAC;AAE1B,0DAA0D;AAC1D,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,aAAa;IAC5B,GAAG,EAAE,MAAM,CAAC;IACZ,wEAAwE;IACxE,aAAa,EAAE,MAAM,CAAC;IACtB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,iFAAiF;AACjF,MAAM,WAAW,aAAa;IAC5B,OAAO,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC;IACpC,OAAO,IAAI,OAAO,CAAC,WAAW,CAAC,CAAC;IAChC,sFAAsF;IACtF,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5C,yFAAyF;IACzF,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACvC,gFAAgF;IAChF,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAClC,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,GAAG,IAAI,MAAM,CAAC;CACf;AAED,MAAM,WAAW,aAAa;IAC5B,0EAA0E;IAC1E,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAC;IACnB,uDAAuD;IACvD,KAAK,EAAE,MAAM,CAAC;IACd,mFAAmF;IACnF,SAAS,EAAE,MAAM,CAAC;IAClB,qDAAqD;IACrD,aAAa,EAAE,MAAM,CAAC;IACtB,6BAA6B;IAC7B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,eAAO,MAAM,sBAAsB,EAAE,aAOpC,CAAC;AAEF,MAAM,MAAM,UAAU,GAClB,WAAW,GACX,aAAa,GACb,aAAa,GACb,MAAM,GACN,aAAa,GACb,SAAS,GACT,aAAa,CAAC;AAElB,MAAM,WAAW,YAAY;IAC3B,0EAA0E;IAC1E,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,iEAAiE;IACjE,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,4EAA4E;IAC5E,eAAe,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,UAAU,CAAC;IACvB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,KAAK,EAAE,YAAY,CAAC;CACrB;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,YAAY,CAAC;IACrB,GAAG,EAAE,GAAG,CAAC;IACT,+EAA+E;IAC/E,UAAU,CAAC,EAAE,MAAM,OAAO,CAAC;IAC3B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,cAAc,CAAA;KAAE,KAAK,IAAI,CAAC;CAC/F;AAWD;;;;GAIG;AACH,qBAAa,SAAS;IACpB,OAAO,CAAC,KAAK,CAAoC;IACjD,UAAU,SAAK;IAEf,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,GAAG,CAAC,KAAK,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,GAAE,MAAM,GAAG,IAAW,GAAG,MAAM;IAiBjF,KAAK,IAAI,aAAa,EAAE;CAGzB;AAkBD;;;;GAIG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,aAAa,EACrB,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,aAAa,CAAC,CAkIxB"}

package/dist/lib/browser/collect/collector.js

@@ -0,0 +1,202 @@
+import { planScrollStep, uniform, } from "./scroll-plan.js";
+export const DEFAULT_COLLECT_LIMITS = {
+    maxItems: null,
+    maxScrolls: 60,
+    maxMs: 300_000,
+    idleSteps: 5,
+    expandPerStep: 0,
+    expandTotal: 50,
+};
+function isEmpty(value) {
+    return (value === null ||
+        value === undefined ||
+        value === "" ||
+        (Array.isArray(value) && value.length === 0));
+}
+/**
+ * Ordered, de-duplicated item store. The first sighting fixes an item's
+ * position and step; later sightings only fill in what was missing: empty
+ * fields, longer text (an expanded "show more"), and new array members.
+ */
+export class ItemStore {
+    byKey = new Map();
+    duplicates = 0;
+    get size() {
+        return this.byKey.size;
+    }
+    add(items, step, maxItems = null) {
+        let added = 0;
+        for (const item of items) {
+            if (!item.key)
+                continue;
+            const existing = this.byKey.get(item.key);
+            if (existing) {
+                this.duplicates++;
+                mergeFields(existing, item.fields);
+                continue;
+            }
+            if (maxItems !== null && this.byKey.size >= maxItems)
+                continue;
+            this.byKey.set(item.key, { key: item.key, firstSeenStep: step, ...item.fields });
+            added++;
+        }
+        return added;
+    }
+    items() {
+        return [...this.byKey.values()];
+    }
+}
+function mergeFields(target, fields) {
+    for (const [name, value] of Object.entries(fields)) {
+        const current = target[name];
+        if (isEmpty(value))
+            continue;
+        if (isEmpty(current)) {
+            target[name] = value;
+        }
+        else if (typeof current === "string" && typeof value === "string") {
+            if (value.length > current.length)
+                target[name] = value;
+        }
+        else if (Array.isArray(current) && Array.isArray(value)) {
+            const merged = [...current];
+            for (const v of value)
+                if (!merged.includes(v))
+                    merged.push(v);
+            target[name] = merged;
+        }
+    }
+}
+/**
+ * Scroll a feed like a reader and collect every item that passes through the
+ * DOM. Each step harvests right after scrolling and again after the pause, so
+ * a virtualized list cannot drop an item between two harvests.
+ */
+export async function runCollect(driver, options) {
+    const { limits, pacing, rng } = options;
+    const store = new ItemStore();
+    const started = driver.now();
+    const stats = {
+        steps: 0,
+        harvests: 0,
+        items: 0,
+        duplicates: 0,
+        expanded: 0,
+        readingPauses: 0,
+        scrollBacks: 0,
+        scrollFallbacks: 0,
+        elapsedMs: 0,
+        stopReason: "idle",
+        stopDetail: null,
+    };
+    const elapsed = () => driver.now() - started;
+    const interrupted = () => options.shouldStop?.() === true;
+    const full = () => limits.maxItems !== null && store.size >= limits.maxItems;
+    const harvest = async (step) => {
+        stats.harvests++;
+        return store.add(await driver.harvest(), step, limits.maxItems);
+    };
+    /** Sleep in slices so an interrupt or the time budget cuts a pause short. */
+    const pause = async (ms) => {
+        let remaining = Math.min(ms, Math.max(0, limits.maxMs - elapsed()));
+        while (remaining > 0 && !interrupted()) {
+            const slice = Math.min(remaining, 250);
+            await driver.sleep(slice);
+            remaining -= slice;
+        }
+    };
+    const expand = async (step) => {
+        if (limits.expandPerStep <= 0)
+            return 0;
+        const budget = Math.min(limits.expandPerStep, limits.expandTotal - stats.expanded);
+        if (budget <= 0)
+            return 0;
+        const clicked = await driver.expand(budget);
+        stats.expanded += clicked;
+        return clicked > 0 ? harvest(step) : 0;
+    };
+    const stop = (reason, detail = null) => {
+        stats.stopReason = reason;
+        stats.stopDetail = detail;
+    };
+    await harvest(0);
+    await expand(0);
+    let idle = 0;
+    let before = await driver.metrics();
+    for (;;) {
+        if (interrupted()) {
+            stop("interrupted");
+            break;
+        }
+        if (full()) {
+            stop("max-items", `collected ${store.size} items`);
+            break;
+        }
+        if (stats.steps >= limits.maxScrolls) {
+            stop("max-scrolls", `took ${stats.steps} scroll steps`);
+            break;
+        }
+        if (elapsed() >= limits.maxMs) {
+            stop("time-budget", `ran ${Math.round(elapsed() / 1000)} s`);
+            break;
+        }
+        const step = ++stats.steps;
+        const plan = planScrollStep(rng, pacing, before.viewportHeight);
+        let added = 0;
+        if (plan.back.length > 0) {
+            stats.scrollBacks++;
+            await driver.wheel(plan.back);
+            added += await harvest(step);
+            await pause(plan.backPauseMs);
+        }
+        if (await driver.wheel(plan.ticks))
+            stats.scrollFallbacks++;
+        added += await harvest(step);
+        if (plan.reading)
+            stats.readingPauses++;
+        // Split the pause so items rendered mid-pause are harvested too.
+        const firstHalf = Math.round(plan.pauseMs * uniform(rng, { min: 0.35, max: 0.65 }));
+        await pause(firstHalf);
+        added += await harvest(step);
+        added += await expand(step);
+        await pause(plan.pauseMs - firstHalf);
+        added += await harvest(step);
+        const after = await driver.metrics();
+        options.onStep?.({ step, items: store.size, added, plan });
+        const grew = after.scrollHeight > before.scrollHeight;
+        const moved = after.scrollTop > before.scrollTop;
+        before = after;
+        if (added > 0 || grew) {
+            idle = 0;
+            continue;
+        }
+        if (full())
+            continue;
+        const blocked = await driver.blocked();
+        if (blocked) {
+            stop("blocked", blocked);
+            break;
+        }
+        idle++;
+        if (idle >= limits.idleSteps) {
+            if (after.atBottom) {
+                stop("end-of-feed", `no new items or page growth for ${idle} steps at the bottom`);
+            }
+            else if (!moved) {
+                stop("idle", `the page stopped scrolling; no new items for ${idle} steps`);
+            }
+            else {
+                stop("idle", `no new items or page growth for ${idle} steps`);
+            }
+            break;
+        }
+    }
+    const items = store.items();
+    stats.items = items.length;
+    stats.duplicates = store.duplicates;
+    stats.elapsedMs = Math.round(elapsed());
+    if (items.length === 0 && stats.stopDetail && stats.stopReason !== "blocked") {
+        stats.stopDetail += " (the item selector matched nothing)";
+    }
+    return { items, stats };
+}

package/dist/lib/browser/collect/fields.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Field extractor grammar shared by presets and `--collect-field`.
+ *
+ *   <name>=<selector>[@<attr>]
+ *   <name>[]=<selector>[@<attr>]     every match, as a de-duplicated array
+ *
+ * The selector runs inside each item. An empty selector or `:scope` means the
+ * item itself. Without `@attr` the value is the element's visible text; with
+ * it, the attribute (href and src resolve to absolute URLs).
+ */
+export interface Extractor {
+    /** CSS selector relative to the item; "" means the item itself. */
+    selector: string;
+    /** Attribute to read; null reads visible text. */
+    attr: string | null;
+}
+export interface FieldSpec extends Extractor {
+    name: string;
+    /** Collect every match as an array instead of the first one. */
+    all: boolean;
+}
+/** Item properties the collector owns; fields cannot reuse these names. */
+export declare const RESERVED_FIELD_NAMES: Set<string>;
+/**
+ * Parse `<selector>[@<attr>]`. The attribute separator is the last `@` that
+ * sits outside quotes and brackets, so `a[href*="@"]@href` works.
+ */
+export declare function parseExtractor(spec: string): Extractor;
+/** Parse `<name>=<extractor>` or `<name>[]=<extractor>`. */
+export declare function parseFieldSpec(raw: string): FieldSpec;
+//# sourceMappingURL=fields.d.ts.map

package/dist/lib/browser/collect/fields.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fields.d.ts","sourceRoot":"","sources":["../../../../src/lib/browser/collect/fields.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,MAAM,WAAW,SAAS;IACxB,mEAAmE;IACnE,QAAQ,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACrB;AAED,MAAM,WAAW,SAAU,SAAQ,SAAS;IAC1C,IAAI,EAAE,MAAM,CAAC;IACb,gEAAgE;IAChE,GAAG,EAAE,OAAO,CAAC;CACd;AAED,2EAA2E;AAC3E,eAAO,MAAM,oBAAoB,aAAoC,CAAC;AAKtE;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CA6BtD;AAED,4DAA4D;AAC5D,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAoBrD"}

package/dist/lib/browser/collect/fields.js

@@ -0,0 +1,75 @@
+/**
+ * Field extractor grammar shared by presets and `--collect-field`.
+ *
+ *   <name>=<selector>[@<attr>]
+ *   <name>[]=<selector>[@<attr>]     every match, as a de-duplicated array
+ *
+ * The selector runs inside each item. An empty selector or `:scope` means the
+ * item itself. Without `@attr` the value is the element's visible text; with
+ * it, the attribute (href and src resolve to absolute URLs).
+ */
+/** Item properties the collector owns; fields cannot reuse these names. */
+export const RESERVED_FIELD_NAMES = new Set(["key", "firstSeenStep"]);
+const NAME_RE = /^[A-Za-z_][\w-]*$/;
+const ATTR_RE = /^[A-Za-z_:][-\w:.]*$/;
+/**
+ * Parse `<selector>[@<attr>]`. The attribute separator is the last `@` that
+ * sits outside quotes and brackets, so `a[href*="@"]@href` works.
+ */
+export function parseExtractor(spec) {
+    const text = spec.trim();
+    let at = -1;
+    let quote = null;
+    let depth = 0;
+    for (let i = 0; i < text.length; i++) {
+        const ch = text[i];
+        if (quote) {
+            if (ch === "\\")
+                i++;
+            else if (ch === quote)
+                quote = null;
+            continue;
+        }
+        if (ch === '"' || ch === "'")
+            quote = ch;
+        else if (ch === "[" || ch === "(")
+            depth++;
+        else if (ch === "]" || ch === ")")
+            depth = Math.max(0, depth - 1);
+        else if (ch === "@" && depth === 0)
+            at = i;
+    }
+    let selector = text;
+    let attr = null;
+    if (at >= 0) {
+        const candidate = text.slice(at + 1).trim();
+        if (!ATTR_RE.test(candidate)) {
+            throw new Error(`Extractor "${spec}": "${candidate}" is not a valid attribute name.`);
+        }
+        attr = candidate;
+        selector = text.slice(0, at).trim();
+    }
+    if (selector === ":scope")
+        selector = "";
+    return { selector, attr };
+}
+/** Parse `<name>=<extractor>` or `<name>[]=<extractor>`. */
+export function parseFieldSpec(raw) {
+    const eq = raw.indexOf("=");
+    if (eq <= 0) {
+        throw new Error(`--collect-field expects <name>=<selector>[@attr] (got "${raw}"). Example: time=time@datetime`);
+    }
+    let name = raw.slice(0, eq).trim();
+    let all = false;
+    if (name.endsWith("[]")) {
+        all = true;
+        name = name.slice(0, -2);
+    }
+    if (!NAME_RE.test(name)) {
+        throw new Error(`Field name "${name}" must start with a letter or _ and use word characters.`);
+    }
+    if (RESERVED_FIELD_NAMES.has(name)) {
+        throw new Error(`Field name "${name}" is reserved by the collector.`);
+    }
+    return { name, all, ...parseExtractor(raw.slice(eq + 1)) };
+}

package/dist/lib/browser/collect/index.d.ts

@@ -0,0 +1,6 @@
+export { type CollectDriver, type CollectedItem, type CollectLimits, type CollectResult, type CollectRunOptions, type CollectStats, DEFAULT_COLLECT_LIMITS, type HarvestedItem, ItemStore, type PageMetrics, runCollect, type StopReason, } from "./collector.js";
+export { type Extractor, type FieldSpec, parseExtractor, parseFieldSpec, RESERVED_FIELD_NAMES, } from "./fields.js";
+export { createPageDriver, type PageDriverDeps } from "./page-driver.js";
+export { COLLECT_PRESETS, type CollectPreset, type CollectSpecOverrides, presetNames, type ResolvedCollectSpec, resolveCollectSpec, } from "./presets.js";
+export { createRng, DEFAULT_SCROLL_PACING, PACING_LIMITS, parseProbability, parseRange, planScrollStep, type Range, type Rng, randomSeed, type ScrollPacing, type ScrollStepPlan, scrollPacingFromEnv, uniform, type WheelTick, wheelTicks, } from "./scroll-plan.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/lib/browser/collect/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/browser/collect/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,aAAa,EAClB,KAAK,aAAa,EAClB,KAAK,aAAa,EAClB,KAAK,aAAa,EAClB,KAAK,iBAAiB,EACtB,KAAK,YAAY,EACjB,sBAAsB,EACtB,KAAK,aAAa,EAClB,SAAS,EACT,KAAK,WAAW,EAChB,UAAU,EACV,KAAK,UAAU,GAChB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,KAAK,SAAS,EACd,KAAK,SAAS,EACd,cAAc,EACd,cAAc,EACd,oBAAoB,GACrB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,gBAAgB,EAAE,KAAK,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACzE,OAAO,EACL,eAAe,EACf,KAAK,aAAa,EAClB,KAAK,oBAAoB,EACzB,WAAW,EACX,KAAK,mBAAmB,EACxB,kBAAkB,GACnB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,SAAS,EACT,qBAAqB,EACrB,aAAa,EACb,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,KAAK,KAAK,EACV,KAAK,GAAG,EACR,UAAU,EACV,KAAK,YAAY,EACjB,KAAK,cAAc,EACnB,mBAAmB,EACnB,OAAO,EACP,KAAK,SAAS,EACd,UAAU,GACX,MAAM,kBAAkB,CAAC"}

package/dist/lib/browser/collect/index.js

@@ -0,0 +1,5 @@
+export { DEFAULT_COLLECT_LIMITS, ItemStore, runCollect, } from "./collector.js";
+export { parseExtractor, parseFieldSpec, RESERVED_FIELD_NAMES, } from "./fields.js";
+export { createPageDriver } from "./page-driver.js";
+export { COLLECT_PRESETS, presetNames, resolveCollectSpec, } from "./presets.js";
+export { createRng, DEFAULT_SCROLL_PACING, PACING_LIMITS, parseProbability, parseRange, planScrollStep, randomSeed, scrollPacingFromEnv, uniform, wheelTicks, } from "./scroll-plan.js";

package/dist/lib/browser/collect/page-driver.d.ts

@@ -0,0 +1,14 @@
+import type { Page } from "playwright";
+import type { CollectDriver } from "./collector.js";
+import type { ResolvedCollectSpec } from "./presets.js";
+import { type Range, type Rng } from "./scroll-plan.js";
+export interface PageDriverDeps {
+    rng: Rng;
+    sleep?: (ms: number) => Promise<void>;
+    now?: () => number;
+    log?: (message: string) => void;
+    /** Pause before each expand click, in ms (default 250-700). */
+    expandPauseMs?: Range;
+}
+export declare function createPageDriver(page: Page, spec: ResolvedCollectSpec, deps: PageDriverDeps): CollectDriver;
+//# sourceMappingURL=page-driver.d.ts.map

package/dist/lib/browser/collect/page-driver.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"page-driver.d.ts","sourceRoot":"","sources":["../../../../src/lib/browser/collect/page-driver.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,KAAK,EAAE,aAAa,EAA8B,MAAM,gBAAgB,CAAC;AAChF,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACxD,OAAO,EAAE,KAAK,KAAK,EAAE,KAAK,GAAG,EAA2B,MAAM,kBAAkB,CAAC;AAmKjF,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,GAAG,CAAC;IACT,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,+DAA+D;IAC/D,aAAa,CAAC,EAAE,KAAK,CAAC;CACvB;AAED,wBAAgB,gBAAgB,CAC9B,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,mBAAmB,EACzB,IAAI,EAAE,cAAc,GACnB,aAAa,CAmJf"}