harnery 0.41.0 → 0.42.0

package/dist/lib/browser/collect/collector.js

@@ -0,0 +1,202 @@
+import { planScrollStep, uniform, } from "./scroll-plan.js";
+export const DEFAULT_COLLECT_LIMITS = {
+    maxItems: null,
+    maxScrolls: 60,
+    maxMs: 300_000,
+    idleSteps: 5,
+    expandPerStep: 0,
+    expandTotal: 50,
+};
+function isEmpty(value) {
+    return (value === null ||
+        value === undefined ||
+        value === "" ||
+        (Array.isArray(value) && value.length === 0));
+}
+/**
+ * Ordered, de-duplicated item store. The first sighting fixes an item's
+ * position and step; later sightings only fill in what was missing: empty
+ * fields, longer text (an expanded "show more"), and new array members.
+ */
+export class ItemStore {
+    byKey = new Map();
+    duplicates = 0;
+    get size() {
+        return this.byKey.size;
+    }
+    add(items, step, maxItems = null) {
+        let added = 0;
+        for (const item of items) {
+            if (!item.key)
+                continue;
+            const existing = this.byKey.get(item.key);
+            if (existing) {
+                this.duplicates++;
+                mergeFields(existing, item.fields);
+                continue;
+            }
+            if (maxItems !== null && this.byKey.size >= maxItems)
+                continue;
+            this.byKey.set(item.key, { key: item.key, firstSeenStep: step, ...item.fields });
+            added++;
+        }
+        return added;
+    }
+    items() {
+        return [...this.byKey.values()];
+    }
+}
+function mergeFields(target, fields) {
+    for (const [name, value] of Object.entries(fields)) {
+        const current = target[name];
+        if (isEmpty(value))
+            continue;
+        if (isEmpty(current)) {
+            target[name] = value;
+        }
+        else if (typeof current === "string" && typeof value === "string") {
+            if (value.length > current.length)
+                target[name] = value;
+        }
+        else if (Array.isArray(current) && Array.isArray(value)) {
+            const merged = [...current];
+            for (const v of value)
+                if (!merged.includes(v))
+                    merged.push(v);
+            target[name] = merged;
+        }
+    }
+}
+/**
+ * Scroll a feed like a reader and collect every item that passes through the
+ * DOM. Each step harvests right after scrolling and again after the pause, so
+ * a virtualized list cannot drop an item between two harvests.
+ */
+export async function runCollect(driver, options) {
+    const { limits, pacing, rng } = options;
+    const store = new ItemStore();
+    const started = driver.now();
+    const stats = {
+        steps: 0,
+        harvests: 0,
+        items: 0,
+        duplicates: 0,
+        expanded: 0,
+        readingPauses: 0,
+        scrollBacks: 0,
+        scrollFallbacks: 0,
+        elapsedMs: 0,
+        stopReason: "idle",
+        stopDetail: null,
+    };
+    const elapsed = () => driver.now() - started;
+    const interrupted = () => options.shouldStop?.() === true;
+    const full = () => limits.maxItems !== null && store.size >= limits.maxItems;
+    const harvest = async (step) => {
+        stats.harvests++;
+        return store.add(await driver.harvest(), step, limits.maxItems);
+    };
+    /** Sleep in slices so an interrupt or the time budget cuts a pause short. */
+    const pause = async (ms) => {
+        let remaining = Math.min(ms, Math.max(0, limits.maxMs - elapsed()));
+        while (remaining > 0 && !interrupted()) {
+            const slice = Math.min(remaining, 250);
+            await driver.sleep(slice);
+            remaining -= slice;
+        }
+    };
+    const expand = async (step) => {
+        if (limits.expandPerStep <= 0)
+            return 0;
+        const budget = Math.min(limits.expandPerStep, limits.expandTotal - stats.expanded);
+        if (budget <= 0)
+            return 0;
+        const clicked = await driver.expand(budget);
+        stats.expanded += clicked;
+        return clicked > 0 ? harvest(step) : 0;
+    };
+    const stop = (reason, detail = null) => {
+        stats.stopReason = reason;
+        stats.stopDetail = detail;
+    };
+    await harvest(0);
+    await expand(0);
+    let idle = 0;
+    let before = await driver.metrics();
+    for (;;) {
+        if (interrupted()) {
+            stop("interrupted");
+            break;
+        }
+        if (full()) {
+            stop("max-items", `collected ${store.size} items`);
+            break;
+        }
+        if (stats.steps >= limits.maxScrolls) {
+            stop("max-scrolls", `took ${stats.steps} scroll steps`);
+            break;
+        }
+        if (elapsed() >= limits.maxMs) {
+            stop("time-budget", `ran ${Math.round(elapsed() / 1000)} s`);
+            break;
+        }
+        const step = ++stats.steps;
+        const plan = planScrollStep(rng, pacing, before.viewportHeight);
+        let added = 0;
+        if (plan.back.length > 0) {
+            stats.scrollBacks++;
+            await driver.wheel(plan.back);
+            added += await harvest(step);
+            await pause(plan.backPauseMs);
+        }
+        if (await driver.wheel(plan.ticks))
+            stats.scrollFallbacks++;
+        added += await harvest(step);
+        if (plan.reading)
+            stats.readingPauses++;
+        // Split the pause so items rendered mid-pause are harvested too.
+        const firstHalf = Math.round(plan.pauseMs * uniform(rng, { min: 0.35, max: 0.65 }));
+        await pause(firstHalf);
+        added += await harvest(step);
+        added += await expand(step);
+        await pause(plan.pauseMs - firstHalf);
+        added += await harvest(step);
+        const after = await driver.metrics();
+        options.onStep?.({ step, items: store.size, added, plan });
+        const grew = after.scrollHeight > before.scrollHeight;
+        const moved = after.scrollTop > before.scrollTop;
+        before = after;
+        if (added > 0 || grew) {
+            idle = 0;
+            continue;
+        }
+        if (full())
+            continue;
+        const blocked = await driver.blocked();
+        if (blocked) {
+            stop("blocked", blocked);
+            break;
+        }
+        idle++;
+        if (idle >= limits.idleSteps) {
+            if (after.atBottom) {
+                stop("end-of-feed", `no new items or page growth for ${idle} steps at the bottom`);
+            }
+            else if (!moved) {
+                stop("idle", `the page stopped scrolling; no new items for ${idle} steps`);
+            }
+            else {
+                stop("idle", `no new items or page growth for ${idle} steps`);
+            }
+            break;
+        }
+    }
+    const items = store.items();
+    stats.items = items.length;
+    stats.duplicates = store.duplicates;
+    stats.elapsedMs = Math.round(elapsed());
+    if (items.length === 0 && stats.stopDetail && stats.stopReason !== "blocked") {
+        stats.stopDetail += " (the item selector matched nothing)";
+    }
+    return { items, stats };
+}

package/dist/lib/browser/collect/fields.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Field extractor grammar shared by presets and `--collect-field`.
+ *
+ *   <name>=<selector>[@<attr>]
+ *   <name>[]=<selector>[@<attr>]     every match, as a de-duplicated array
+ *
+ * The selector runs inside each item. An empty selector or `:scope` means the
+ * item itself. Without `@attr` the value is the element's visible text; with
+ * it, the attribute (href and src resolve to absolute URLs).
+ */
+export interface Extractor {
+    /** CSS selector relative to the item; "" means the item itself. */
+    selector: string;
+    /** Attribute to read; null reads visible text. */
+    attr: string | null;
+}
+export interface FieldSpec extends Extractor {
+    name: string;
+    /** Collect every match as an array instead of the first one. */
+    all: boolean;
+}
+/** Item properties the collector owns; fields cannot reuse these names. */
+export declare const RESERVED_FIELD_NAMES: Set<string>;
+/**
+ * Parse `<selector>[@<attr>]`. The attribute separator is the last `@` that
+ * sits outside quotes and brackets, so `a[href*="@"]@href` works.
+ */
+export declare function parseExtractor(spec: string): Extractor;
+/** Parse `<name>=<extractor>` or `<name>[]=<extractor>`. */
+export declare function parseFieldSpec(raw: string): FieldSpec;
+//# sourceMappingURL=fields.d.ts.map

package/dist/lib/browser/collect/fields.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fields.d.ts","sourceRoot":"","sources":["../../../../src/lib/browser/collect/fields.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,MAAM,WAAW,SAAS;IACxB,mEAAmE;IACnE,QAAQ,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACrB;AAED,MAAM,WAAW,SAAU,SAAQ,SAAS;IAC1C,IAAI,EAAE,MAAM,CAAC;IACb,gEAAgE;IAChE,GAAG,EAAE,OAAO,CAAC;CACd;AAED,2EAA2E;AAC3E,eAAO,MAAM,oBAAoB,aAAoC,CAAC;AAKtE;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CA6BtD;AAED,4DAA4D;AAC5D,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAoBrD"}

package/dist/lib/browser/collect/fields.js

@@ -0,0 +1,75 @@
+/**
+ * Field extractor grammar shared by presets and `--collect-field`.
+ *
+ *   <name>=<selector>[@<attr>]
+ *   <name>[]=<selector>[@<attr>]     every match, as a de-duplicated array
+ *
+ * The selector runs inside each item. An empty selector or `:scope` means the
+ * item itself. Without `@attr` the value is the element's visible text; with
+ * it, the attribute (href and src resolve to absolute URLs).
+ */
+/** Item properties the collector owns; fields cannot reuse these names. */
+export const RESERVED_FIELD_NAMES = new Set(["key", "firstSeenStep"]);
+const NAME_RE = /^[A-Za-z_][\w-]*$/;
+const ATTR_RE = /^[A-Za-z_:][-\w:.]*$/;
+/**
+ * Parse `<selector>[@<attr>]`. The attribute separator is the last `@` that
+ * sits outside quotes and brackets, so `a[href*="@"]@href` works.
+ */
+export function parseExtractor(spec) {
+    const text = spec.trim();
+    let at = -1;
+    let quote = null;
+    let depth = 0;
+    for (let i = 0; i < text.length; i++) {
+        const ch = text[i];
+        if (quote) {
+            if (ch === "\\")
+                i++;
+            else if (ch === quote)
+                quote = null;
+            continue;
+        }
+        if (ch === '"' || ch === "'")
+            quote = ch;
+        else if (ch === "[" || ch === "(")
+            depth++;
+        else if (ch === "]" || ch === ")")
+            depth = Math.max(0, depth - 1);
+        else if (ch === "@" && depth === 0)
+            at = i;
+    }
+    let selector = text;
+    let attr = null;
+    if (at >= 0) {
+        const candidate = text.slice(at + 1).trim();
+        if (!ATTR_RE.test(candidate)) {
+            throw new Error(`Extractor "${spec}": "${candidate}" is not a valid attribute name.`);
+        }
+        attr = candidate;
+        selector = text.slice(0, at).trim();
+    }
+    if (selector === ":scope")
+        selector = "";
+    return { selector, attr };
+}
+/** Parse `<name>=<extractor>` or `<name>[]=<extractor>`. */
+export function parseFieldSpec(raw) {
+    const eq = raw.indexOf("=");
+    if (eq <= 0) {
+        throw new Error(`--collect-field expects <name>=<selector>[@attr] (got "${raw}"). Example: time=time@datetime`);
+    }
+    let name = raw.slice(0, eq).trim();
+    let all = false;
+    if (name.endsWith("[]")) {
+        all = true;
+        name = name.slice(0, -2);
+    }
+    if (!NAME_RE.test(name)) {
+        throw new Error(`Field name "${name}" must start with a letter or _ and use word characters.`);
+    }
+    if (RESERVED_FIELD_NAMES.has(name)) {
+        throw new Error(`Field name "${name}" is reserved by the collector.`);
+    }
+    return { name, all, ...parseExtractor(raw.slice(eq + 1)) };
+}

package/dist/lib/browser/collect/index.d.ts

@@ -0,0 +1,6 @@
+export { type CollectDriver, type CollectedItem, type CollectLimits, type CollectResult, type CollectRunOptions, type CollectStats, DEFAULT_COLLECT_LIMITS, type HarvestedItem, ItemStore, type PageMetrics, runCollect, type StopReason, } from "./collector.js";
+export { type Extractor, type FieldSpec, parseExtractor, parseFieldSpec, RESERVED_FIELD_NAMES, } from "./fields.js";
+export { createPageDriver, type PageDriverDeps } from "./page-driver.js";
+export { COLLECT_PRESETS, type CollectPreset, type CollectSpecOverrides, presetNames, type ResolvedCollectSpec, resolveCollectSpec, } from "./presets.js";
+export { createRng, DEFAULT_SCROLL_PACING, PACING_LIMITS, parseProbability, parseRange, planScrollStep, type Range, type Rng, randomSeed, type ScrollPacing, type ScrollStepPlan, scrollPacingFromEnv, uniform, type WheelTick, wheelTicks, } from "./scroll-plan.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/lib/browser/collect/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/lib/browser/collect/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,aAAa,EAClB,KAAK,aAAa,EAClB,KAAK,aAAa,EAClB,KAAK,aAAa,EAClB,KAAK,iBAAiB,EACtB,KAAK,YAAY,EACjB,sBAAsB,EACtB,KAAK,aAAa,EAClB,SAAS,EACT,KAAK,WAAW,EAChB,UAAU,EACV,KAAK,UAAU,GAChB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,KAAK,SAAS,EACd,KAAK,SAAS,EACd,cAAc,EACd,cAAc,EACd,oBAAoB,GACrB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,gBAAgB,EAAE,KAAK,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACzE,OAAO,EACL,eAAe,EACf,KAAK,aAAa,EAClB,KAAK,oBAAoB,EACzB,WAAW,EACX,KAAK,mBAAmB,EACxB,kBAAkB,GACnB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,SAAS,EACT,qBAAqB,EACrB,aAAa,EACb,gBAAgB,EAChB,UAAU,EACV,cAAc,EACd,KAAK,KAAK,EACV,KAAK,GAAG,EACR,UAAU,EACV,KAAK,YAAY,EACjB,KAAK,cAAc,EACnB,mBAAmB,EACnB,OAAO,EACP,KAAK,SAAS,EACd,UAAU,GACX,MAAM,kBAAkB,CAAC"}

package/dist/lib/browser/collect/index.js

@@ -0,0 +1,5 @@
+export { DEFAULT_COLLECT_LIMITS, ItemStore, runCollect, } from "./collector.js";
+export { parseExtractor, parseFieldSpec, RESERVED_FIELD_NAMES, } from "./fields.js";
+export { createPageDriver } from "./page-driver.js";
+export { COLLECT_PRESETS, presetNames, resolveCollectSpec, } from "./presets.js";
+export { createRng, DEFAULT_SCROLL_PACING, PACING_LIMITS, parseProbability, parseRange, planScrollStep, randomSeed, scrollPacingFromEnv, uniform, wheelTicks, } from "./scroll-plan.js";

package/dist/lib/browser/collect/page-driver.d.ts

@@ -0,0 +1,14 @@
+import type { Page } from "playwright";
+import type { CollectDriver } from "./collector.js";
+import type { ResolvedCollectSpec } from "./presets.js";
+import { type Range, type Rng } from "./scroll-plan.js";
+export interface PageDriverDeps {
+    rng: Rng;
+    sleep?: (ms: number) => Promise<void>;
+    now?: () => number;
+    log?: (message: string) => void;
+    /** Pause before each expand click, in ms (default 250-700). */
+    expandPauseMs?: Range;
+}
+export declare function createPageDriver(page: Page, spec: ResolvedCollectSpec, deps: PageDriverDeps): CollectDriver;
+//# sourceMappingURL=page-driver.d.ts.map

package/dist/lib/browser/collect/page-driver.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"page-driver.d.ts","sourceRoot":"","sources":["../../../../src/lib/browser/collect/page-driver.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,KAAK,EAAE,aAAa,EAA8B,MAAM,gBAAgB,CAAC;AAChF,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACxD,OAAO,EAAE,KAAK,KAAK,EAAE,KAAK,GAAG,EAA2B,MAAM,kBAAkB,CAAC;AAmKjF,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,GAAG,CAAC;IACT,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAChC,+DAA+D;IAC/D,aAAa,CAAC,EAAE,KAAK,CAAC;CACvB;AAED,wBAAgB,gBAAgB,CAC9B,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,mBAAmB,EACzB,IAAI,EAAE,cAAc,GACnB,aAAa,CAmJf"}

package/dist/lib/browser/collect/page-driver.js

@@ -0,0 +1,306 @@
+import { uniform } from "./scroll-plan.js";
+/** Attribute the driver puts on expand buttons it has handled. */
+const EXPAND_MARK = "data-harn-collect-expand";
+/** Generic signs of a rate limit or login wall, checked for every preset. */
+const GENERIC_BLOCKED_TEXT = [
+    "rate limit exceeded",
+    "too many requests",
+    "(log in|sign in|sign up) to (continue|see more|keep (reading|scrolling))",
+];
+/** Words that mark a button as an action rather than a "show more" toggle. */
+const UNSAFE_EXPAND_WORDS = "like|unlike|follow|reply|repost|retweet|share|subscribe|join|delete|block|report|mute|send|post|vote|bookmark|save";
+/** Runs in the page. Must stay self-contained: no closures over module scope. */
+function harvestInPage(args) {
+    const urlish = /href|src|permalink|url/i;
+    const read = (el, attr) => {
+        if (attr === null) {
+            const text = el.innerText ?? el.textContent ?? "";
+            return text.trim() || null;
+        }
+        let value = el.getAttribute(attr);
+        if (value === null)
+            return null;
+        value = value.trim();
+        if (value && urlish.test(attr) && !/^(data|blob|javascript):/i.test(value)) {
+            try {
+                value = new URL(value, location.href).href;
+            }
+            catch {
+                // Keep the raw attribute when it is not a URL.
+            }
+        }
+        return value || null;
+    };
+    const pick = (root, selector) => {
+        if (!selector)
+            return [root];
+        try {
+            return Array.from(root.querySelectorAll(selector));
+        }
+        catch {
+            return [];
+        }
+    };
+    const first = (root, selector, attr) => {
+        for (const el of pick(root, selector)) {
+            const v = read(el, attr);
+            if (v)
+                return v;
+        }
+        return null;
+    };
+    const permalinkRe = args.keyPattern
+        ? new RegExp(args.keyPattern)
+        : /\/(status|statuses|post|posts|p|reel|reels|video|videos|comments|watch|item|items|story|stories|article|articles|entry|thread|threads)\/[^/?#]+|\/@[^/]+\/\d+|[?&](v|id|lc|p|story_fbid)=[\w-]+/i;
+    const here = location.href.split("#")[0];
+    const permalink = (item) => {
+        const anchors = item.matches("a[href]") ? [item] : [];
+        anchors.push(...Array.from(item.querySelectorAll("a[href]")));
+        for (const a of anchors) {
+            const href = a.href;
+            if (!href || href.split("#")[0] === here)
+                continue;
+            try {
+                const u = new URL(href);
+                if (permalinkRe.test(u.pathname + u.search))
+                    return u.href.split("#")[0] ?? null;
+            }
+            catch {
+                // Ignore malformed hrefs.
+            }
+        }
+        return null;
+    };
+    const textHash = (text) => {
+        let h = 0x811c9dc5;
+        for (let i = 0; i < text.length; i++) {
+            h ^= text.charCodeAt(i);
+            h = Math.imul(h, 0x01000193) >>> 0;
+        }
+        return `text:${h.toString(16).padStart(8, "0")}`;
+    };
+    let items = [];
+    try {
+        items = Array.from(document.querySelectorAll(args.item));
+    }
+    catch {
+        return [];
+    }
+    const out = [];
+    for (const item of items) {
+        let key = args.key ? first(item, args.key.selector, args.key.attr) : null;
+        if (key && args.key?.attr && urlish.test(args.key.attr))
+            key = key.split("#")[0] ?? key;
+        if (!key)
+            key = permalink(item);
+        if (!key) {
+            const text = (item.innerText ?? "").replace(/\s+/g, " ").trim();
+            if (!text)
+                continue; // A skeleton or placeholder row: nothing to key on yet.
+            key = textHash(text);
+        }
+        const fields = {};
+        for (const f of args.fields) {
+            if (f.all) {
+                const values = [];
+                for (const el of pick(item, f.selector)) {
+                    const v = read(el, f.attr);
+                    if (v && !values.includes(v))
+                        values.push(v);
+                }
+                fields[f.name] = values;
+            }
+            else {
+                fields[f.name] = first(item, f.selector, f.attr);
+            }
+        }
+        out.push({ key, fields });
+    }
+    return out;
+}
+/** Runs in the page: scroll state of the element that actually scrolls the feed. */
+function metricsInPage(itemSelector) {
+    const doc = document.scrollingElement ?? document.documentElement;
+    let container = doc;
+    let firstItem = null;
+    try {
+        firstItem = document.querySelector(itemSelector);
+    }
+    catch {
+        firstItem = null;
+    }
+    for (let el = firstItem?.parentElement ?? null; el && el !== document.body; el = el.parentElement) {
+        const oy = getComputedStyle(el).overflowY;
+        if ((oy === "auto" || oy === "scroll") && el.scrollHeight > el.clientHeight + 1) {
+            container = el;
+            break;
+        }
+    }
+    const isDoc = container === doc;
+    const scrollTop = isDoc ? window.scrollY : container.scrollTop;
+    const viewportHeight = isDoc ? window.innerHeight : container.clientHeight;
+    const scrollHeight = container.scrollHeight;
+    let pointerX = window.innerWidth / 2;
+    let pointerY = window.innerHeight / 2;
+    const target = firstItem ?? (isDoc ? null : container);
+    if (target) {
+        const r = (isDoc ? target : container).getBoundingClientRect();
+        pointerX = Math.min(Math.max(r.left + r.width / 2, 1), window.innerWidth - 1);
+        if (!isDoc)
+            pointerY = Math.min(Math.max(r.top + r.height / 2, 1), window.innerHeight - 1);
+    }
+    return {
+        scrollTop,
+        scrollHeight,
+        viewportHeight,
+        atBottom: scrollTop + viewportHeight >= scrollHeight - 4,
+        pointerX,
+        pointerY,
+    };
+}
+export function createPageDriver(page, spec, deps) {
+    const sleep = deps.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+    const now = deps.now ?? (() => performance.now());
+    const harvestArgs = {
+        item: spec.item,
+        key: spec.key,
+        keyPattern: spec.keyPattern,
+        fields: spec.fields.map(({ name, selector, attr, all }) => ({ name, selector, attr, all })),
+    };
+    let expandEnabled = spec.expand !== null;
+    let expandSeq = 0;
+    let pointer = null;
+    const metrics = async () => await page.evaluate(metricsInPage, spec.item);
+    return {
+        async harvest() {
+            return await page.evaluate(harvestInPage, harvestArgs);
+        },
+        async metrics() {
+            const { scrollTop, scrollHeight, viewportHeight, atBottom } = await metrics();
+            return { scrollTop, scrollHeight, viewportHeight, atBottom };
+        },
+        async wheel(ticks) {
+            if (ticks.length === 0)
+                return false;
+            const start = await metrics();
+            // Rest the pointer over the feed, drifting a little between steps the
+            // way a hand does, so wheel events reach an inner scroller too.
+            const jitter = () => uniform(deps.rng, { min: -40, max: 40 });
+            const target = {
+                x: Math.max(1, start.pointerX + jitter()),
+                y: Math.max(1, start.pointerY + jitter()),
+            };
+            if (!pointer || Math.hypot(pointer.x - target.x, pointer.y - target.y) > 30) {
+                await page.mouse.move(target.x, target.y, { steps: 4 });
+                pointer = target;
+            }
+            for (const tick of ticks) {
+                await page.mouse.wheel(0, tick.deltaY);
+                await sleep(tick.gapMs);
+            }
+            const total = ticks.reduce((sum, t) => sum + t.deltaY, 0);
+            if (total <= 0)
+                return false;
+            // Wheel scrolling settles asynchronously; give it a beat before judging.
+            await sleep(120);
+            let after = await metrics();
+            if (after.scrollTop > start.scrollTop || after.atBottom)
+                return false;
+            await page.keyboard.press("PageDown").catch(() => { });
+            await sleep(150);
+            after = await metrics();
+            if (after.scrollTop > start.scrollTop || after.atBottom)
+                return true;
+            await page.evaluate(({ selector, dy }) => {
+                const first = document.querySelector(selector);
+                let el = first?.parentElement ?? null;
+                while (el && el !== document.body) {
+                    const oy = getComputedStyle(el).overflowY;
+                    if ((oy === "auto" || oy === "scroll") && el.scrollHeight > el.clientHeight + 1) {
+                        el.scrollBy(0, dy);
+                        return;
+                    }
+                    el = el.parentElement;
+                }
+                window.scrollBy(0, dy);
+            }, { selector: spec.item, dy: total });
+            return true;
+        },
+        async expand(limit) {
+            if (!expandEnabled || spec.expand === null || limit <= 0)
+                return 0;
+            const base = expandSeq;
+            const marked = await page.evaluate(({ item, expand, mark, unsafe, limit, base }) => {
+                const unsafeRe = new RegExp(`\\b(${unsafe})\\b`, "i");
+                const ids = [];
+                let seq = base;
+                for (const it of Array.from(document.querySelectorAll(item))) {
+                    for (const el of Array.from(it.querySelectorAll(expand))) {
+                        if (ids.length >= limit)
+                            return ids;
+                        if (el.hasAttribute(mark))
+                            continue;
+                        if (el.closest("a[href]"))
+                            continue; // Links navigate away.
+                        const label = `${el.getAttribute("aria-label") ?? ""} ${el.getAttribute("data-testid") ?? ""} ${el.innerText ?? ""}`;
+                        if (unsafeRe.test(label))
+                            continue;
+                        const r = el.getBoundingClientRect();
+                        if (r.width === 0 || r.height === 0 || r.bottom < 0 || r.top > window.innerHeight) {
+                            continue;
+                        }
+                        const id = String(++seq);
+                        el.setAttribute(mark, id);
+                        ids.push(id);
+                    }
+                }
+                return ids;
+            }, {
+                item: spec.item,
+                expand: spec.expand,
+                mark: EXPAND_MARK,
+                unsafe: UNSAFE_EXPAND_WORDS,
+                limit,
+                base,
+            });
+            expandSeq = base + marked.length;
+            let clicked = 0;
+            for (const id of marked) {
+                await sleep(Math.round(uniform(deps.rng, deps.expandPauseMs ?? { min: 250, max: 700 })));
+                const before = page.url();
+                try {
+                    await page.locator(`[${EXPAND_MARK}="${id}"]`).click({ timeout: 2_000 });
+                    clicked++;
+                }
+                catch {
+                    continue;
+                }
+                if (page.url() !== before) {
+                    // The button navigated: go back to the feed and stop expanding.
+                    deps.log?.(`collect: an expand click navigated to ${page.url()}; expansion disabled.`);
+                    expandEnabled = false;
+                    await page.goBack().catch(() => { });
+                    break;
+                }
+            }
+            return clicked;
+        },
+        async blocked() {
+            return await page.evaluate(({ patterns }) => {
+                const dialogs = Array.from(document.querySelectorAll('[role="dialog"], [aria-modal="true"], dialog[open]'))
+                    .map((d) => d.innerText ?? "")
+                    .join("\n");
+                const body = (document.body?.innerText ?? "").slice(0, 200_000);
+                for (const source of patterns) {
+                    const re = new RegExp(source, "i");
+                    const m = re.exec(dialogs) ?? re.exec(body);
+                    if (m)
+                        return `page shows "${m[0].slice(0, 80)}"`;
+                }
+                return null;
+            }, { patterns: [...spec.blockedText, ...GENERIC_BLOCKED_TEXT] });
+        },
+        sleep,
+        now,
+    };
+}