hazo_scrape 1.0.1 → 1.2.0

package/dist/fetch/index.js

@@ -0,0 +1,326 @@
+// hazo_scrape/src/fetch/index.ts — polite/resilient HTTP core (fetchDocument)
+//
+// Phase 2 scope: safeFetch wrapping, retry + backoff + jitter, Retry-After
+// handling, timeout, and the UA-pool/header builder. Phase 3 adds the
+// robots.txt gate and per-host rate limiter, wired in below. Phase 4 adds
+// proxy rotation (see ./proxy.js — pure ProxyPool + undici ProxyAgent
+// wiring), also wired in below. Phase 5 adds a content-addressed on-disk
+// response cache (see ./cache.js) — the cache check runs after the robots
+// gate but BEFORE the rate-limit wait, so a cache hit never pays the
+// per-host throttle; see the `[Phase 5]` comments below for the key
+// derivation rationale.
+//
+// Phase 6 adds the scrapeTable orchestrator (see ./scrape.js — wires
+// fetchDocument to the parse layer's extractTable/mapColumns/mapRows) and
+// re-exports this whole fetch layer from `src/index.ts` plus a `./fetch`
+// package export map entry, so `hazo_scrape` and `hazo_scrape/fetch` both
+// expose fetchDocument/scrapeTable to consumers.
+import { safeFetch, SafeFetchError } from 'hazo_secure/fetch';
+import { HazoUnavailableError, HazoRateLimitError, sleep } from 'hazo_core';
+import { resolveFetchConfig } from './config.js';
+import { buildHeaders, UA_POOL } from './headers.js';
+import { checkRobots, RobotsDisallowedError } from './robots.js';
+import { rateLimitWait } from './ratelimit.js';
+import { ProxyPool, buildProxyFetchImpl } from './proxy.js';
+import { computeCacheKey, readCache, writeCache } from './cache.js';
+export { RobotsDisallowedError, resetRobotsCache } from './robots.js';
+export { resetRateLimiter } from './ratelimit.js';
+export { ProxyPool, buildProxyFetchImpl, resetProxyAgents } from './proxy.js';
+export { scrapeTable } from './scrape.js';
+// One ProxyPool per distinct (urls, cooldownMs) combination, so rotation
+// state (cursor + cooldowns) survives across calls that share the same
+// configured proxy list, but different configs don't cross-contaminate.
+const proxyPools = new Map();
+/** Clears all cached ProxyPools. For tests. */
+export function resetProxyPools() {
+    proxyPools.clear();
+}
+function getProxyPool(urls, cooldownMs) {
+    const key = `${urls.join('|')}@${cooldownMs}`;
+    const existing = proxyPools.get(key);
+    if (existing)
+        return existing;
+    const pool = new ProxyPool(urls, cooldownMs);
+    proxyPools.set(key, pool);
+    return pool;
+}
+// SafeFetchError codes that indicate the request itself is invalid/forbidden
+// and will never succeed on retry — rethrow immediately, no backoff.
+const NON_RETRYABLE_SAFE_FETCH_CODES = [
+    'invalid_url',
+    'protocol_not_allowed',
+    'host_not_allowed',
+    'private_ip_blocked',
+];
+/**
+ * Resolves defaults + optional INI config (`opts.configPath`), then overlays
+ * per-call `opts` fields on top — the caller's explicit per-call values
+ * always win over both defaults and the INI file.
+ */
+function resolveRequestConfig(opts) {
+    const base = resolveFetchConfig(opts.configPath ? { configPath: opts.configPath } : undefined);
+    const http = {
+        ...base.http,
+        ...(opts.userAgent !== undefined ? { userAgent: opts.userAgent } : {}),
+        ...(opts.acceptLanguage !== undefined ? { acceptLanguage: opts.acceptLanguage } : {}),
+        ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}),
+        ...(opts.maxRedirects !== undefined ? { maxRedirects: opts.maxRedirects } : {}),
+    };
+    const retry = {
+        ...base.retry,
+        ...(opts.retry?.maxAttempts !== undefined ? { maxAttempts: opts.retry.maxAttempts } : {}),
+        ...(opts.retry?.backoffMs !== undefined ? { backoffMs: opts.retry.backoffMs } : {}),
+        ...(opts.retry?.backoffFactor !== undefined ? { backoffFactor: opts.retry.backoffFactor } : {}),
+        ...(opts.retry?.jitter !== undefined ? { jitter: opts.retry.jitter } : {}),
+        ...(opts.retry?.retryOn !== undefined ? { retryOn: opts.retry.retryOn } : {}),
+    };
+    const rateLimit = {
+        ...base.rateLimit,
+        ...(opts.rateLimit?.minIntervalMs !== undefined ? { minIntervalMs: opts.rateLimit.minIntervalMs } : {}),
+        ...(opts.rateLimit?.perHost !== undefined ? { perHost: opts.rateLimit.perHost } : {}),
+    };
+    const proxy = {
+        ...base.proxy,
+        ...(opts.proxy?.urls !== undefined ? { urls: opts.proxy.urls } : {}),
+        ...(opts.proxy?.username !== undefined ? { username: opts.proxy.username } : {}),
+        ...(opts.proxy?.password !== undefined ? { password: opts.proxy.password } : {}),
+        ...(opts.proxy?.rotation !== undefined ? { rotation: opts.proxy.rotation } : {}),
+        ...(opts.proxy?.cooldownMs !== undefined ? { cooldownMs: opts.proxy.cooldownMs } : {}),
+    };
+    const cache = {
+        ...base.cache,
+        ...(opts.cache?.enabled !== undefined ? { enabled: opts.cache.enabled } : {}),
+        ...(opts.cache?.dir !== undefined ? { dir: opts.cache.dir } : {}),
+        ...(opts.cache?.ttlSeconds !== undefined ? { ttlSeconds: opts.cache.ttlSeconds } : {}),
+    };
+    const robots = {
+        ...base.robots,
+        ...(opts.robots?.respect !== undefined ? { respect: opts.robots.respect } : {}),
+    };
+    return { http, retry, rateLimit, proxy, cache, robots };
+}
+/** `backoffMs * backoffFactor ** (attempt - 1)`, plus optional jitter. */
+function computeBackoffDelayMs(cfg, attempt) {
+    const base = cfg.retry.backoffMs * cfg.retry.backoffFactor ** (attempt - 1);
+    if (!cfg.retry.jitter)
+        return base;
+    // Equal-jitter: keeps delay in [base, 1.5*base) so tests that disable
+    // jitter (`jitter:false`) can assert the exact base delay.
+    return base + Math.random() * base * 0.5;
+}
+/** Parses a `Retry-After` header value (delta-seconds or an HTTP-date) into ms. */
+function parseRetryAfterMs(value) {
+    if (!value)
+        return undefined;
+    const trimmed = value.trim();
+    if (/^\d+$/.test(trimmed)) {
+        return Number(trimmed) * 1000;
+    }
+    const dateMs = Date.parse(trimmed);
+    if (!Number.isNaN(dateMs)) {
+        return Math.max(0, dateMs - Date.now());
+    }
+    return undefined;
+}
+function collectHeaders(res) {
+    const out = {};
+    res.headers.forEach((value, key) => {
+        out[key] = value;
+    });
+    return out;
+}
+export async function fetchDocument(url, opts = {}) {
+    const startedAt = Date.now();
+    // 1. Resolve config: defaults + optional INI, overlaid by per-call opts.
+    const cfg = resolveRequestConfig(opts);
+    // Representative UA for robots matching + logging. Rotation still applies
+    // to the actual outgoing request headers via buildHeaders/pickUserAgent.
+    const effectiveUa = opts.userAgent ?? cfg.http.userAgent ?? UA_POOL[0];
+    // 2. Robots gate. `fetchRobotsTxt` recurses into fetchDocument with the
+    // robots gate disabled (recursion guard) and rate-limiting disabled (a
+    // robots.txt fetch must not itself wait behind the same host's throttle).
+    const robotsFetchOpts = {
+        ...(opts.fetchImpl ? { fetchImpl: opts.fetchImpl } : {}),
+        ...(opts.configPath ? { configPath: opts.configPath } : {}),
+        robots: { respect: false },
+        rateLimit: { minIntervalMs: 0 },
+    };
+    const fetchRobotsTxt = async (robotsUrl) => {
+        try {
+            const res = await fetchDocument(robotsUrl, robotsFetchOpts);
+            return { status: res.status, body: res.body };
+        }
+        catch {
+            return null;
+        }
+    };
+    const { allowed, crawlDelayMs } = await checkRobots({
+        url,
+        effectiveUa,
+        respect: cfg.robots.respect,
+        cacheTtlSeconds: cfg.robots.cacheTtlSeconds,
+        fetchRobotsTxt,
+    });
+    if (!allowed) {
+        throw new RobotsDisallowedError(url);
+    }
+    // 3. [Phase 5] Cache check — deliberately placed BEFORE the rate-limit
+    // wait (step 4) so a cache hit never pays the per-host throttle. Keyed on
+    // the pre-redirect request `url` (not the eventual `finalUrl`, which
+    // isn't known until after the network fetch) — see cache.ts for the full
+    // rationale. Only GET is cached/looked up.
+    const acceptLanguage = opts.acceptLanguage ?? cfg.http.acceptLanguage;
+    const cacheKey = cfg.cache.enabled ? await computeCacheKey('GET', url, acceptLanguage) : null;
+    if (cfg.cache.enabled && cacheKey) {
+        const hit = await readCache(cfg.cache, cacheKey);
+        if (hit) {
+            return {
+                url,
+                finalUrl: hit.finalUrl,
+                status: hit.status,
+                statusText: hit.statusText,
+                ok: hit.status >= 200 && hit.status < 300,
+                headers: hit.headers,
+                body: hit.body,
+                contentType: hit.contentType,
+                fromCache: true,
+                attempts: 0,
+                timingMs: Date.now() - startedAt,
+                usedProxy: undefined,
+            };
+        }
+    }
+    // 4. Rate-limit wait — honors the configured minimum interval, or the
+    // robots.txt Crawl-delay for this host, whichever is larger.
+    const host = new URL(url).host;
+    const effectiveIntervalMs = Math.max(cfg.rateLimit.minIntervalMs, crawlDelayMs ?? 0);
+    await rateLimitWait(host, effectiveIntervalMs, { perHost: cfg.rateLimit.perHost });
+    // 5. Retry loop.
+    const maxAttempts = Math.max(1, cfg.retry.maxAttempts);
+    let lastStatus;
+    let lastError;
+    for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+        const headers = buildHeaders(cfg, opts, attempt);
+        // Proxy dispatcher selection. An injected test `fetchImpl` always wins —
+        // we never proxy on top of a caller-supplied fetch seam.
+        let attemptFetchImpl;
+        let usedProxyUrl;
+        let proxyPool;
+        if (cfg.proxy.urls.length > 0 && !opts.fetchImpl) {
+            proxyPool = getProxyPool(cfg.proxy.urls, cfg.proxy.cooldownMs);
+            const proxyUrl = proxyPool.acquire();
+            if (proxyUrl === null) {
+                throw new HazoUnavailableError({
+                    code: 'HAZO_SCRAPE_PROXY_EXHAUSTED',
+                    pkg: 'hazo_scrape',
+                    message: `All ${cfg.proxy.urls.length} configured proxy(ies) are cooling down for ${url}`,
+                    context: { url, proxies: cfg.proxy.urls.length },
+                });
+            }
+            attemptFetchImpl = buildProxyFetchImpl(proxyUrl, { username: cfg.proxy.username, password: cfg.proxy.password });
+            usedProxyUrl = proxyUrl;
+        }
+        else {
+            attemptFetchImpl = opts.fetchImpl;
+        }
+        try {
+            const res = await safeFetch(url, {
+                policy: {
+                    timeoutMs: cfg.http.timeoutMs,
+                    maxRedirects: cfg.http.maxRedirects,
+                    blockPrivateIps: true,
+                    allowedProtocols: ['http:', 'https:'],
+                },
+                deps: attemptFetchImpl ? { fetchImpl: attemptFetchImpl } : undefined,
+                method: 'GET',
+                headers,
+            });
+            const isRetryableStatus = cfg.retry.retryOn.includes(res.status);
+            if (isRetryableStatus) {
+                lastStatus = res.status;
+                if (attempt < maxAttempts) {
+                    const retryAfterMs = parseRetryAfterMs(res.headers.get('retry-after'));
+                    const backoffDelay = computeBackoffDelayMs(cfg, attempt);
+                    const delay = retryAfterMs !== undefined ? Math.max(backoffDelay, retryAfterMs) : backoffDelay;
+                    await sleep(delay);
+                    continue;
+                }
+                // Last attempt exhausted on a retryable status — fall through to
+                // step 6, which classifies and throws.
+                break;
+            }
+            // Terminal: 2xx/3xx, or a terminal 4xx not in retryOn. Returned as a
+            // FetchResult (ok:false for 4xx) — never thrown.
+            const body = await res.text();
+            const finalUrl = res.url || url;
+            const responseHeaders = collectHeaders(res);
+            const contentType = res.headers.get('content-type') ?? undefined;
+            // [Phase 5] Only 2xx responses are cache-worthy — terminal 4xx/etc are
+            // never stored. Written under the same pre-redirect `url`-derived key
+            // used for the lookup in step 3.
+            if (res.ok && cfg.cache.enabled && cacheKey) {
+                await writeCache(cfg.cache, cacheKey, {
+                    finalUrl,
+                    status: res.status,
+                    statusText: res.statusText,
+                    headers: responseHeaders,
+                    body,
+                    contentType,
+                    storedAt: Date.now(),
+                });
+            }
+            return {
+                url,
+                finalUrl,
+                status: res.status,
+                statusText: res.statusText,
+                ok: res.ok,
+                headers: responseHeaders,
+                body,
+                contentType,
+                fromCache: false,
+                attempts: attempt,
+                timingMs: Date.now() - startedAt,
+                usedProxy: usedProxyUrl,
+            };
+        }
+        catch (err) {
+            if (err instanceof SafeFetchError) {
+                if (NON_RETRYABLE_SAFE_FETCH_CODES.includes(err.code)) {
+                    throw err;
+                }
+                // Transient (timeout, dns_failed, redirect_limit, redirect_loop):
+                // retry if attempts remain. If this attempt went through a proxy,
+                // cool it down so the next attempt rotates to a different one.
+                if (usedProxyUrl && proxyPool) {
+                    proxyPool.reportFailure(usedProxyUrl);
+                }
+                lastError = err;
+                if (attempt < maxAttempts) {
+                    const delay = computeBackoffDelayMs(cfg, attempt);
+                    await sleep(delay);
+                    continue;
+                }
+                break;
+            }
+            // Not a SafeFetchError — unexpected failure, propagate as-is.
+            throw err;
+        }
+    }
+    // 6. Retries exhausted via repeated retryable statuses or transient errors.
+    if (lastStatus === 429) {
+        throw new HazoRateLimitError({
+            code: 'HAZO_SCRAPE_RATE_LIMITED',
+            pkg: 'hazo_scrape',
+            message: `Rate limited fetching ${url} after ${maxAttempts} attempt(s)`,
+            context: { url, attempts: maxAttempts },
+        });
+    }
+    throw new HazoUnavailableError({
+        code: 'HAZO_SCRAPE_UNAVAILABLE',
+        pkg: 'hazo_scrape',
+        message: `Failed to fetch ${url} after ${maxAttempts} attempt(s)`,
+        context: { url, attempts: maxAttempts },
+        ...(lastError !== undefined ? { cause: lastError } : {}),
+    });
+}

package/dist/fetch/proxy.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Round-robin proxy rotation with per-proxy cooldowns. Pure and
+ * synchronous — no I/O, no undici — so it can be unit-tested without any
+ * real proxy or network access.
+ */
+export declare class ProxyPool {
+    private readonly urls;
+    private readonly cooldownMs;
+    private cursor;
+    private readonly cooledUntil;
+    constructor(urls: string[], cooldownMs: number);
+    /**
+     * Returns the next non-cooled proxy URL in round-robin order, advancing
+     * the cursor past it. Returns `null` if the pool is empty or every proxy
+     * is currently cooled.
+     */
+    acquire(now?: number): string | null;
+    /** Marks `url` cooled until `now + cooldownMs`; `acquire()` skips it until then. */
+    reportFailure(url: string, now?: number): void;
+    /** Resets the cursor and clears all cooldowns. For tests. */
+    reset(): void;
+}
+/**
+ * Builds a `deps.fetchImpl`-shaped function that routes through the given
+ * proxy via an undici `ProxyAgent`.
+ *
+ * TRADEOFF: this bypasses `safeFetch`'s SSRF connect-IP dispatcher —
+ * `createSecureDispatcher()` validates the resolved IP at connect time, but
+ * that dispatcher is undici-specific and proxied requests need a
+ * `ProxyAgent` dispatcher instead. There is no supported way to compose the
+ * two (a dispatcher that both proxies AND re-validates the post-DNS IP), so
+ * proxied requests intentionally lose the private-IP/DNS-rebinding guard.
+ * Accepted for Phase 4: proxy URLs are operator-configured, not
+ * user-supplied, so the SSRF surface this guard protects against isn't
+ * reachable via the proxy path.
+ */
+export declare function buildProxyFetchImpl(proxyUrl: string, auth?: {
+    username?: string;
+    password?: string;
+}): typeof fetch;
+/** Clears the cached ProxyAgents. For tests/cleanup. */
+export declare function resetProxyAgents(): void;
+//# sourceMappingURL=proxy.d.ts.map

package/dist/fetch/proxy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"proxy.d.ts","sourceRoot":"","sources":["../../src/fetch/proxy.ts"],"names":[],"mappings":"AAaA;;;;GAIG;AACH,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAW;IAChC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,MAAM,CAAK;IACnB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA6B;gBAE7C,IAAI,EAAE,MAAM,EAAE,EAAE,UAAU,EAAE,MAAM;IAK9C;;;;OAIG;IACH,OAAO,CAAC,GAAG,GAAE,MAAmB,GAAG,MAAM,GAAG,IAAI;IAkBhD,oFAAoF;IACpF,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,GAAE,MAAmB,GAAG,IAAI;IAI1D,6DAA6D;IAC7D,KAAK,IAAI,IAAI;CAId;AAsBD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,OAAO,KAAK,CASnH;AAED,wDAAwD;AACxD,wBAAgB,gBAAgB,IAAI,IAAI,CAEvC"}

package/dist/fetch/proxy.js

@@ -0,0 +1,96 @@
+// hazo_scrape/src/fetch/proxy.ts — proxy rotation pool + undici wiring
+//
+// Two halves, deliberately kept apart:
+//   1. `ProxyPool` — pure round-robin rotation + cooldown bookkeeping. No
+//      undici import, no network access. Fully unit-testable with a fake
+//      clock (the optional `now` param on every method).
+//   2. `buildProxyFetchImpl` / `resetProxyAgents` — the undici-specific
+//      wiring that turns a proxy URL into a `deps.fetchImpl`-shaped function
+//      for `safeFetch`. This is where the SSRF-bypass tradeoff lives (see
+//      comment below).
+import { ProxyAgent, fetch as undiciFetch } from 'undici';
+/**
+ * Round-robin proxy rotation with per-proxy cooldowns. Pure and
+ * synchronous — no I/O, no undici — so it can be unit-tested without any
+ * real proxy or network access.
+ */
+export class ProxyPool {
+    constructor(urls, cooldownMs) {
+        this.cursor = 0;
+        this.cooledUntil = new Map();
+        this.urls = urls;
+        this.cooldownMs = cooldownMs;
+    }
+    /**
+     * Returns the next non-cooled proxy URL in round-robin order, advancing
+     * the cursor past it. Returns `null` if the pool is empty or every proxy
+     * is currently cooled.
+     */
+    acquire(now = Date.now()) {
+        const count = this.urls.length;
+        if (count === 0)
+            return null;
+        for (let i = 0; i < count; i++) {
+            const index = (this.cursor + i) % count;
+            const url = this.urls[index];
+            const cooledUntil = this.cooledUntil.get(url);
+            if (cooledUntil === undefined || cooledUntil <= now) {
+                this.cursor = (index + 1) % count;
+                return url;
+            }
+        }
+        // All proxies cooled — leave the cursor where it is.
+        return null;
+    }
+    /** Marks `url` cooled until `now + cooldownMs`; `acquire()` skips it until then. */
+    reportFailure(url, now = Date.now()) {
+        this.cooledUntil.set(url, now + this.cooldownMs);
+    }
+    /** Resets the cursor and clears all cooldowns. For tests. */
+    reset() {
+        this.cursor = 0;
+        this.cooledUntil.clear();
+    }
+}
+// --- undici wiring -----------------------------------------------------
+// Cached ProxyAgents, keyed by proxy URL, so repeated requests through the
+// same proxy reuse one agent (and its connection pool) instead of building a
+// fresh one per attempt.
+const proxyAgents = new Map();
+function getOrCreateProxyAgent(proxyUrl, auth) {
+    const cached = proxyAgents.get(proxyUrl);
+    if (cached)
+        return cached;
+    const token = auth?.username
+        ? `Basic ${Buffer.from(`${auth.username}:${auth.password ?? ''}`).toString('base64')}`
+        : undefined;
+    const agent = new ProxyAgent(token ? { uri: proxyUrl, token } : { uri: proxyUrl });
+    proxyAgents.set(proxyUrl, agent);
+    return agent;
+}
+/**
+ * Builds a `deps.fetchImpl`-shaped function that routes through the given
+ * proxy via an undici `ProxyAgent`.
+ *
+ * TRADEOFF: this bypasses `safeFetch`'s SSRF connect-IP dispatcher —
+ * `createSecureDispatcher()` validates the resolved IP at connect time, but
+ * that dispatcher is undici-specific and proxied requests need a
+ * `ProxyAgent` dispatcher instead. There is no supported way to compose the
+ * two (a dispatcher that both proxies AND re-validates the post-DNS IP), so
+ * proxied requests intentionally lose the private-IP/DNS-rebinding guard.
+ * Accepted for Phase 4: proxy URLs are operator-configured, not
+ * user-supplied, so the SSRF surface this guard protects against isn't
+ * reachable via the proxy path.
+ */
+export function buildProxyFetchImpl(proxyUrl, auth) {
+    const dispatcher = getOrCreateProxyAgent(proxyUrl, auth);
+    // undici's `fetch`/`Response` are structurally compatible with the global
+    // (DOM lib) `fetch` type for our purposes, but their type declarations
+    // differ enough that TS won't accept a direct assignment. Bridge with a
+    // thin wrapper and a single documented `any` cast.
+    return ((input, init) => undiciFetch(input, { ...init, dispatcher }));
+}
+/** Clears the cached ProxyAgents. For tests/cleanup. */
+export function resetProxyAgents() {
+    proxyAgents.clear();
+}

package/dist/fetch/ratelimit.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Waits until a slot for `host` (or the shared `'*'` key, when
+ * `opts.perHost === false`) is available. The slot granted to this call is
+ * `max(now, key's current next-available time)`; the key's next-available
+ * time is then advanced to `slot + intervalMs`. Because both steps happen
+ * before the `await sleep(...)`, they run atomically with respect to other
+ * concurrent callers (JS has no preemption between awaits), which is what
+ * makes back-to-back/concurrent calls space out correctly instead of racing.
+ */
+export declare function rateLimitWait(host: string, intervalMs: number, opts?: {
+    perHost?: boolean;
+}): Promise<void>;
+/** Clears all recorded rate-limit state. Call between test cases for isolation. */
+export declare function resetRateLimiter(): void;
+//# sourceMappingURL=ratelimit.d.ts.map

package/dist/fetch/ratelimit.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ratelimit.d.ts","sourceRoot":"","sources":["../../src/fetch/ratelimit.ts"],"names":[],"mappings":"AAcA;;;;;;;;GAQG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,OAAO,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAYjH;AAED,mFAAmF;AACnF,wBAAgB,gBAAgB,IAAI,IAAI,CAEvC"}

package/dist/fetch/ratelimit.js

@@ -0,0 +1,35 @@
+// hazo_scrape/src/fetch/ratelimit.ts — per-host rate limiter
+//
+// Phase 3: enforces a minimum interval between requests sharing a key (the
+// target host, or a single shared key when `perHost:false`). Concurrent
+// callers for the same key are serialized into a queue — each caller's
+// granted slot is computed and reserved synchronously (before any `await`),
+// so overlapping calls never race past the same slot; N concurrent calls
+// end up spaced exactly `intervalMs` apart.
+import { sleep } from 'hazo_core';
+// Module-level map of rate-limit key -> next time (ms epoch) a slot may be granted.
+const nextAvailable = new Map();
+/**
+ * Waits until a slot for `host` (or the shared `'*'` key, when
+ * `opts.perHost === false`) is available. The slot granted to this call is
+ * `max(now, key's current next-available time)`; the key's next-available
+ * time is then advanced to `slot + intervalMs`. Because both steps happen
+ * before the `await sleep(...)`, they run atomically with respect to other
+ * concurrent callers (JS has no preemption between awaits), which is what
+ * makes back-to-back/concurrent calls space out correctly instead of racing.
+ */
+export async function rateLimitWait(host, intervalMs, opts) {
+    const key = opts?.perHost === false ? '*' : host;
+    const now = Date.now();
+    const prevNextAvailable = nextAvailable.get(key) ?? 0;
+    const slot = Math.max(now, prevNextAvailable);
+    nextAvailable.set(key, slot + intervalMs);
+    const waitMs = slot - now;
+    if (waitMs > 0) {
+        await sleep(waitMs);
+    }
+}
+/** Clears all recorded rate-limit state. Call between test cases for isolation. */
+export function resetRateLimiter() {
+    nextAvailable.clear();
+}

package/dist/fetch/robots.d.ts

@@ -0,0 +1,30 @@
+import { HazoError } from 'hazo_core';
+/** Thrown by the fetch pipeline when `checkRobots` reports `allowed:false`. */
+export declare class RobotsDisallowedError extends HazoError {
+    constructor(url: string);
+}
+export interface CheckRobotsParams {
+    url: string;
+    effectiveUa: string;
+    respect: boolean;
+    cacheTtlSeconds: number;
+    fetchRobotsTxt: (robotsUrl: string) => Promise<{
+        status: number;
+        body: string;
+    } | null>;
+}
+export interface CheckRobotsResult {
+    allowed: boolean;
+    crawlDelayMs?: number;
+}
+/**
+ * Evaluates `url` against its origin's robots.txt for `effectiveUa`. Caches
+ * per origin for `cacheTtlSeconds`. Fails open (`allowed:true`, cached
+ * permissively for the TTL) whenever the robots.txt fetch throws, returns
+ * null, or comes back with a non-2xx status — so a broken robots.txt never
+ * blocks scraping and never causes a refetch on every single request.
+ */
+export declare function checkRobots(params: CheckRobotsParams): Promise<CheckRobotsResult>;
+/** Clears the robots.txt cache. Call between test cases for isolation. */
+export declare function resetRobotsCache(): void;
+//# sourceMappingURL=robots.d.ts.map

package/dist/fetch/robots.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../../src/fetch/robots.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAWtC,+EAA+E;AAC/E,qBAAa,qBAAsB,SAAQ,SAAS;gBACtC,GAAG,EAAE,MAAM;CAQxB;AAED,MAAM,WAAW,iBAAiB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAC;CACzF;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AA8CD;;;;;;GAMG;AACH,wBAAsB,WAAW,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAsBvF;AAED,0EAA0E;AAC1E,wBAAgB,gBAAgB,IAAI,IAAI,CAEvC"}

package/dist/fetch/robots.js

@@ -0,0 +1,84 @@
+// hazo_scrape/src/fetch/robots.ts — robots.txt gate
+//
+// Phase 3: fetches and caches robots.txt per origin, evaluates whether a
+// given URL is allowed for the effective UA, and surfaces any `Crawl-delay`
+// directive so the rate limiter (ratelimit.ts) can honor it. Fails open
+// (treats the fetch as allowed) whenever robots.txt cannot be retrieved or
+// parsed — a broken/unreachable robots.txt must never brick scraping.
+import robotsParser from 'robots-parser';
+import { HazoError } from 'hazo_core';
+// Module-level cache of origin -> parsed robots.txt (+ crawl-delay + TTL).
+const cache = new Map();
+/** Thrown by the fetch pipeline when `checkRobots` reports `allowed:false`. */
+export class RobotsDisallowedError extends HazoError {
+    constructor(url) {
+        super({
+            code: 'HAZO_SCRAPE_ROBOTS_DISALLOWED',
+            pkg: 'hazo_scrape',
+            message: `Fetching disallowed by robots.txt: ${url}`,
+            context: { url },
+        });
+    }
+}
+/** A parsed-empty robots.txt: every path is allowed, no crawl-delay. */
+function permissiveEntry(robotsUrl, expiresAt) {
+    return { robot: robotsParser(robotsUrl, ''), expiresAt };
+}
+async function fetchAndCache(params) {
+    const { origin, robotsUrl, url, effectiveUa, cacheTtlSeconds, fetchRobotsTxt } = params;
+    const expiresAt = Date.now() + cacheTtlSeconds * 1000;
+    let result;
+    try {
+        result = await fetchRobotsTxt(robotsUrl);
+    }
+    catch (err) {
+        console.warn(`[hazo_scrape] robots.txt fetch failed, proceeding: ${url}`, err);
+        const entry = permissiveEntry(robotsUrl, expiresAt);
+        cache.set(origin, entry);
+        return entry;
+    }
+    if (result === null || result.status < 200 || result.status >= 300) {
+        console.warn(`[hazo_scrape] robots.txt fetch failed, proceeding: ${url}`);
+        const entry = permissiveEntry(robotsUrl, expiresAt);
+        cache.set(origin, entry);
+        return entry;
+    }
+    const robot = robotsParser(robotsUrl, result.body);
+    const crawlDelaySeconds = robot.getCrawlDelay(effectiveUa);
+    const entry = {
+        robot,
+        ...(crawlDelaySeconds !== undefined ? { crawlDelayMs: crawlDelaySeconds * 1000 } : {}),
+        expiresAt,
+    };
+    cache.set(origin, entry);
+    return entry;
+}
+/**
+ * Evaluates `url` against its origin's robots.txt for `effectiveUa`. Caches
+ * per origin for `cacheTtlSeconds`. Fails open (`allowed:true`, cached
+ * permissively for the TTL) whenever the robots.txt fetch throws, returns
+ * null, or comes back with a non-2xx status — so a broken robots.txt never
+ * blocks scraping and never causes a refetch on every single request.
+ */
+export async function checkRobots(params) {
+    const { url, effectiveUa, respect, cacheTtlSeconds, fetchRobotsTxt } = params;
+    if (!respect) {
+        return { allowed: true };
+    }
+    const origin = new URL(url).origin;
+    const robotsUrl = `${origin}/robots.txt`;
+    const now = Date.now();
+    let entry = cache.get(origin);
+    if (!entry || now >= entry.expiresAt) {
+        entry = await fetchAndCache({ origin, robotsUrl, url, effectiveUa, cacheTtlSeconds, fetchRobotsTxt });
+    }
+    // isAllowed returns boolean|undefined — undefined (no matching rule, or
+    // URL not valid for this robots.txt) is treated as allowed; only an
+    // explicit `false` disallows.
+    const allowed = entry.robot.isAllowed(url, effectiveUa) !== false;
+    return { allowed, ...(entry.crawlDelayMs !== undefined ? { crawlDelayMs: entry.crawlDelayMs } : {}) };
+}
+/** Clears the robots.txt cache. Call between test cases for isolation. */
+export function resetRobotsCache() {
+    cache.clear();
+}

package/dist/fetch/scrape.d.ts

@@ -0,0 +1,29 @@
+import type { FetchOptions } from './types.js';
+import type { ColumnKeywords, ColumnMap, MappedRow } from '../parse/index.js';
+export interface ScrapeTableOptions extends FetchOptions {
+    /** CSS selector identifying the table element, passed to extractTable. */
+    select?: string;
+    /** Passed to mapColumns — guards date-shaped headers from non-date keys. */
+    dateGuard?: boolean;
+    /** Passed to mapColumns — keys that must have >=1 candidate for ok:true. */
+    required?: string[];
+}
+export interface TableResult {
+    headers: string[];
+    columnMap: ColumnMap;
+    /** Convenience mirror of columnMap.unmatched. */
+    unmatched: string[];
+    rows: MappedRow[];
+    /** = !columnMap.ok — true when a required key found no column. */
+    partial: boolean;
+    /** Surfaced from extractTable (e.g. no table found on the page). */
+    warning?: string;
+    source: {
+        url: string;
+        finalUrl: string;
+        fromCache: boolean;
+        usedLlm: false;
+    };
+}
+export declare function scrapeTable(url: string, keywords: ColumnKeywords, opts?: ScrapeTableOptions): Promise<TableResult>;
+//# sourceMappingURL=scrape.d.ts.map