hazo_scrape 1.0.1 → 1.2.0

package/CHANGE_LOG.md

@@ -1,5 +1,24 @@
 # hazo_scrape — Change Log
 
+## 1.2.0 — 2026-07-22
+
+### Added — `mapRows` optional `opts.dateFormats`
+- `mapRows(table, map, keywords, opts?)` — new optional 4th parameter, `{ dateFormats?: string[] }`, forwarded verbatim to `parseDate`'s `opts.formats` for every date-typed cell. Purely additive: omitting `opts` preserves the exact prior behavior (a genuinely ambiguous slash date, e.g. `"09/11/2012"`, still resolves to `null` — see the pinned "never guess" test in `__tests__/map_rows.test.ts`).
+- Motivation: a caller that KNOWS a source's date convention (e.g. `extractor_asx`'s `SourceRegistryEntry.nuances.date_formats`, which existed in that consumer's schema but had nothing to plug into) can now pass a `DD/MM/YYYY`/`MM/DD/YYYY` hint through `mapRows` to resolve what would otherwise be an unresolvable ambiguity, without this engine ever guessing on its own.
+
+## 1.1.0 — 2026-07-19
+
+### Added — fetch layer (`hazo_scrape/fetch`, server-only)
+- `fetchDocument(url, opts?)` — polite/resilient HTTP built on `hazo_secure`'s `safeFetch` (SSRF guard, undici, timeout, correlation-id): retry with exponential backoff + jitter, `Retry-After` handling, per-host rate limiting, rotating browser-User-Agent pool (+ `sec-ch-ua`), full-RFC `robots.txt` gate (fail-open on robots fetch error; honors `Crawl-delay`), optional content-addressed on-disk response cache with TTL, and an optional undici `ProxyAgent` rotation pool (round-robin + failure cooldown).
+- `scrapeTable(url, keywords, opts?)` — one-call bridge chaining `fetchDocument` → `extractTable` → `mapColumns` → `mapRows`, returning `{ headers, columnMap, unmatched, rows, partial, warning?, source }`.
+- New `./fetch` subpath export; the root `.` and `./fetch` entries are **server-only** (they pull in undici / hazo_secure / node:fs / node:crypto). `./parse` remains the client-safe, network-free entry.
+- New INI config sections `[ratelimit]` and `[robots]`, plus `[http]` (`ua_pool_enabled`, `accept_language`), `[retry]` (`jitter`), and `[proxy]` (multi-URL rotation, `cooldown_ms`) additions. See `config/hazo_scrape_config.ini.sample`.
+
+### Notes
+- The content-addressed cache keys responses with a `node:crypto` SHA-256 hash (16 hex chars) over `method|url|accept-language` — no external hashing dependency.
+- Proxied requests route through undici's `ProxyAgent` and therefore bypass `safeFetch`'s SSRF connect-IP guard (accepted: operator-configured proxies).
+- The parse core (`hazo_scrape/parse`) is unchanged and remains fully network-free.
+
 ## 1.0.0 — 2026-07-17
 
 - Initial release.

package/README.md

@@ -132,6 +132,111 @@ type MappedRow = Record<string, Cell[]>;
   deliberately outside this engine. `parseNumber` and `parseDate` return the
   literal parsed value (or `null`) and nothing else.
 
+## Fetch layer (`hazo_scrape/fetch`)
+
+Everything above is `hazo_scrape/parse` — network-free and safe to bundle for
+the client. The fetch layer is the opposite: it's **server-only**. Import it
+from `hazo_scrape/fetch` (or the root `hazo_scrape` entry, which re-exports
+it) inside server code only — it pulls in `undici`, `hazo_secure`, `node:fs`,
+and `node:crypto`, and will break a client bundle if imported there.
+
+```ts
+import { scrapeTable, type ColumnKeywords } from 'hazo_scrape/fetch';
+
+const keywords: ColumnKeywords = {
+  amount: { keywords: ['amount', 'dividend'], type: 'number' },
+  exDate: { keywords: ['ex date', 'ex-dividend'], type: 'date' },
+};
+
+const result = await scrapeTable('https://example.com/dividends', keywords, {
+  dateGuard: true,
+  required: ['amount'],
+});
+
+if (result.partial) console.warn('missing required column(s):', result.unmatched);
+console.log(result.rows[0]?.amount?.[0]?.value, result.source.finalUrl);
+```
+
+### `fetchDocument(url, opts?)`
+
+Polite, resilient HTTP built on `hazo_secure`'s `safeFetch` (SSRF guard,
+undici, timeout, correlation-id). Resolves an optional INI config
+(`opts.configPath`) and overlays per-call `opts` on top before each request,
+then: checks `robots.txt` (fail-open on a robots fetch error), checks the
+on-disk cache, waits out the per-host rate limit, and runs the retry loop
+(exponential backoff + jitter, honoring `Retry-After`) with a rotating UA on
+each attempt. Returns a `FetchResult`, never throws for a terminal HTTP
+status (4xx/2xx/3xx all resolve normally) — it only throws for robots
+disallow, exhausted retries, or a non-retryable `safeFetch` error (bad URL,
+disallowed protocol/host, private-IP block).
+
+```ts
+interface FetchResult {
+  url: string;              // requested URL
+  finalUrl: string;         // URL after following redirects
+  status: number;
+  statusText: string;
+  ok: boolean;
+  headers: Record<string, string>;
+  body: string;
+  contentType?: string;
+  fromCache: boolean;
+  attempts: number;         // 0 when served from cache
+  timingMs: number;
+  usedProxy?: string;       // set when a configured proxy served this attempt
+}
+```
+
+Notable `opts` (all optional):
+
+- `retry: { maxAttempts, backoffMs, backoffFactor, jitter, retryOn }` — attempt
+  count, exponential backoff base/factor, jitter toggle, and which HTTP
+  statuses are retried (default `[429, 500, 502, 503, 504]`).
+- `rateLimit: { minIntervalMs, perHost }` — minimum spacing between requests;
+  the actual wait is `max(minIntervalMs, robots Crawl-delay)`.
+- `robots: { respect }` — set `false` to skip the robots.txt gate entirely.
+- `cache: { enabled, dir, ttlSeconds }` — content-addressed on-disk response
+  cache, off by default.
+- `proxy: { urls, username, password, rotation, cooldownMs }` — round-robin
+  `ProxyAgent` pool with failure cooldown; proxied requests bypass
+  `safeFetch`'s SSRF connect-IP guard (accepted trade-off for
+  operator-configured proxies).
+- `userAgent` / `uaPool` — pin a single UA (also disables rotation for that
+  call) or supply/override the rotation pool.
+- `fetchImpl` — test seam, passed straight through to `safeFetch`'s
+  `deps.fetchImpl`.
+
+### `scrapeTable(url, keywords, opts?)`
+
+One-call bridge from the fetch layer to the network-free parse layer:
+`fetchDocument` → `extractTable` → `mapColumns` → `mapRows`. Takes every
+`FetchOptions` field plus `select` (CSS selector passed to `extractTable`),
+`dateGuard`, and `required` (both passed straight through to `mapColumns`).
+
+```ts
+interface TableResult {
+  headers: string[];
+  columnMap: ColumnMap;
+  unmatched: string[];      // mirrors columnMap.unmatched
+  rows: MappedRow[];
+  partial: boolean;         // = !columnMap.ok
+  warning?: string;         // surfaced from extractTable, e.g. no table found
+  source: { url: string; finalUrl: string; fromCache: boolean; usedLlm: false };
+}
+```
+
+### Politeness & configuration
+
+`robots.txt` is respected by default and fails **open** (a robots.txt fetch
+error is treated as "allowed", never as "blocked"); it honors `Crawl-delay`
+when present. Requests are rate-limited per host by default. Each attempt
+rotates through a browser User-Agent pool (with matching `sec-ch-ua`) unless
+you pass `userAgent` for a single identifiable UA or set
+`ua_pool_enabled = false`. The on-disk response cache is **off** by default.
+All of this is configurable via an INI file — see
+`config/hazo_scrape_config.ini.sample` for the full set of `[http]`,
+`[retry]`, `[ratelimit]`, `[robots]`, `[cache]`, and `[proxy]` keys.
+
 ## Tailwind v4 (`@source` required)
 
 If this package renders UI, add the following to your app's CSS entry:

package/dist/fetch/cache.d.ts

@@ -0,0 +1,39 @@
+export interface CachedRecord {
+    finalUrl: string;
+    status: number;
+    statusText: string;
+    headers: Record<string, string>;
+    body: string;
+    contentType?: string;
+    storedAt: number;
+}
+interface CacheConfig {
+    enabled: boolean;
+    dir: string;
+    ttlSeconds: number;
+}
+/**
+ * Computes the content-address cache key for a request. Hashes the
+ * canonical string `${method}|${requestUrl}|${acceptLanguage}` — see the
+ * module-level comment above for why `requestUrl` is the pre-redirect URL.
+ *
+ * Returns a 16-hex-char (64-bit) truncated sha256 — ample to avoid
+ * collisions across a single host's cached pages while keeping filenames
+ * short. Kept `async` so callers need no change if the hash source ever
+ * moves back to an async implementation.
+ */
+export declare function computeCacheKey(method: string, requestUrl: string, acceptLanguage: string): Promise<string>;
+/**
+ * Reads a cached record for `key`, if present and not expired. Returns
+ * `null` on a miss, on expiry (best-effort deletes the stale file), or on
+ * any read/parse error — the cache never throws.
+ */
+export declare function readCache(cacheCfg: CacheConfig, key: string): Promise<CachedRecord | null>;
+/**
+ * Writes `record` under `key`. No-op when caching is disabled. Write
+ * failures are swallowed (with a `console.warn`) — a cache-write failure
+ * must never break an otherwise-successful fetch.
+ */
+export declare function writeCache(cacheCfg: CacheConfig, key: string, record: CachedRecord): Promise<void>;
+export {};
+//# sourceMappingURL=cache.d.ts.map

package/dist/fetch/cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../src/fetch/cache.ts"],"names":[],"mappings":"AAyBA,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,WAAW;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;GASG;AACH,wBAAsB,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAGjH;AAED;;;;GAIG;AACH,wBAAsB,SAAS,CAAC,QAAQ,EAAE,WAAW,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAuBhG;AAED;;;;GAIG;AACH,wBAAsB,UAAU,CAAC,QAAQ,EAAE,WAAW,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAWxG"}

package/dist/fetch/cache.js

@@ -0,0 +1,82 @@
+// hazo_scrape/src/fetch/cache.ts — content-addressed on-disk response cache
+//
+// Phase 5: a best-effort local cache for fetchDocument's GET responses,
+// keyed on a content-address hash of `method|requestUrl|acceptLanguage`.
+// Hashing uses Node's built-in `node:crypto` (sha256, truncated) — chosen
+// over an external hashing library so the cache path pulls in zero extra
+// dependencies and always loads cleanly. See the trade-off ledger.
+//
+// Deliberate deviation from the original plan: the plan's content address is
+// `method|finalUrl|varyHeaders`. But the cache must be CHECKED *before* the
+// network fetch happens, at which point the post-redirect `finalUrl` isn't
+// known yet — keying on it would make a pre-fetch lookup impossible. So we
+// key on the REQUESTED url (pre-redirect) for both read and write, and store
+// the actual `finalUrl` inside the cached record instead. The canonical key
+// string is `${method}|${requestUrl}|${acceptLanguage}` (acceptLanguage is
+// the single vary header tracked so far).
+//
+// The cache is best-effort: any read/write failure (missing file, corrupt
+// JSON, disk error) degrades to a miss / silent no-op rather than throwing —
+// a broken cache must never break a fetch that would otherwise succeed.
+import { readFile, writeFile, mkdir, unlink } from 'node:fs/promises';
+import { join } from 'node:path';
+import { createHash } from 'node:crypto';
+/**
+ * Computes the content-address cache key for a request. Hashes the
+ * canonical string `${method}|${requestUrl}|${acceptLanguage}` — see the
+ * module-level comment above for why `requestUrl` is the pre-redirect URL.
+ *
+ * Returns a 16-hex-char (64-bit) truncated sha256 — ample to avoid
+ * collisions across a single host's cached pages while keeping filenames
+ * short. Kept `async` so callers need no change if the hash source ever
+ * moves back to an async implementation.
+ */
+export async function computeCacheKey(method, requestUrl, acceptLanguage) {
+    const canonical = `${method}|${requestUrl}|${acceptLanguage}`;
+    return createHash('sha256').update(canonical, 'utf8').digest('hex').slice(0, 16);
+}
+/**
+ * Reads a cached record for `key`, if present and not expired. Returns
+ * `null` on a miss, on expiry (best-effort deletes the stale file), or on
+ * any read/parse error — the cache never throws.
+ */
+export async function readCache(cacheCfg, key) {
+    if (!cacheCfg.enabled)
+        return null;
+    const filePath = join(cacheCfg.dir, `${key}.json`);
+    try {
+        const raw = await readFile(filePath, 'utf8');
+        const record = JSON.parse(raw);
+        if (Date.now() - record.storedAt > cacheCfg.ttlSeconds * 1000) {
+            // Expired — best-effort cleanup, but a failure here is not fatal.
+            try {
+                await unlink(filePath);
+            }
+            catch {
+                // ignore
+            }
+            return null;
+        }
+        return record;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Writes `record` under `key`. No-op when caching is disabled. Write
+ * failures are swallowed (with a `console.warn`) — a cache-write failure
+ * must never break an otherwise-successful fetch.
+ */
+export async function writeCache(cacheCfg, key, record) {
+    if (!cacheCfg.enabled)
+        return;
+    const filePath = join(cacheCfg.dir, `${key}.json`);
+    try {
+        await mkdir(cacheCfg.dir, { recursive: true });
+        await writeFile(filePath, JSON.stringify(record), 'utf8');
+    }
+    catch (err) {
+        console.warn(`hazo_scrape: failed to write cache entry ${filePath}:`, err);
+    }
+}

package/dist/fetch/config.d.ts

@@ -0,0 +1,44 @@
+import type { ConfigProvider } from 'hazo_config';
+export interface ResolvedFetchConfig {
+    http: {
+        userAgent?: string;
+        uaPoolEnabled: boolean;
+        acceptLanguage: string;
+        accept: string;
+        timeoutMs: number;
+        maxRedirects: number;
+    };
+    retry: {
+        maxAttempts: number;
+        backoffMs: number;
+        backoffFactor: number;
+        retryOn: number[];
+        jitter: boolean;
+    };
+    rateLimit: {
+        minIntervalMs: number;
+        perHost: boolean;
+    };
+    proxy: {
+        urls: string[];
+        username?: string;
+        password?: string;
+        rotation: 'round_robin';
+        cooldownMs: number;
+    };
+    cache: {
+        enabled: boolean;
+        dir: string;
+        ttlSeconds: number;
+    };
+    robots: {
+        respect: boolean;
+        cacheTtlSeconds: number;
+    };
+}
+export declare const DEFAULT_FETCH_CONFIG: ResolvedFetchConfig;
+export declare function resolveFetchConfig(source?: {
+    configPath?: string;
+    provider?: ConfigProvider;
+}): ResolvedFetchConfig;
+//# sourceMappingURL=config.d.ts.map

package/dist/fetch/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/fetch/config.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAElD,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE;QACJ,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,aAAa,EAAE,OAAO,CAAC;QACvB,cAAc,EAAE,MAAM,CAAC;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,KAAK,EAAE;QACL,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,aAAa,EAAE,MAAM,CAAC;QACtB,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,MAAM,EAAE,OAAO,CAAC;KACjB,CAAC;IACF,SAAS,EAAE;QACT,aAAa,EAAE,MAAM,CAAC;QACtB,OAAO,EAAE,OAAO,CAAC;KAClB,CAAC;IACF,KAAK,EAAE;QACL,IAAI,EAAE,MAAM,EAAE,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,EAAE,aAAa,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,KAAK,EAAE;QACL,OAAO,EAAE,OAAO,CAAC;QACjB,GAAG,EAAE,MAAM,CAAC;QACZ,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,MAAM,EAAE;QACN,OAAO,EAAE,OAAO,CAAC;QACjB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAED,eAAO,MAAM,oBAAoB,EAAE,mBAiClC,CAAC;AAmIF,wBAAgB,kBAAkB,CAAC,MAAM,CAAC,EAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,cAAc,CAAA;CAAE,GAAG,mBAAmB,CAkBnH"}

package/dist/fetch/config.js

@@ -0,0 +1,174 @@
+// hazo_scrape/src/fetch/config.ts — typed config resolver for the fetch layer
+//
+// Reads `hazo_scrape_config.ini` (see config/hazo_scrape_config.ini.sample)
+// via `HazoConfig` from `hazo_config/server` and produces a fully-typed,
+// fully-defaulted `ResolvedFetchConfig`. This module does NO network or
+// filesystem access beyond an optional `existsSync` check and, when a
+// `configPath` is supplied, constructing a `HazoConfig` (which itself reads
+// the INI file synchronously). There is no fetch logic here — that lands in
+// a later phase.
+import { existsSync } from 'node:fs';
+import { HazoConfig } from 'hazo_config/server';
+export const DEFAULT_FETCH_CONFIG = {
+    http: {
+        uaPoolEnabled: true,
+        acceptLanguage: 'en-AU,en;q=0.9',
+        accept: 'text/html,application/xhtml+xml',
+        timeoutMs: 15000,
+        maxRedirects: 5,
+    },
+    retry: {
+        maxAttempts: 3,
+        backoffMs: 500,
+        backoffFactor: 2,
+        retryOn: [429, 500, 502, 503, 504],
+        jitter: true,
+    },
+    rateLimit: {
+        minIntervalMs: 1000,
+        perHost: true,
+    },
+    proxy: {
+        urls: [],
+        rotation: 'round_robin',
+        cooldownMs: 60000,
+    },
+    cache: {
+        enabled: false,
+        dir: '.cache/hazo_scrape',
+        ttlSeconds: 86400,
+    },
+    robots: {
+        respect: true,
+        cacheTtlSeconds: 86400,
+    },
+};
+// --- small local coercion helpers -----------------------------------------
+function bool(raw, fallback) {
+    if (raw === undefined)
+        return fallback;
+    const v = raw.trim().toLowerCase();
+    if (v === '')
+        return fallback;
+    if (v === 'true' || v === '1' || v === 'yes')
+        return true;
+    if (v === 'false' || v === '0' || v === 'no')
+        return false;
+    return fallback;
+}
+function num(raw, fallback) {
+    if (raw === undefined)
+        return fallback;
+    const trimmed = raw.trim();
+    if (trimmed === '')
+        return fallback;
+    const n = trimmed.includes('.') ? parseFloat(trimmed) : parseInt(trimmed, 10);
+    return Number.isNaN(n) ? fallback : n;
+}
+function str(raw, fallback) {
+    if (raw === undefined)
+        return fallback;
+    const trimmed = raw.trim();
+    return trimmed === '' ? fallback : raw;
+}
+function csv(raw) {
+    if (raw === undefined)
+        return [];
+    return raw
+        .split(',')
+        .map((s) => s.trim())
+        .filter((s) => s.length > 0);
+}
+function csvNums(raw) {
+    return csv(raw)
+        .map((s) => Number(s))
+        .filter((n) => !Number.isNaN(n));
+}
+// --- deep clone of the defaults (never mutate DEFAULT_FETCH_CONFIG) -------
+function cloneDefaults() {
+    const d = DEFAULT_FETCH_CONFIG;
+    return {
+        http: { ...d.http },
+        retry: { ...d.retry, retryOn: [...d.retry.retryOn] },
+        rateLimit: { ...d.rateLimit },
+        proxy: { ...d.proxy, urls: [...d.proxy.urls] },
+        cache: { ...d.cache },
+        robots: { ...d.robots },
+    };
+}
+// --- section builders -------------------------------------------------------
+function buildHttp(provider, defaults) {
+    const userAgentRaw = provider.get('http', 'user_agent');
+    const userAgent = userAgentRaw !== undefined && userAgentRaw.trim() !== '' ? userAgentRaw : undefined;
+    return {
+        ...(userAgent !== undefined ? { userAgent } : {}),
+        uaPoolEnabled: bool(provider.get('http', 'ua_pool_enabled'), defaults.uaPoolEnabled),
+        acceptLanguage: str(provider.get('http', 'accept_language'), defaults.acceptLanguage),
+        accept: str(provider.get('http', 'accept'), defaults.accept),
+        timeoutMs: num(provider.get('http', 'timeout_ms'), defaults.timeoutMs),
+        maxRedirects: num(provider.get('http', 'max_redirects'), defaults.maxRedirects),
+    };
+}
+function buildRetry(provider, defaults) {
+    const retryOnParsed = csvNums(provider.get('retry', 'retry_on'));
+    return {
+        maxAttempts: num(provider.get('retry', 'max_attempts'), defaults.maxAttempts),
+        backoffMs: num(provider.get('retry', 'backoff_ms'), defaults.backoffMs),
+        backoffFactor: num(provider.get('retry', 'backoff_factor'), defaults.backoffFactor),
+        retryOn: retryOnParsed.length > 0 ? retryOnParsed : [...defaults.retryOn],
+        jitter: bool(provider.get('retry', 'jitter'), defaults.jitter),
+    };
+}
+function buildRateLimit(provider, defaults) {
+    return {
+        minIntervalMs: num(provider.get('ratelimit', 'min_interval_ms'), defaults.minIntervalMs),
+        perHost: bool(provider.get('ratelimit', 'per_host'), defaults.perHost),
+    };
+}
+function buildProxy(provider, defaults) {
+    const urlsParsed = csv(provider.get('proxy', 'url'));
+    const usernameRaw = provider.get('proxy', 'username');
+    const passwordRaw = provider.get('proxy', 'password');
+    const username = usernameRaw !== undefined && usernameRaw.trim() !== '' ? usernameRaw : undefined;
+    const password = passwordRaw !== undefined && passwordRaw.trim() !== '' ? passwordRaw : undefined;
+    const rotationRaw = provider.get('proxy', 'rotation');
+    const rotation = rotationRaw === 'round_robin' ? 'round_robin' : defaults.rotation;
+    return {
+        urls: urlsParsed.length > 0 ? urlsParsed : [...defaults.urls],
+        ...(username !== undefined ? { username } : {}),
+        ...(password !== undefined ? { password } : {}),
+        rotation,
+        cooldownMs: num(provider.get('proxy', 'cooldown_ms'), defaults.cooldownMs),
+    };
+}
+function buildCache(provider, defaults) {
+    return {
+        enabled: bool(provider.get('cache', 'enabled'), defaults.enabled),
+        dir: str(provider.get('cache', 'dir'), defaults.dir),
+        ttlSeconds: num(provider.get('cache', 'ttl_seconds'), defaults.ttlSeconds),
+    };
+}
+function buildRobots(provider, defaults) {
+    return {
+        respect: bool(provider.get('robots', 'respect'), defaults.respect),
+        cacheTtlSeconds: num(provider.get('robots', 'cache_ttl_seconds'), defaults.cacheTtlSeconds),
+    };
+}
+// --- public resolver ---------------------------------------------------------
+export function resolveFetchConfig(source) {
+    const defaults = cloneDefaults();
+    let provider = source?.provider;
+    if (!provider && source?.configPath && existsSync(source.configPath)) {
+        provider = new HazoConfig({ filePath: source.configPath });
+    }
+    if (!provider)
+        return defaults;
+    return {
+        http: buildHttp(provider, defaults.http),
+        retry: buildRetry(provider, defaults.retry),
+        rateLimit: buildRateLimit(provider, defaults.rateLimit),
+        proxy: buildProxy(provider, defaults.proxy),
+        cache: buildCache(provider, defaults.cache),
+        robots: buildRobots(provider, defaults.robots),
+    };
+}

package/dist/fetch/headers.d.ts

@@ -0,0 +1,19 @@
+import type { ResolvedFetchConfig } from './config.js';
+import type { FetchOptions } from './types.js';
+/**
+ * A small pool of realistic, current desktop browser User-Agent strings
+ * (Chrome/Edge on Windows, Chrome on macOS/Linux, Firefox, Safari). Rotated
+ * round-robin by `buildHeaders` — never `Math.random()`, so rotation stays
+ * deterministic and testable.
+ */
+export declare const UA_POOL: string[];
+/** Resets the round-robin cursor to 0. Call between test cases for determinism. */
+export declare function resetUaRotation(): void;
+/**
+ * Builds the outgoing request headers for one fetch attempt. `attempt` is
+ * accepted for signature stability / future use (e.g. attempt-aware header
+ * variation) but does not currently affect UA selection — rotation is a
+ * pure module-level round-robin, independent of retry attempt number.
+ */
+export declare function buildHeaders(cfg: ResolvedFetchConfig, opts: FetchOptions, attempt: number): Record<string, string>;
+//# sourceMappingURL=headers.d.ts.map

package/dist/fetch/headers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"headers.d.ts","sourceRoot":"","sources":["../../src/fetch/headers.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE/C;;;;;GAKG;AACH,eAAO,MAAM,OAAO,EAAE,MAAM,EAO3B,CAAC;AAMF,mFAAmF;AACnF,wBAAgB,eAAe,IAAI,IAAI,CAEtC;AAgDD;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,mBAAmB,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAqBlH"}

package/dist/fetch/headers.js

@@ -0,0 +1,85 @@
+// hazo_scrape/src/fetch/headers.ts — UA pool + outgoing header builder
+/**
+ * A small pool of realistic, current desktop browser User-Agent strings
+ * (Chrome/Edge on Windows, Chrome on macOS/Linux, Firefox, Safari). Rotated
+ * round-robin by `buildHeaders` — never `Math.random()`, so rotation stays
+ * deterministic and testable.
+ */
+export const UA_POOL = [
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0',
+];
+// Module-level round-robin cursor. Deliberately NOT Math.random() so tests
+// can assert exact UA selection across successive calls/attempts.
+let uaRotationCounter = 0;
+/** Resets the round-robin cursor to 0. Call between test cases for determinism. */
+export function resetUaRotation() {
+    uaRotationCounter = 0;
+}
+const CHROMIUM_VERSION_RE = /Chrome\/(\d+)/;
+function isChromiumUa(ua) {
+    return CHROMIUM_VERSION_RE.test(ua);
+}
+function chromiumMajorVersion(ua) {
+    const match = CHROMIUM_VERSION_RE.exec(ua);
+    return match?.[1];
+}
+function platformFor(ua) {
+    if (ua.includes('Windows'))
+        return 'Windows';
+    if (ua.includes('Macintosh') || ua.includes('Mac OS X'))
+        return 'macOS';
+    if (ua.includes('Linux'))
+        return 'Linux';
+    return 'Unknown';
+}
+function isEdgeUa(ua) {
+    return ua.includes('Edg/');
+}
+function pickUserAgent(cfg, opts) {
+    if (opts.userAgent)
+        return opts.userAgent;
+    const poolDisabledByOpts = opts.uaPool === false;
+    const poolArrayFromOpts = Array.isArray(opts.uaPool) ? opts.uaPool : undefined;
+    const poolEnabled = poolDisabledByOpts ? false : (poolArrayFromOpts ? true : cfg.http.uaPoolEnabled);
+    if (cfg.http.userAgent && !poolEnabled) {
+        return cfg.http.userAgent;
+    }
+    if (poolEnabled) {
+        const pool = poolArrayFromOpts ?? UA_POOL;
+        const chosen = pool[uaRotationCounter % pool.length];
+        uaRotationCounter += 1;
+        return chosen;
+    }
+    // Pool disabled, no configured/caller UA — fall back to configured UA if
+    // present, else the first pool entry.
+    return cfg.http.userAgent ?? UA_POOL[0];
+}
+/**
+ * Builds the outgoing request headers for one fetch attempt. `attempt` is
+ * accepted for signature stability / future use (e.g. attempt-aware header
+ * variation) but does not currently affect UA selection — rotation is a
+ * pure module-level round-robin, independent of retry attempt number.
+ */
+export function buildHeaders(cfg, opts, attempt) {
+    void attempt;
+    const userAgent = pickUserAgent(cfg, opts);
+    const headers = {
+        'User-Agent': userAgent,
+        Accept: cfg.http.accept,
+        'Accept-Language': opts.acceptLanguage ?? cfg.http.acceptLanguage,
+    };
+    if (isChromiumUa(userAgent)) {
+        const version = chromiumMajorVersion(userAgent) ?? '131';
+        const brand = isEdgeUa(userAgent) ? 'Microsoft Edge' : 'Google Chrome';
+        headers['sec-ch-ua'] = `"Chromium";v="${version}", "${brand}";v="${version}", "Not?A_Brand";v="99"`;
+        headers['sec-ch-ua-mobile'] = '?0';
+        headers['sec-ch-ua-platform'] = `"${platformFor(userAgent)}"`;
+    }
+    // Caller-supplied headers always win.
+    return { ...headers, ...(opts.headers ?? {}) };
+}

package/dist/fetch/index.d.ts

@@ -0,0 +1,10 @@
+import type { FetchOptions, FetchResult } from './types.js';
+export { RobotsDisallowedError, resetRobotsCache } from './robots.js';
+export { resetRateLimiter } from './ratelimit.js';
+export { ProxyPool, buildProxyFetchImpl, resetProxyAgents } from './proxy.js';
+export { scrapeTable } from './scrape.js';
+export type { ScrapeTableOptions, TableResult } from './scrape.js';
+/** Clears all cached ProxyPools. For tests. */
+export declare function resetProxyPools(): void;
+export declare function fetchDocument(url: string, opts?: FetchOptions): Promise<FetchResult>;
+//# sourceMappingURL=index.d.ts.map

package/dist/fetch/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/fetch/index.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE5D,OAAO,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,YAAY,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAOnE,+CAA+C;AAC/C,wBAAgB,eAAe,IAAI,IAAI,CAEtC;AA0GD,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,GAAE,YAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CAiN9F"}