hazo_scrape 1.0.1 → 1.2.0

package/dist/fetch/scrape.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../src/fetch/scrape.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE/C,OAAO,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAE9E,MAAM,WAAW,kBAAmB,SAAQ,YAAY;IACtD,0EAA0E;IAC1E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,4EAA4E;IAC5E,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,4EAA4E;IAC5E,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,SAAS,CAAC;IACrB,iDAAiD;IACjD,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,IAAI,EAAE,SAAS,EAAE,CAAC;IAClB,kEAAkE;IAClE,OAAO,EAAE,OAAO,CAAC;IACjB,oEAAoE;IACpE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,OAAO,CAAC;QAAC,OAAO,EAAE,KAAK,CAAA;KAAE,CAAC;CAC/E;AAED,wBAAsB,WAAW,CAC/B,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,cAAc,EACxB,IAAI,GAAE,kBAAuB,GAC5B,OAAO,CAAC,WAAW,CAAC,CAqBtB"}

package/dist/fetch/scrape.js

@@ -0,0 +1,27 @@
+// hazo_scrape/src/fetch/scrape.ts — Phase 6: scrapeTable orchestrator
+//
+// Wires the fetch layer (fetchDocument) to the pure, network-free parse layer
+// (extractTable / mapColumns / mapRows). This module stays fully generic — no
+// ASX/stock/dividend-specific logic — and never calls an LLM (`usedLlm` is
+// always the literal `false` this session; a future phase may add an LLM
+// fallback path that flips this at runtime).
+import { fetchDocument } from './index.js';
+import { extractTable, mapColumns, mapRows } from '../parse/index.js';
+export async function scrapeTable(url, keywords, opts = {}) {
+    const res = await fetchDocument(url, opts);
+    const table = extractTable(res.body, opts.select ? { select: opts.select } : undefined);
+    const columnMap = mapColumns(table.headers, keywords, {
+        dateGuard: opts.dateGuard ?? false,
+        ...(opts.required ? { required: opts.required } : {}),
+    });
+    const rows = mapRows({ headers: table.headers, rows: table.rows }, columnMap, keywords);
+    return {
+        headers: table.headers,
+        columnMap,
+        unmatched: columnMap.unmatched,
+        rows,
+        partial: !columnMap.ok,
+        ...(table.warning ? { warning: table.warning } : {}),
+        source: { url, finalUrl: res.finalUrl, fromCache: res.fromCache, usedLlm: false },
+    };
+}

package/dist/fetch/types.d.ts

@@ -0,0 +1,55 @@
+import type { ResolvedFetchConfig } from './config.js';
+export type EffectiveConfig = ResolvedFetchConfig;
+export interface FetchOptions {
+    /** Optional INI config path. Omitted → defaults only (see resolveFetchConfig). */
+    configPath?: string;
+    userAgent?: string;
+    uaPool?: string[] | boolean;
+    headers?: Record<string, string>;
+    acceptLanguage?: string;
+    timeoutMs?: number;
+    maxRedirects?: number;
+    retry?: {
+        maxAttempts?: number;
+        backoffMs?: number;
+        backoffFactor?: number;
+        jitter?: boolean;
+        retryOn?: number[];
+    };
+    rateLimit?: {
+        minIntervalMs?: number;
+        perHost?: boolean;
+    };
+    proxy?: {
+        urls?: string[];
+        username?: string;
+        password?: string;
+        rotation?: 'round_robin';
+        cooldownMs?: number;
+    };
+    cache?: {
+        enabled?: boolean;
+        dir?: string;
+        ttlSeconds?: number;
+    };
+    robots?: {
+        respect?: boolean;
+    };
+    /** Test seam — passed through to safeFetch's `deps.fetchImpl`. */
+    fetchImpl?: typeof fetch;
+}
+export interface FetchResult {
+    url: string;
+    finalUrl: string;
+    status: number;
+    statusText: string;
+    ok: boolean;
+    headers: Record<string, string>;
+    body: string;
+    contentType?: string;
+    fromCache: boolean;
+    attempts: number;
+    timingMs: number;
+    usedProxy?: string;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/fetch/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/fetch/types.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAKvD,MAAM,MAAM,eAAe,GAAG,mBAAmB,CAAC;AAElD,MAAM,WAAW,YAAY;IAC3B,kFAAkF;IAClF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE;QACN,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACpB,CAAC;IACF,SAAS,CAAC,EAAE;QACV,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,KAAK,CAAC,EAAE;QACN,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,KAAK,CAAC,EAAE;QACN,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,MAAM,CAAC,EAAE;QACP,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,kEAAkE;IAClE,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;CAC1B;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,EAAE,EAAE,OAAO,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB"}

package/dist/fetch/types.js

@@ -0,0 +1,2 @@
+// hazo_scrape/src/fetch/types.ts — public + internal types for the fetch layer
+export {};

package/dist/index.d.ts

@@ -1,2 +1,3 @@
 export * from './lib/index.js';
+export * from './fetch/index.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,cAAc,gBAAgB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAQA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,kBAAkB,CAAC"}

package/dist/index.js

@@ -1,3 +1,9 @@
 // hazo_scrape/src/index.ts — Server entry point
 // Import hazo_core via peer dep: import { HazoError } from 'hazo_core';
+//
+// SERVER-ONLY: this `.` entry (and the `./fetch` subpath entry) pull in
+// undici, hazo_secure, node:fs, and node:crypto via the fetch layer below —
+// do not import either from client/browser bundles. `./parse` remains the
+// client-safe, network-free entry (pure HTML parsing, no I/O).
 export * from './lib/index.js';
+export * from './fetch/index.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hazo_scrape",
-  "version": "1.0.1",
+  "version": "1.2.0",
   "description": "Generic source-agnostic web scraping engine with a network-free parse core.",
   "type": "module",
   "module": "./dist/index.js",
@@ -14,6 +14,10 @@
       "types": "./dist/parse/index.d.ts",
       "import": "./dist/parse/index.js"
     },
+    "./fetch": {
+      "types": "./dist/fetch/index.d.ts",
+      "import": "./dist/fetch/index.js"
+    },
     "./package.json": "./package.json"
   },
   "files": [
@@ -55,6 +59,10 @@
   "author": "Pubs Abayasiri",
   "license": "MIT",
   "dependencies": {
-    "cheerio": "^1.2.0"
+    "cheerio": "^1.2.0",
+    "hazo_config": "^2.4.1",
+    "hazo_secure": "^1.4.0",
+    "robots-parser": "^3.0.1",
+    "undici": "^7.28.0"
   }
 }