hazo_scrape 1.4.0 → 1.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGE_LOG.md CHANGED
@@ -1,5 +1,32 @@
1
1
  # hazo_scrape — Change Log
2
2
 
3
+ ## 1.5.2 — 2026-07-24
4
+
5
+ ### Fixed — `extractTableFromPdf` mid-word text-run splits
6
+ - A word a PDF splits across two adjacent text runs (pdfjs reports the second run beginning, within 1px, exactly where the first ended, with no whitespace at either boundary — e.g. `"Septemb"` + `"er"`) is now glued back with **no space** (`"September"`) instead of joined with a space (`"Septemb er"`). Genuinely separate tokens on the same line — which render a visible gap and/or carry a boundary space in the run text — still get their single separating space. Surfaced by BHP's historical dividend PDF, where a split month name (`"4 Septemb er 2020"`) made a record-date cell unparseable. `lineToCells` now decides spacing per-item from each text run's `x + width` and boundary whitespace rather than blindly space-joining.
7
+
8
+ ## 1.5.1 — 2026-07-24
9
+
10
+ ### Fixed — `extractTableFromPdf` multi-page row merging
11
+ - Visual lines are now clustered **per page** instead of over the concatenation of all pages' text items. Text-item y-coordinates repeat page to page (row N sits at the same y on every page), so the previous cross-page clustering collapsed each page's row N into a single garbled line — a multi-page table (e.g. BHP's 6-page historical dividend list) came out unusable. Column-band detection still spans all items (column geometry is page-independent); page order is preserved; a single-page document is unaffected.
12
+
13
+ ## 1.5.0 — 2026-07-24
14
+
15
+ ### Added — `extractTableFromPdf` (PDF text-layer → table)
16
+ - `extractTableFromPdf(data, opts?)` reconstructs a `{ headers, rows }` grid — the same `ExtractTableResult` shape `extractTable(html)` produces — from a PDF's positioned text layer, via `pdfjs-dist`. It clusters text items into visual lines by y-position and into columns by x-band, so a caller's existing `mapColumns`/`mapRows` pipeline consumes HTML and PDF tables identically.
17
+ - `opts.columnBands?: number[]` — left x-edges of each column; omit to auto-detect by x-gap (a 25px gap starts a new column).
18
+ - `opts.headerLabels?: string[]` — verbatim headers positionally aligned to the bands; when set, every clustered line is treated as a data row (for PDFs whose header spans multiple visual lines or uses columns no keyword classifier should see).
19
+ - `opts.headerRows?: number` — else merge the first N lines into headers (default 1); ignored when `headerLabels` is set.
20
+ - `opts.mergeContinuationRows?: boolean` — a line whose leftmost band is empty is folded into the previous row (per-band, space-joined) instead of becoming its own row — for PDFs that wrap a cell's overflow (e.g. a dual-currency amount) onto a second line.
21
+ - `opts.rowTolerance?: number` (default 3) and `opts.maxPages?: number` round out the tuning surface.
22
+ - New export surface: `src/pdf/` re-exported from the package root (`.`) — SERVER-ONLY (pulls in `pdfjs-dist`), never from the client-safe `./parse` entry.
23
+ - New dependency: `pdfjs-dist@^4.10.38` (imported from `pdfjs-dist/legacy/build/pdf.mjs`; no worker/font setup needed for text-only extraction).
24
+
25
+ ### Added — `fetchPdf` + `FetchOptions.binary` (raw-byte fetch)
26
+ - `fetchPdf(url, opts?)` fetches a PDF (or any binary document) as raw bytes, reusing `fetchDocument`'s robots / rate-limit / retry / proxy machinery. On a 2xx response, `result.bytes` (`Uint8Array`) holds the payload.
27
+ - `FetchOptions` gains `binary?: boolean`; `FetchResult` gains `bytes?: Uint8Array`. When `binary: true`, `fetchDocument` reads the response as bytes instead of text (`body` is `''`) and the response is never written to the on-disk cache.
28
+ - Motivation: `extractTableFromPdf` needs raw bytes, not decoded text, and a caller (e.g. `extractor_asx`) should not have to hand-roll a second fetch path to get them.
29
+
3
30
  ## 1.4.0 — 2026-07-24
4
31
 
5
32
  ### Added — `parseDate` optional `opts.extractLeading` (+ `mapRows` `opts.dateExtractLeading`)
@@ -7,4 +7,10 @@ export type { ScrapeTableOptions, TableResult } from './scrape.js';
7
7
  /** Clears all cached ProxyPools. For tests. */
8
8
  export declare function resetProxyPools(): void;
9
9
  export declare function fetchDocument(url: string, opts?: FetchOptions): Promise<FetchResult>;
10
+ /**
11
+ * Fetch a PDF (or any binary document) as raw bytes, reusing fetchDocument's
12
+ * robots / rate-limit / retry / proxy machinery. On a 2xx, `result.bytes` holds
13
+ * the payload; binary responses are never cached.
14
+ */
15
+ export declare function fetchPdf(url: string, opts?: FetchOptions): Promise<FetchResult>;
10
16
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/fetch/index.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE5D,OAAO,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,YAAY,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAOnE,+CAA+C;AAC/C,wBAAgB,eAAe,IAAI,IAAI,CAEtC;AA0GD,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,GAAE,YAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CAiN9F"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/fetch/index.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE5D,OAAO,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,YAAY,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAOnE,+CAA+C;AAC/C,wBAAgB,eAAe,IAAI,IAAI,CAEtC;AA0GD,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,GAAE,YAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CAoN9F;AAED;;;;GAIG;AACH,wBAAsB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,GAAE,YAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CAEzF"}
@@ -251,14 +251,16 @@ export async function fetchDocument(url, opts = {}) {
251
251
  }
252
252
  // Terminal: 2xx/3xx, or a terminal 4xx not in retryOn. Returned as a
253
253
  // FetchResult (ok:false for 4xx) — never thrown.
254
- const body = await res.text();
254
+ const isBinary = opts.binary === true;
255
+ const bytes = isBinary ? new Uint8Array(await res.arrayBuffer()) : undefined;
256
+ const body = isBinary ? '' : await res.text();
255
257
  const finalUrl = res.url || url;
256
258
  const responseHeaders = collectHeaders(res);
257
259
  const contentType = res.headers.get('content-type') ?? undefined;
258
260
  // [Phase 5] Only 2xx responses are cache-worthy — terminal 4xx/etc are
259
261
  // never stored. Written under the same pre-redirect `url`-derived key
260
262
  // used for the lookup in step 3.
261
- if (res.ok && cfg.cache.enabled && cacheKey) {
263
+ if (res.ok && !isBinary && cfg.cache.enabled && cacheKey) {
262
264
  await writeCache(cfg.cache, cacheKey, {
263
265
  finalUrl,
264
266
  status: res.status,
@@ -277,6 +279,7 @@ export async function fetchDocument(url, opts = {}) {
277
279
  ok: res.ok,
278
280
  headers: responseHeaders,
279
281
  body,
282
+ bytes,
280
283
  contentType,
281
284
  fromCache: false,
282
285
  attempts: attempt,
@@ -324,3 +327,11 @@ export async function fetchDocument(url, opts = {}) {
324
327
  ...(lastError !== undefined ? { cause: lastError } : {}),
325
328
  });
326
329
  }
330
+ /**
331
+ * Fetch a PDF (or any binary document) as raw bytes, reusing fetchDocument's
332
+ * robots / rate-limit / retry / proxy machinery. On a 2xx, `result.bytes` holds
333
+ * the payload; binary responses are never cached.
334
+ */
335
+ export async function fetchPdf(url, opts = {}) {
336
+ return fetchDocument(url, { ...opts, binary: true });
337
+ }
@@ -37,6 +37,8 @@ export interface FetchOptions {
37
37
  };
38
38
  /** Test seam — passed through to safeFetch's `deps.fetchImpl`. */
39
39
  fetchImpl?: typeof fetch;
40
+ /** When true, read the response as raw bytes into `FetchResult.bytes` instead of text. Not cached. */
41
+ binary?: boolean;
40
42
  }
41
43
  export interface FetchResult {
42
44
  url: string;
@@ -51,5 +53,7 @@ export interface FetchResult {
51
53
  attempts: number;
52
54
  timingMs: number;
53
55
  usedProxy?: string;
56
+ /** Raw response bytes — populated only when the request was made with `binary: true`. */
57
+ bytes?: Uint8Array;
54
58
  }
55
59
  //# sourceMappingURL=types.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/fetch/types.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAKvD,MAAM,MAAM,eAAe,GAAG,mBAAmB,CAAC;AAElD,MAAM,WAAW,YAAY;IAC3B,kFAAkF;IAClF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE;QACN,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACpB,CAAC;IACF,SAAS,CAAC,EAAE;QACV,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,KAAK,CAAC,EAAE;QACN,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,KAAK,CAAC,EAAE;QACN,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,MAAM,CAAC,EAAE;QACP,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,kEAAkE;IAClE,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;CAC1B;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,EAAE,EAAE,OAAO,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB"}
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/fetch/types.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAKvD,MAAM,MAAM,eAAe,GAAG,mBAAmB,CAAC;AAElD,MAAM,WAAW,YAAY;IAC3B,kFAAkF;IAClF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE;QACN,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACpB,CAAC;IACF,SAAS,CAAC,EAAE;QACV,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,KAAK,CAAC,EAAE;QACN,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,KAAK,CAAC,EAAE;QACN,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,MAAM,CAAC,EAAE;QACP,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,kEAAkE;IAClE,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,sGAAsG;IACtG,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,EAAE,EAAE,OAAO,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yFAAyF;IACzF,KAAK,CAAC,EAAE,UAAU,CAAC;CACpB"}
package/dist/index.d.ts CHANGED
@@ -1,3 +1,4 @@
1
1
  export * from './lib/index.js';
2
2
  export * from './fetch/index.js';
3
+ export * from './pdf/index.js';
3
4
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAQA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,kBAAkB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAQA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,kBAAkB,CAAC;AACjC,cAAc,gBAAgB,CAAC"}
package/dist/index.js CHANGED
@@ -7,3 +7,4 @@
7
7
  // client-safe, network-free entry (pure HTML parsing, no I/O).
8
8
  export * from './lib/index.js';
9
9
  export * from './fetch/index.js';
10
+ export * from './pdf/index.js';
@@ -0,0 +1,11 @@
1
+ import type { ExtractTableResult } from '../parse/extract_table.js';
2
+ export type ExtractTablePdfOptions = {
3
+ headerLabels?: string[];
4
+ headerRows?: number;
5
+ columnBands?: number[];
6
+ mergeContinuationRows?: boolean;
7
+ rowTolerance?: number;
8
+ maxPages?: number;
9
+ };
10
+ export declare function extractTableFromPdf(data: Uint8Array, opts?: ExtractTablePdfOptions): Promise<ExtractTableResult>;
11
+ //# sourceMappingURL=extract_table_pdf.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"extract_table_pdf.d.ts","sourceRoot":"","sources":["../../src/pdf/extract_table_pdf.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,MAAM,MAAM,sBAAsB,GAAG;IACnC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AA4EF,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,UAAU,EAChB,IAAI,GAAE,sBAA2B,GAChC,OAAO,CAAC,kBAAkB,CAAC,CAwD7B"}
@@ -0,0 +1,141 @@
1
+ // hazo_scrape/src/pdf/extract_table_pdf.ts — deterministic PDF text-layer → table.
2
+ //
3
+ // Reconstructs a {headers, rows} grid from a PDF's positioned text items by
4
+ // clustering items into visual lines (by y) and columns (by x-band), mirroring
5
+ // the ExtractTableResult shape of extract_table.ts so a caller's
6
+ // mapColumns/mapRows pipeline consumes HTML and PDF tables identically.
7
+ //
8
+ // SERVER-ONLY: pulls in pdfjs-dist. Exported from the package root ('.'),
9
+ // never from the client-safe './parse' entry.
10
+ import { getDocument } from 'pdfjs-dist/legacy/build/pdf.mjs';
11
+ /** Cluster items into visual lines by y within `tol`, ordered top→bottom. */
12
+ function clusterLines(items, tol) {
13
+ const sorted = [...items].sort((a, b) => b.y - a.y);
14
+ const lines = [];
15
+ for (const it of sorted) {
16
+ const line = lines.find((l) => Math.abs(l.y - it.y) <= tol);
17
+ if (line)
18
+ line.items.push(it);
19
+ else
20
+ lines.push({ y: it.y, items: [it] });
21
+ }
22
+ return lines;
23
+ }
24
+ /** Auto-detect column left-edges by cutting the sorted x-starts at the widest gaps. */
25
+ function detectBands(items) {
26
+ const xs = [...new Set(items.map((i) => Math.round(i.x)))].sort((a, b) => a - b);
27
+ if (xs.length === 0)
28
+ return [0];
29
+ const bands = [xs[0]];
30
+ for (let i = 1; i < xs.length; i++) {
31
+ if (xs[i] - xs[i - 1] > 25)
32
+ bands.push(xs[i]); // 25px gap = new column
33
+ }
34
+ return bands;
35
+ }
36
+ /** Assign x to the last band whose left-edge is <= x. */
37
+ function bandIndex(bands, x) {
38
+ let idx = 0;
39
+ for (let i = 0; i < bands.length; i++)
40
+ if (x + 3 >= bands[i])
41
+ idx = i;
42
+ return idx;
43
+ }
44
+ // A word the PDF split across two text runs reports the second run beginning
45
+ // (within GLUE_PX) exactly where the first ended, with NO whitespace at either
46
+ // run's boundary — e.g. "Septemb"|"er". A real inter-word space renders a
47
+ // visible gap (and/or a boundary space in the run text). GLUE_PX is the max
48
+ // horizontal gap still considered "same word".
49
+ const GLUE_PX = 1;
50
+ /**
51
+ * Join a band's items (sorted by x) into one cell string. Consecutive items
52
+ * that are horizontally CONTIGUOUS and carry no whitespace at their shared
53
+ * boundary are glued with no space (they are pieces of one word split across
54
+ * text runs); otherwise a single space separates them. Without this, a
55
+ * mid-word split surfaces as "Septemb er 2020" and fails downstream date/number
56
+ * parsing.
57
+ */
58
+ function joinCellItems(items) {
59
+ let out = '';
60
+ let prev = null;
61
+ for (const it of items) {
62
+ if (!it.str.trim())
63
+ continue;
64
+ if (prev != null) {
65
+ const contiguous = it.x - (prev.x + prev.w) < GLUE_PX;
66
+ const boundarySpace = /\s$/.test(prev.str) || /^\s/.test(it.str);
67
+ out += contiguous && !boundarySpace ? '' : ' ';
68
+ }
69
+ out += it.str.trim();
70
+ prev = it;
71
+ }
72
+ return out.replace(/\s+/g, ' ').trim();
73
+ }
74
+ /** Concatenate a line's items into one cell per band (items sorted by x). */
75
+ function lineToCells(line, bands) {
76
+ const cells = bands.map(() => []);
77
+ for (const it of [...line.items].sort((a, b) => a.x - b.x)) {
78
+ if (!it.str.trim())
79
+ continue;
80
+ cells[bandIndex(bands, it.x)].push(it);
81
+ }
82
+ return cells.map(joinCellItems);
83
+ }
84
+ export async function extractTableFromPdf(data, opts = {}) {
85
+ const tol = opts.rowTolerance ?? 3;
86
+ const doc = await getDocument({ data, isEvalSupported: false, useSystemFonts: false }).promise;
87
+ const pageCount = opts.maxPages ? Math.min(doc.numPages, opts.maxPages) : doc.numPages;
88
+ const pages = [];
89
+ for (let p = 1; p <= pageCount; p++) {
90
+ const page = await doc.getPage(p);
91
+ const content = await page.getTextContent();
92
+ const items = [];
93
+ for (const it of content.items) {
94
+ if (typeof it.str !== 'string' || !it.str.trim())
95
+ continue;
96
+ items.push({ x: it.transform[4], y: it.transform[5], w: it.width ?? 0, str: it.str });
97
+ }
98
+ pages.push(items);
99
+ }
100
+ // Column geometry is page-independent, so detect bands from every item; but
101
+ // cluster visual lines PER PAGE. Text-item y-coordinates repeat page to page
102
+ // (row N sits at the same y on every page), so clustering the concatenation
103
+ // would merge each page's row N into a single garbled line. Page order is
104
+ // preserved, and a single-page document behaves exactly as before.
105
+ const bands = opts.columnBands ?? detectBands(pages.flat());
106
+ let grid = [];
107
+ for (const items of pages) {
108
+ for (const line of clusterLines(items, tol))
109
+ grid.push(lineToCells(line, bands));
110
+ }
111
+ // Continuation merge: a line whose leftmost cell is empty belongs to the row above.
112
+ if (opts.mergeContinuationRows) {
113
+ const merged = [];
114
+ for (const row of grid) {
115
+ if (merged.length > 0 && row[0].trim() === '') {
116
+ const prev = merged[merged.length - 1];
117
+ for (let i = 0; i < row.length; i++) {
118
+ if (row[i].trim())
119
+ prev[i] = [prev[i], row[i]].filter(Boolean).join(' ').trim();
120
+ }
121
+ }
122
+ else {
123
+ merged.push([...row]);
124
+ }
125
+ }
126
+ grid = merged;
127
+ }
128
+ let headers;
129
+ let rows;
130
+ if (opts.headerLabels) {
131
+ headers = opts.headerLabels;
132
+ rows = grid;
133
+ }
134
+ else {
135
+ const n = Math.max(1, opts.headerRows ?? 1);
136
+ const headerLines = grid.slice(0, n);
137
+ headers = bands.map((_, c) => headerLines.map((l) => l[c] ?? '').join(' ').replace(/\s+/g, ' ').trim());
138
+ rows = grid.slice(n);
139
+ }
140
+ return { headers, rows };
141
+ }
@@ -0,0 +1,2 @@
1
+ export * from './extract_table_pdf.js';
2
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/pdf/index.ts"],"names":[],"mappings":"AACA,cAAc,wBAAwB,CAAC"}
@@ -0,0 +1,2 @@
1
+ // hazo_scrape/src/pdf/index.ts — server-only PDF table extraction surface.
2
+ export * from './extract_table_pdf.js';
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "hazo_scrape",
3
- "version": "1.4.0",
3
+ "version": "1.5.2",
4
4
  "description": "Generic source-agnostic web scraping engine with a network-free parse core.",
5
5
  "type": "module",
6
6
  "module": "./dist/index.js",
@@ -62,6 +62,7 @@
62
62
  "cheerio": "^1.2.0",
63
63
  "hazo_config": "^2.4.1",
64
64
  "hazo_secure": "^1.4.0",
65
+ "pdfjs-dist": "^4.10.38",
65
66
  "robots-parser": "^3.0.1",
66
67
  "undici": "^7.28.0"
67
68
  }