hazo_scrape 1.4.0 → 1.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGE_LOG.md +27 -0
- package/dist/fetch/index.d.ts +6 -0
- package/dist/fetch/index.d.ts.map +1 -1
- package/dist/fetch/index.js +13 -2
- package/dist/fetch/types.d.ts +4 -0
- package/dist/fetch/types.d.ts.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/pdf/extract_table_pdf.d.ts +11 -0
- package/dist/pdf/extract_table_pdf.d.ts.map +1 -0
- package/dist/pdf/extract_table_pdf.js +141 -0
- package/dist/pdf/index.d.ts +2 -0
- package/dist/pdf/index.d.ts.map +1 -0
- package/dist/pdf/index.js +2 -0
- package/package.json +2 -1
package/CHANGE_LOG.md
CHANGED
|
@@ -1,5 +1,32 @@
|
|
|
1
1
|
# hazo_scrape — Change Log
|
|
2
2
|
|
|
3
|
+
## 1.5.2 — 2026-07-24
|
|
4
|
+
|
|
5
|
+
### Fixed — `extractTableFromPdf` mid-word text-run splits
|
|
6
|
+
- A word a PDF splits across two adjacent text runs (pdfjs reports the second run beginning, within 1px, exactly where the first ended, with no whitespace at either boundary — e.g. `"Septemb"` + `"er"`) is now glued back with **no space** (`"September"`) instead of joined with a space (`"Septemb er"`). Genuinely separate tokens on the same line — which render a visible gap and/or carry a boundary space in the run text — still get their single separating space. Surfaced by BHP's historical dividend PDF, where a split month name (`"4 Septemb er 2020"`) made a record-date cell unparseable. `lineToCells` now decides spacing per-item from each text run's `x + width` and boundary whitespace rather than blindly space-joining.
|
|
7
|
+
|
|
8
|
+
## 1.5.1 — 2026-07-24
|
|
9
|
+
|
|
10
|
+
### Fixed — `extractTableFromPdf` multi-page row merging
|
|
11
|
+
- Visual lines are now clustered **per page** instead of over the concatenation of all pages' text items. Text-item y-coordinates repeat page to page (row N sits at the same y on every page), so the previous cross-page clustering collapsed each page's row N into a single garbled line — a multi-page table (e.g. BHP's 6-page historical dividend list) came out unusable. Column-band detection still spans all items (column geometry is page-independent); page order is preserved; a single-page document is unaffected.
|
|
12
|
+
|
|
13
|
+
## 1.5.0 — 2026-07-24
|
|
14
|
+
|
|
15
|
+
### Added — `extractTableFromPdf` (PDF text-layer → table)
|
|
16
|
+
- `extractTableFromPdf(data, opts?)` reconstructs a `{ headers, rows }` grid — the same `ExtractTableResult` shape `extractTable(html)` produces — from a PDF's positioned text layer, via `pdfjs-dist`. It clusters text items into visual lines by y-position and into columns by x-band, so a caller's existing `mapColumns`/`mapRows` pipeline consumes HTML and PDF tables identically.
|
|
17
|
+
- `opts.columnBands?: number[]` — left x-edges of each column; omit to auto-detect by x-gap (a 25px gap starts a new column).
|
|
18
|
+
- `opts.headerLabels?: string[]` — verbatim headers positionally aligned to the bands; when set, every clustered line is treated as a data row (for PDFs whose header spans multiple visual lines or uses columns no keyword classifier should see).
|
|
19
|
+
- `opts.headerRows?: number` — else merge the first N lines into headers (default 1); ignored when `headerLabels` is set.
|
|
20
|
+
- `opts.mergeContinuationRows?: boolean` — a line whose leftmost band is empty is folded into the previous row (per-band, space-joined) instead of becoming its own row — for PDFs that wrap a cell's overflow (e.g. a dual-currency amount) onto a second line.
|
|
21
|
+
- `opts.rowTolerance?: number` (default 3) and `opts.maxPages?: number` round out the tuning surface.
|
|
22
|
+
- New export surface: `src/pdf/` re-exported from the package root (`.`) — SERVER-ONLY (pulls in `pdfjs-dist`), never from the client-safe `./parse` entry.
|
|
23
|
+
- New dependency: `pdfjs-dist@^4.10.38` (imported from `pdfjs-dist/legacy/build/pdf.mjs`; no worker/font setup needed for text-only extraction).
|
|
24
|
+
|
|
25
|
+
### Added — `fetchPdf` + `FetchOptions.binary` (raw-byte fetch)
|
|
26
|
+
- `fetchPdf(url, opts?)` fetches a PDF (or any binary document) as raw bytes, reusing `fetchDocument`'s robots / rate-limit / retry / proxy machinery. On a 2xx response, `result.bytes` (`Uint8Array`) holds the payload.
|
|
27
|
+
- `FetchOptions` gains `binary?: boolean`; `FetchResult` gains `bytes?: Uint8Array`. When `binary: true`, `fetchDocument` reads the response as bytes instead of text (`body` is `''`) and the response is never written to the on-disk cache.
|
|
28
|
+
- Motivation: `extractTableFromPdf` needs raw bytes, not decoded text, and a caller (e.g. `extractor_asx`) should not have to hand-roll a second fetch path to get them.
|
|
29
|
+
|
|
3
30
|
## 1.4.0 — 2026-07-24
|
|
4
31
|
|
|
5
32
|
### Added — `parseDate` optional `opts.extractLeading` (+ `mapRows` `opts.dateExtractLeading`)
|
package/dist/fetch/index.d.ts
CHANGED
|
@@ -7,4 +7,10 @@ export type { ScrapeTableOptions, TableResult } from './scrape.js';
|
|
|
7
7
|
/** Clears all cached ProxyPools. For tests. */
|
|
8
8
|
export declare function resetProxyPools(): void;
|
|
9
9
|
export declare function fetchDocument(url: string, opts?: FetchOptions): Promise<FetchResult>;
|
|
10
|
+
/**
|
|
11
|
+
* Fetch a PDF (or any binary document) as raw bytes, reusing fetchDocument's
|
|
12
|
+
* robots / rate-limit / retry / proxy machinery. On a 2xx, `result.bytes` holds
|
|
13
|
+
* the payload; binary responses are never cached.
|
|
14
|
+
*/
|
|
15
|
+
export declare function fetchPdf(url: string, opts?: FetchOptions): Promise<FetchResult>;
|
|
10
16
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/fetch/index.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE5D,OAAO,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,YAAY,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAOnE,+CAA+C;AAC/C,wBAAgB,eAAe,IAAI,IAAI,CAEtC;AA0GD,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,GAAE,YAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/fetch/index.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE5D,OAAO,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,YAAY,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAOnE,+CAA+C;AAC/C,wBAAgB,eAAe,IAAI,IAAI,CAEtC;AA0GD,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,GAAE,YAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CAoN9F;AAED;;;;GAIG;AACH,wBAAsB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,GAAE,YAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CAEzF"}
|
package/dist/fetch/index.js
CHANGED
|
@@ -251,14 +251,16 @@ export async function fetchDocument(url, opts = {}) {
|
|
|
251
251
|
}
|
|
252
252
|
// Terminal: 2xx/3xx, or a terminal 4xx not in retryOn. Returned as a
|
|
253
253
|
// FetchResult (ok:false for 4xx) — never thrown.
|
|
254
|
-
const
|
|
254
|
+
const isBinary = opts.binary === true;
|
|
255
|
+
const bytes = isBinary ? new Uint8Array(await res.arrayBuffer()) : undefined;
|
|
256
|
+
const body = isBinary ? '' : await res.text();
|
|
255
257
|
const finalUrl = res.url || url;
|
|
256
258
|
const responseHeaders = collectHeaders(res);
|
|
257
259
|
const contentType = res.headers.get('content-type') ?? undefined;
|
|
258
260
|
// [Phase 5] Only 2xx responses are cache-worthy — terminal 4xx/etc are
|
|
259
261
|
// never stored. Written under the same pre-redirect `url`-derived key
|
|
260
262
|
// used for the lookup in step 3.
|
|
261
|
-
if (res.ok && cfg.cache.enabled && cacheKey) {
|
|
263
|
+
if (res.ok && !isBinary && cfg.cache.enabled && cacheKey) {
|
|
262
264
|
await writeCache(cfg.cache, cacheKey, {
|
|
263
265
|
finalUrl,
|
|
264
266
|
status: res.status,
|
|
@@ -277,6 +279,7 @@ export async function fetchDocument(url, opts = {}) {
|
|
|
277
279
|
ok: res.ok,
|
|
278
280
|
headers: responseHeaders,
|
|
279
281
|
body,
|
|
282
|
+
bytes,
|
|
280
283
|
contentType,
|
|
281
284
|
fromCache: false,
|
|
282
285
|
attempts: attempt,
|
|
@@ -324,3 +327,11 @@ export async function fetchDocument(url, opts = {}) {
|
|
|
324
327
|
...(lastError !== undefined ? { cause: lastError } : {}),
|
|
325
328
|
});
|
|
326
329
|
}
|
|
330
|
+
/**
|
|
331
|
+
* Fetch a PDF (or any binary document) as raw bytes, reusing fetchDocument's
|
|
332
|
+
* robots / rate-limit / retry / proxy machinery. On a 2xx, `result.bytes` holds
|
|
333
|
+
* the payload; binary responses are never cached.
|
|
334
|
+
*/
|
|
335
|
+
export async function fetchPdf(url, opts = {}) {
|
|
336
|
+
return fetchDocument(url, { ...opts, binary: true });
|
|
337
|
+
}
|
package/dist/fetch/types.d.ts
CHANGED
|
@@ -37,6 +37,8 @@ export interface FetchOptions {
|
|
|
37
37
|
};
|
|
38
38
|
/** Test seam — passed through to safeFetch's `deps.fetchImpl`. */
|
|
39
39
|
fetchImpl?: typeof fetch;
|
|
40
|
+
/** When true, read the response as raw bytes into `FetchResult.bytes` instead of text. Not cached. */
|
|
41
|
+
binary?: boolean;
|
|
40
42
|
}
|
|
41
43
|
export interface FetchResult {
|
|
42
44
|
url: string;
|
|
@@ -51,5 +53,7 @@ export interface FetchResult {
|
|
|
51
53
|
attempts: number;
|
|
52
54
|
timingMs: number;
|
|
53
55
|
usedProxy?: string;
|
|
56
|
+
/** Raw response bytes — populated only when the request was made with `binary: true`. */
|
|
57
|
+
bytes?: Uint8Array;
|
|
54
58
|
}
|
|
55
59
|
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/fetch/types.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAKvD,MAAM,MAAM,eAAe,GAAG,mBAAmB,CAAC;AAElD,MAAM,WAAW,YAAY;IAC3B,kFAAkF;IAClF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE;QACN,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACpB,CAAC;IACF,SAAS,CAAC,EAAE;QACV,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,KAAK,CAAC,EAAE;QACN,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,KAAK,CAAC,EAAE;QACN,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,MAAM,CAAC,EAAE;QACP,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,kEAAkE;IAClE,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/fetch/types.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAKvD,MAAM,MAAM,eAAe,GAAG,mBAAmB,CAAC;AAElD,MAAM,WAAW,YAAY;IAC3B,kFAAkF;IAClF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE;QACN,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACpB,CAAC;IACF,SAAS,CAAC,EAAE;QACV,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,KAAK,CAAC,EAAE;QACN,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,KAAK,CAAC,EAAE;QACN,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,MAAM,CAAC,EAAE;QACP,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,kEAAkE;IAClE,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,sGAAsG;IACtG,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,EAAE,EAAE,OAAO,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yFAAyF;IACzF,KAAK,CAAC,EAAE,UAAU,CAAC;CACpB"}
|
package/dist/index.d.ts
CHANGED
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAQA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,kBAAkB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAQA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,kBAAkB,CAAC;AACjC,cAAc,gBAAgB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { ExtractTableResult } from '../parse/extract_table.js';
|
|
2
|
+
export type ExtractTablePdfOptions = {
|
|
3
|
+
headerLabels?: string[];
|
|
4
|
+
headerRows?: number;
|
|
5
|
+
columnBands?: number[];
|
|
6
|
+
mergeContinuationRows?: boolean;
|
|
7
|
+
rowTolerance?: number;
|
|
8
|
+
maxPages?: number;
|
|
9
|
+
};
|
|
10
|
+
export declare function extractTableFromPdf(data: Uint8Array, opts?: ExtractTablePdfOptions): Promise<ExtractTableResult>;
|
|
11
|
+
//# sourceMappingURL=extract_table_pdf.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extract_table_pdf.d.ts","sourceRoot":"","sources":["../../src/pdf/extract_table_pdf.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,MAAM,MAAM,sBAAsB,GAAG;IACnC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AA4EF,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,UAAU,EAChB,IAAI,GAAE,sBAA2B,GAChC,OAAO,CAAC,kBAAkB,CAAC,CAwD7B"}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
// hazo_scrape/src/pdf/extract_table_pdf.ts — deterministic PDF text-layer → table.
|
|
2
|
+
//
|
|
3
|
+
// Reconstructs a {headers, rows} grid from a PDF's positioned text items by
|
|
4
|
+
// clustering items into visual lines (by y) and columns (by x-band), mirroring
|
|
5
|
+
// the ExtractTableResult shape of extract_table.ts so a caller's
|
|
6
|
+
// mapColumns/mapRows pipeline consumes HTML and PDF tables identically.
|
|
7
|
+
//
|
|
8
|
+
// SERVER-ONLY: pulls in pdfjs-dist. Exported from the package root ('.'),
|
|
9
|
+
// never from the client-safe './parse' entry.
|
|
10
|
+
import { getDocument } from 'pdfjs-dist/legacy/build/pdf.mjs';
|
|
11
|
+
/** Cluster items into visual lines by y within `tol`, ordered top→bottom. */
|
|
12
|
+
function clusterLines(items, tol) {
|
|
13
|
+
const sorted = [...items].sort((a, b) => b.y - a.y);
|
|
14
|
+
const lines = [];
|
|
15
|
+
for (const it of sorted) {
|
|
16
|
+
const line = lines.find((l) => Math.abs(l.y - it.y) <= tol);
|
|
17
|
+
if (line)
|
|
18
|
+
line.items.push(it);
|
|
19
|
+
else
|
|
20
|
+
lines.push({ y: it.y, items: [it] });
|
|
21
|
+
}
|
|
22
|
+
return lines;
|
|
23
|
+
}
|
|
24
|
+
/** Auto-detect column left-edges by cutting the sorted x-starts at the widest gaps. */
|
|
25
|
+
function detectBands(items) {
|
|
26
|
+
const xs = [...new Set(items.map((i) => Math.round(i.x)))].sort((a, b) => a - b);
|
|
27
|
+
if (xs.length === 0)
|
|
28
|
+
return [0];
|
|
29
|
+
const bands = [xs[0]];
|
|
30
|
+
for (let i = 1; i < xs.length; i++) {
|
|
31
|
+
if (xs[i] - xs[i - 1] > 25)
|
|
32
|
+
bands.push(xs[i]); // 25px gap = new column
|
|
33
|
+
}
|
|
34
|
+
return bands;
|
|
35
|
+
}
|
|
36
|
+
/** Assign x to the last band whose left-edge is <= x. */
|
|
37
|
+
function bandIndex(bands, x) {
|
|
38
|
+
let idx = 0;
|
|
39
|
+
for (let i = 0; i < bands.length; i++)
|
|
40
|
+
if (x + 3 >= bands[i])
|
|
41
|
+
idx = i;
|
|
42
|
+
return idx;
|
|
43
|
+
}
|
|
44
|
+
// A word the PDF split across two text runs reports the second run beginning
|
|
45
|
+
// (within GLUE_PX) exactly where the first ended, with NO whitespace at either
|
|
46
|
+
// run's boundary — e.g. "Septemb"|"er". A real inter-word space renders a
|
|
47
|
+
// visible gap (and/or a boundary space in the run text). GLUE_PX is the max
|
|
48
|
+
// horizontal gap still considered "same word".
|
|
49
|
+
const GLUE_PX = 1;
|
|
50
|
+
/**
|
|
51
|
+
* Join a band's items (sorted by x) into one cell string. Consecutive items
|
|
52
|
+
* that are horizontally CONTIGUOUS and carry no whitespace at their shared
|
|
53
|
+
* boundary are glued with no space (they are pieces of one word split across
|
|
54
|
+
* text runs); otherwise a single space separates them. Without this, a
|
|
55
|
+
* mid-word split surfaces as "Septemb er 2020" and fails downstream date/number
|
|
56
|
+
* parsing.
|
|
57
|
+
*/
|
|
58
|
+
function joinCellItems(items) {
|
|
59
|
+
let out = '';
|
|
60
|
+
let prev = null;
|
|
61
|
+
for (const it of items) {
|
|
62
|
+
if (!it.str.trim())
|
|
63
|
+
continue;
|
|
64
|
+
if (prev != null) {
|
|
65
|
+
const contiguous = it.x - (prev.x + prev.w) < GLUE_PX;
|
|
66
|
+
const boundarySpace = /\s$/.test(prev.str) || /^\s/.test(it.str);
|
|
67
|
+
out += contiguous && !boundarySpace ? '' : ' ';
|
|
68
|
+
}
|
|
69
|
+
out += it.str.trim();
|
|
70
|
+
prev = it;
|
|
71
|
+
}
|
|
72
|
+
return out.replace(/\s+/g, ' ').trim();
|
|
73
|
+
}
|
|
74
|
+
/** Concatenate a line's items into one cell per band (items sorted by x). */
|
|
75
|
+
function lineToCells(line, bands) {
|
|
76
|
+
const cells = bands.map(() => []);
|
|
77
|
+
for (const it of [...line.items].sort((a, b) => a.x - b.x)) {
|
|
78
|
+
if (!it.str.trim())
|
|
79
|
+
continue;
|
|
80
|
+
cells[bandIndex(bands, it.x)].push(it);
|
|
81
|
+
}
|
|
82
|
+
return cells.map(joinCellItems);
|
|
83
|
+
}
|
|
84
|
+
export async function extractTableFromPdf(data, opts = {}) {
|
|
85
|
+
const tol = opts.rowTolerance ?? 3;
|
|
86
|
+
const doc = await getDocument({ data, isEvalSupported: false, useSystemFonts: false }).promise;
|
|
87
|
+
const pageCount = opts.maxPages ? Math.min(doc.numPages, opts.maxPages) : doc.numPages;
|
|
88
|
+
const pages = [];
|
|
89
|
+
for (let p = 1; p <= pageCount; p++) {
|
|
90
|
+
const page = await doc.getPage(p);
|
|
91
|
+
const content = await page.getTextContent();
|
|
92
|
+
const items = [];
|
|
93
|
+
for (const it of content.items) {
|
|
94
|
+
if (typeof it.str !== 'string' || !it.str.trim())
|
|
95
|
+
continue;
|
|
96
|
+
items.push({ x: it.transform[4], y: it.transform[5], w: it.width ?? 0, str: it.str });
|
|
97
|
+
}
|
|
98
|
+
pages.push(items);
|
|
99
|
+
}
|
|
100
|
+
// Column geometry is page-independent, so detect bands from every item; but
|
|
101
|
+
// cluster visual lines PER PAGE. Text-item y-coordinates repeat page to page
|
|
102
|
+
// (row N sits at the same y on every page), so clustering the concatenation
|
|
103
|
+
// would merge each page's row N into a single garbled line. Page order is
|
|
104
|
+
// preserved, and a single-page document behaves exactly as before.
|
|
105
|
+
const bands = opts.columnBands ?? detectBands(pages.flat());
|
|
106
|
+
let grid = [];
|
|
107
|
+
for (const items of pages) {
|
|
108
|
+
for (const line of clusterLines(items, tol))
|
|
109
|
+
grid.push(lineToCells(line, bands));
|
|
110
|
+
}
|
|
111
|
+
// Continuation merge: a line whose leftmost cell is empty belongs to the row above.
|
|
112
|
+
if (opts.mergeContinuationRows) {
|
|
113
|
+
const merged = [];
|
|
114
|
+
for (const row of grid) {
|
|
115
|
+
if (merged.length > 0 && row[0].trim() === '') {
|
|
116
|
+
const prev = merged[merged.length - 1];
|
|
117
|
+
for (let i = 0; i < row.length; i++) {
|
|
118
|
+
if (row[i].trim())
|
|
119
|
+
prev[i] = [prev[i], row[i]].filter(Boolean).join(' ').trim();
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
else {
|
|
123
|
+
merged.push([...row]);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
grid = merged;
|
|
127
|
+
}
|
|
128
|
+
let headers;
|
|
129
|
+
let rows;
|
|
130
|
+
if (opts.headerLabels) {
|
|
131
|
+
headers = opts.headerLabels;
|
|
132
|
+
rows = grid;
|
|
133
|
+
}
|
|
134
|
+
else {
|
|
135
|
+
const n = Math.max(1, opts.headerRows ?? 1);
|
|
136
|
+
const headerLines = grid.slice(0, n);
|
|
137
|
+
headers = bands.map((_, c) => headerLines.map((l) => l[c] ?? '').join(' ').replace(/\s+/g, ' ').trim());
|
|
138
|
+
rows = grid.slice(n);
|
|
139
|
+
}
|
|
140
|
+
return { headers, rows };
|
|
141
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/pdf/index.ts"],"names":[],"mappings":"AACA,cAAc,wBAAwB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "hazo_scrape",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.5.2",
|
|
4
4
|
"description": "Generic source-agnostic web scraping engine with a network-free parse core.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"module": "./dist/index.js",
|
|
@@ -62,6 +62,7 @@
|
|
|
62
62
|
"cheerio": "^1.2.0",
|
|
63
63
|
"hazo_config": "^2.4.1",
|
|
64
64
|
"hazo_secure": "^1.4.0",
|
|
65
|
+
"pdfjs-dist": "^4.10.38",
|
|
65
66
|
"robots-parser": "^3.0.1",
|
|
66
67
|
"undici": "^7.28.0"
|
|
67
68
|
}
|