hazo_scrape 1.2.1 → 1.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGE_LOG.md +46 -0
- package/README.md +14 -3
- package/dist/fetch/index.d.ts +6 -0
- package/dist/fetch/index.d.ts.map +1 -1
- package/dist/fetch/index.js +13 -2
- package/dist/fetch/types.d.ts +4 -0
- package/dist/fetch/types.d.ts.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/parse/extract_table.d.ts +28 -0
- package/dist/parse/extract_table.d.ts.map +1 -1
- package/dist/parse/extract_table.js +87 -5
- package/dist/parse/map_rows.d.ts +1 -0
- package/dist/parse/map_rows.d.ts.map +1 -1
- package/dist/parse/map_rows.js +9 -4
- package/dist/parse/parse_date.d.ts +1 -0
- package/dist/parse/parse_date.d.ts.map +1 -1
- package/dist/parse/parse_date.js +21 -4
- package/dist/parse/parse_number.d.ts.map +1 -1
- package/dist/parse/parse_number.js +6 -0
- package/dist/pdf/extract_table_pdf.d.ts +11 -0
- package/dist/pdf/extract_table_pdf.d.ts.map +1 -0
- package/dist/pdf/extract_table_pdf.js +141 -0
- package/dist/pdf/index.d.ts +2 -0
- package/dist/pdf/index.d.ts.map +1 -0
- package/dist/pdf/index.js +2 -0
- package/package.json +2 -1
package/CHANGE_LOG.md
CHANGED
|
@@ -1,5 +1,51 @@
|
|
|
1
1
|
# hazo_scrape — Change Log
|
|
2
2
|
|
|
3
|
+
## 1.5.2 — 2026-07-24
|
|
4
|
+
|
|
5
|
+
### Fixed — `extractTableFromPdf` mid-word text-run splits
|
|
6
|
+
- A word a PDF splits across two adjacent text runs (pdfjs reports the second run beginning, within 1px, exactly where the first ended, with no whitespace at either boundary — e.g. `"Septemb"` + `"er"`) is now glued back with **no space** (`"September"`) instead of joined with a space (`"Septemb er"`). Genuinely separate tokens on the same line — which render a visible gap and/or carry a boundary space in the run text — still get their single separating space. Surfaced by BHP's historical dividend PDF, where a split month name (`"4 Septemb er 2020"`) made a record-date cell unparseable. `lineToCells` now decides spacing per-item from each text run's `x + width` and boundary whitespace rather than blindly space-joining.
|
|
7
|
+
|
|
8
|
+
## 1.5.1 — 2026-07-24
|
|
9
|
+
|
|
10
|
+
### Fixed — `extractTableFromPdf` multi-page row merging
|
|
11
|
+
- Visual lines are now clustered **per page** instead of over the concatenation of all pages' text items. Text-item y-coordinates repeat page to page (row N sits at the same y on every page), so the previous cross-page clustering collapsed each page's row N into a single garbled line — a multi-page table (e.g. BHP's 6-page historical dividend list) came out unusable. Column-band detection still spans all items (column geometry is page-independent); page order is preserved; a single-page document is unaffected.
|
|
12
|
+
|
|
13
|
+
## 1.5.0 — 2026-07-24
|
|
14
|
+
|
|
15
|
+
### Added — `extractTableFromPdf` (PDF text-layer → table)
|
|
16
|
+
- `extractTableFromPdf(data, opts?)` reconstructs a `{ headers, rows }` grid — the same `ExtractTableResult` shape `extractTable(html)` produces — from a PDF's positioned text layer, via `pdfjs-dist`. It clusters text items into visual lines by y-position and into columns by x-band, so a caller's existing `mapColumns`/`mapRows` pipeline consumes HTML and PDF tables identically.
|
|
17
|
+
- `opts.columnBands?: number[]` — left x-edges of each column; omit to auto-detect by x-gap (a 25px gap starts a new column).
|
|
18
|
+
- `opts.headerLabels?: string[]` — verbatim headers positionally aligned to the bands; when set, every clustered line is treated as a data row (for PDFs whose header spans multiple visual lines or uses columns no keyword classifier should see).
|
|
19
|
+
- `opts.headerRows?: number` — else merge the first N lines into headers (default 1); ignored when `headerLabels` is set.
|
|
20
|
+
- `opts.mergeContinuationRows?: boolean` — a line whose leftmost band is empty is folded into the previous row (per-band, space-joined) instead of becoming its own row — for PDFs that wrap a cell's overflow (e.g. a dual-currency amount) onto a second line.
|
|
21
|
+
- `opts.rowTolerance?: number` (default 3) and `opts.maxPages?: number` round out the tuning surface.
|
|
22
|
+
- New export surface: `src/pdf/` re-exported from the package root (`.`) — SERVER-ONLY (pulls in `pdfjs-dist`), never from the client-safe `./parse` entry.
|
|
23
|
+
- New dependency: `pdfjs-dist@^4.10.38` (imported from `pdfjs-dist/legacy/build/pdf.mjs`; no worker/font setup needed for text-only extraction).
|
|
24
|
+
|
|
25
|
+
### Added — `fetchPdf` + `FetchOptions.binary` (raw-byte fetch)
|
|
26
|
+
- `fetchPdf(url, opts?)` fetches a PDF (or any binary document) as raw bytes, reusing `fetchDocument`'s robots / rate-limit / retry / proxy machinery. On a 2xx response, `result.bytes` (`Uint8Array`) holds the payload.
|
|
27
|
+
- `FetchOptions` gains `binary?: boolean`; `FetchResult` gains `bytes?: Uint8Array`. When `binary: true`, `fetchDocument` reads the response as bytes instead of text (`body` is `''`) and the response is never written to the on-disk cache.
|
|
28
|
+
- Motivation: `extractTableFromPdf` needs raw bytes, not decoded text, and a caller (e.g. `extractor_asx`) should not have to hand-roll a second fetch path to get them.
|
|
29
|
+
|
|
30
|
+
## 1.4.0 — 2026-07-24
|
|
31
|
+
|
|
32
|
+
### Added — `parseDate` optional `opts.extractLeading` (+ `mapRows` `opts.dateExtractLeading`)
|
|
33
|
+
- `parseDate(raw, { extractLeading: true })` accepts a date at the START of the cell even when trailing text follows it — e.g. `parseDate('25/06/2024 - special dividend', { formats: ['DD/MM/YYYY'], extractLeading: true })` now returns `'2024-06-25'` (previously `null`, since every date pattern was `^…$`-anchored). Only the leading date token is read; trailing text is ignored, never parsed. Leading junk is still rejected (the date must be the first token), a partial year is guarded against (`'25/06/20245'` → `null`), and the date itself is still matched exactly and disambiguated by the same rules — the "never guess" contract holds. Default off; omitting it preserves the exact prior whole-string behaviour.
|
|
34
|
+
- `mapRows(table, map, keywords, { dateExtractLeading: true })` forwards this to `parseDate` for every date-typed cell (alongside the existing `dateFormats`). Purely additive.
|
|
35
|
+
- Motivation: real IR pages (e.g. WBC's dividend history) staple a label onto a date cell — `"25/06/2024 - special dividend"` — which the strict parser correctly refused, dropping the payment date to `null`. A caller that KNOWS a source does this can now opt in via `SourceRegistryEntry.nuances` without this engine ever guessing.
|
|
36
|
+
|
|
37
|
+
## 1.3.0 — 2026-07-24
|
|
38
|
+
|
|
39
|
+
### Added — `extractTable` div-grid mode (`opts.grid`)
|
|
40
|
+
- `extractTable(html, { grid })` reads a "table" built from styled `<div>`s (CSS grid/flex) instead of a real `<table>`. `GridSelector = { container?, row, cell, headerRow? }` — all plain CSS selectors. When `container` matches more than one grid, the wrapper with the most data rows wins (mirrors the existing pickBestTable "the real data grid is the biggest" bias). Motivation: IR pages like CBA's `dividend-information.html` render dividend history as `.complex-table` div-grids that a static fetch sees in markup but `extractTable` (table-only) could not read — surfacing as `0 rows × 0 cols`. Purely additive; omit `grid` for unchanged behavior. Takes precedence over `select`.
|
|
41
|
+
|
|
42
|
+
### Fixed — `extractTable` row-header alignment + repeated in-cell header labels
|
|
43
|
+
- Data-row cells are now read as `th, td` (not `td` alone), so an accessible table whose first column is a `<th scope="row">` row-label (e.g. NAB's payment-history table) stays column-aligned with its header row instead of shifting every column left by one. Pure-`<th>` section rows (zero `<td>`) are still skipped as before.
|
|
44
|
+
- A leading copy of a cell's own column header baked INTO the cell is stripped (responsive-label pattern: NAB emits `<div>Payment date</div><div>2 July 2026</div>`, so `.text()` collapses to `"Payment date2 July 2026"`). Guarded to an exact non-empty header prefix leaving a non-empty remainder, so a legitimate value is never truncated. Together these were why NAB scraped 90 rows but mapped 0.
|
|
45
|
+
|
|
46
|
+
### Fixed — `parseNumber` "cents" unit word
|
|
47
|
+
- `parseNumber('85 cents')` now returns `85` (previously `null` — `Number('85 cents')` is `NaN`). Only TOKENIZES the minor-unit word; the cents→dollars SCALING remains the caller's job via a domain `amount_unit` (unchanged, per the file's no-cents-conversion contract). Word-boundaried, so `'recent'` is untouched.
|
|
48
|
+
|
|
3
49
|
## 1.2.1 — 2026-07-23
|
|
4
50
|
|
|
5
51
|
### Fixed — republish of 1.2.0's `dist`
|
package/README.md
CHANGED
|
@@ -77,11 +77,14 @@ the first. `opts.dateGuard` drops date-looking headers from any key whose
|
|
|
77
77
|
`type !== 'date'`. `opts.required` lists keys that must have at least one
|
|
78
78
|
candidate for `map.ok` to be `true`.
|
|
79
79
|
|
|
80
|
-
### `mapRows(table: { headers; rows }, map: ColumnMap, keywords: ColumnKeywords): MappedRow[]`
|
|
80
|
+
### `mapRows(table: { headers; rows }, map: ColumnMap, keywords: ColumnKeywords, opts?: { dateFormats?: string[]; dateExtractLeading?: boolean }): MappedRow[]`
|
|
81
81
|
|
|
82
82
|
Joins a `ColumnMap` back onto `table.rows`, producing one `MappedRow` per data
|
|
83
83
|
row. Each matched key holds an array of `Cell`s (one per candidate header),
|
|
84
|
-
each parsed according to that key's declared `type`.
|
|
84
|
+
each parsed according to that key's declared `type`. `opts.dateFormats` and
|
|
85
|
+
`opts.dateExtractLeading` are forwarded verbatim to `parseDate`'s `opts.formats`
|
|
86
|
+
and `opts.extractLeading` for every date-typed cell; both default off, so
|
|
87
|
+
omitting `opts` preserves the exact prior behaviour.
|
|
85
88
|
|
|
86
89
|
### `parseNumber(raw: string): number | null`
|
|
87
90
|
|
|
@@ -90,13 +93,21 @@ and `%`, then parses what remains as a number. Returns `null` when nothing
|
|
|
90
93
|
parseable is left. Does **not** convert cents to dollars — domain scaling is
|
|
91
94
|
the caller's job.
|
|
92
95
|
|
|
93
|
-
### `parseDate(raw: string, opts?: { formats?: string[] }): string | null`
|
|
96
|
+
### `parseDate(raw: string, opts?: { formats?: string[]; extractLeading?: boolean }): string | null`
|
|
94
97
|
|
|
95
98
|
Parses a handful of common date text formats (ISO, `D Mon YYYY`, `Mon D,
|
|
96
99
|
YYYY`, and `D/M/Y` slash dates) into an ISO `yyyy-mm-dd` string. Returns
|
|
97
100
|
`null` — never a guess — for anything unrecognised or genuinely ambiguous
|
|
98
101
|
(e.g. `05/03/2026` with no `opts.formats` hint and both components `<= 12`).
|
|
99
102
|
|
|
103
|
+
`opts.extractLeading` (default off) accepts a date at the **start** of the cell
|
|
104
|
+
even when trailing text follows it — `parseDate('25/06/2024 - special dividend',
|
|
105
|
+
{ formats: ['DD/MM/YYYY'], extractLeading: true })` → `'2024-06-25'`. Only the
|
|
106
|
+
leading date token is read; the trailing text is ignored, never parsed. Leading
|
|
107
|
+
junk is still rejected (the date must be the first token) and the date itself is
|
|
108
|
+
still matched exactly and disambiguated by the same rules — so the "never guess"
|
|
109
|
+
contract holds. For real IR pages that staple a label onto a date cell.
|
|
110
|
+
|
|
100
111
|
### Types
|
|
101
112
|
|
|
102
113
|
```ts
|
package/dist/fetch/index.d.ts
CHANGED
|
@@ -7,4 +7,10 @@ export type { ScrapeTableOptions, TableResult } from './scrape.js';
|
|
|
7
7
|
/** Clears all cached ProxyPools. For tests. */
|
|
8
8
|
export declare function resetProxyPools(): void;
|
|
9
9
|
export declare function fetchDocument(url: string, opts?: FetchOptions): Promise<FetchResult>;
|
|
10
|
+
/**
|
|
11
|
+
* Fetch a PDF (or any binary document) as raw bytes, reusing fetchDocument's
|
|
12
|
+
* robots / rate-limit / retry / proxy machinery. On a 2xx, `result.bytes` holds
|
|
13
|
+
* the payload; binary responses are never cached.
|
|
14
|
+
*/
|
|
15
|
+
export declare function fetchPdf(url: string, opts?: FetchOptions): Promise<FetchResult>;
|
|
10
16
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/fetch/index.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE5D,OAAO,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,YAAY,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAOnE,+CAA+C;AAC/C,wBAAgB,eAAe,IAAI,IAAI,CAEtC;AA0GD,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,GAAE,YAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/fetch/index.ts"],"names":[],"mappings":"AA0BA,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE5D,OAAO,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAC9E,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,YAAY,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAOnE,+CAA+C;AAC/C,wBAAgB,eAAe,IAAI,IAAI,CAEtC;AA0GD,wBAAsB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,GAAE,YAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CAoN9F;AAED;;;;GAIG;AACH,wBAAsB,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,GAAE,YAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CAEzF"}
|
package/dist/fetch/index.js
CHANGED
|
@@ -251,14 +251,16 @@ export async function fetchDocument(url, opts = {}) {
|
|
|
251
251
|
}
|
|
252
252
|
// Terminal: 2xx/3xx, or a terminal 4xx not in retryOn. Returned as a
|
|
253
253
|
// FetchResult (ok:false for 4xx) — never thrown.
|
|
254
|
-
const
|
|
254
|
+
const isBinary = opts.binary === true;
|
|
255
|
+
const bytes = isBinary ? new Uint8Array(await res.arrayBuffer()) : undefined;
|
|
256
|
+
const body = isBinary ? '' : await res.text();
|
|
255
257
|
const finalUrl = res.url || url;
|
|
256
258
|
const responseHeaders = collectHeaders(res);
|
|
257
259
|
const contentType = res.headers.get('content-type') ?? undefined;
|
|
258
260
|
// [Phase 5] Only 2xx responses are cache-worthy — terminal 4xx/etc are
|
|
259
261
|
// never stored. Written under the same pre-redirect `url`-derived key
|
|
260
262
|
// used for the lookup in step 3.
|
|
261
|
-
if (res.ok && cfg.cache.enabled && cacheKey) {
|
|
263
|
+
if (res.ok && !isBinary && cfg.cache.enabled && cacheKey) {
|
|
262
264
|
await writeCache(cfg.cache, cacheKey, {
|
|
263
265
|
finalUrl,
|
|
264
266
|
status: res.status,
|
|
@@ -277,6 +279,7 @@ export async function fetchDocument(url, opts = {}) {
|
|
|
277
279
|
ok: res.ok,
|
|
278
280
|
headers: responseHeaders,
|
|
279
281
|
body,
|
|
282
|
+
bytes,
|
|
280
283
|
contentType,
|
|
281
284
|
fromCache: false,
|
|
282
285
|
attempts: attempt,
|
|
@@ -324,3 +327,11 @@ export async function fetchDocument(url, opts = {}) {
|
|
|
324
327
|
...(lastError !== undefined ? { cause: lastError } : {}),
|
|
325
328
|
});
|
|
326
329
|
}
|
|
330
|
+
/**
|
|
331
|
+
* Fetch a PDF (or any binary document) as raw bytes, reusing fetchDocument's
|
|
332
|
+
* robots / rate-limit / retry / proxy machinery. On a 2xx, `result.bytes` holds
|
|
333
|
+
* the payload; binary responses are never cached.
|
|
334
|
+
*/
|
|
335
|
+
export async function fetchPdf(url, opts = {}) {
|
|
336
|
+
return fetchDocument(url, { ...opts, binary: true });
|
|
337
|
+
}
|
package/dist/fetch/types.d.ts
CHANGED
|
@@ -37,6 +37,8 @@ export interface FetchOptions {
|
|
|
37
37
|
};
|
|
38
38
|
/** Test seam — passed through to safeFetch's `deps.fetchImpl`. */
|
|
39
39
|
fetchImpl?: typeof fetch;
|
|
40
|
+
/** When true, read the response as raw bytes into `FetchResult.bytes` instead of text. Not cached. */
|
|
41
|
+
binary?: boolean;
|
|
40
42
|
}
|
|
41
43
|
export interface FetchResult {
|
|
42
44
|
url: string;
|
|
@@ -51,5 +53,7 @@ export interface FetchResult {
|
|
|
51
53
|
attempts: number;
|
|
52
54
|
timingMs: number;
|
|
53
55
|
usedProxy?: string;
|
|
56
|
+
/** Raw response bytes — populated only when the request was made with `binary: true`. */
|
|
57
|
+
bytes?: Uint8Array;
|
|
54
58
|
}
|
|
55
59
|
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/fetch/types.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAKvD,MAAM,MAAM,eAAe,GAAG,mBAAmB,CAAC;AAElD,MAAM,WAAW,YAAY;IAC3B,kFAAkF;IAClF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE;QACN,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACpB,CAAC;IACF,SAAS,CAAC,EAAE;QACV,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,KAAK,CAAC,EAAE;QACN,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,KAAK,CAAC,EAAE;QACN,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,MAAM,CAAC,EAAE;QACP,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,kEAAkE;IAClE,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/fetch/types.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAKvD,MAAM,MAAM,eAAe,GAAG,mBAAmB,CAAC;AAElD,MAAM,WAAW,YAAY;IAC3B,kFAAkF;IAClF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,KAAK,CAAC,EAAE;QACN,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;KACpB,CAAC;IACF,SAAS,CAAC,EAAE;QACV,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,KAAK,CAAC,EAAE;QACN,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,aAAa,CAAC;QACzB,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,KAAK,CAAC,EAAE;QACN,OAAO,CAAC,EAAE,OAAO,CAAC;QAClB,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,UAAU,CAAC,EAAE,MAAM,CAAC;KACrB,CAAC;IACF,MAAM,CAAC,EAAE;QACP,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB,CAAC;IACF,kEAAkE;IAClE,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,sGAAsG;IACtG,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,EAAE,EAAE,OAAO,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yFAAyF;IACzF,KAAK,CAAC,EAAE,UAAU,CAAC;CACpB"}
|
package/dist/index.d.ts
CHANGED
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAQA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,kBAAkB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAQA,cAAc,gBAAgB,CAAC;AAC/B,cAAc,kBAAkB,CAAC;AACjC,cAAc,gBAAgB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -3,8 +3,36 @@ export type ExtractTableResult = {
|
|
|
3
3
|
rows: string[][];
|
|
4
4
|
warning?: string;
|
|
5
5
|
};
|
|
6
|
+
/**
|
|
7
|
+
* Div-grid mode selectors — for "tables" built from styled <div>s (CSS
|
|
8
|
+
* grid/flex) instead of a real <table>. A static HTML fetch sees these in
|
|
9
|
+
* the markup even when the page's data table proper is JS-rendered (e.g.
|
|
10
|
+
* CBA's `.complex-table`). All selectors are plain CSS.
|
|
11
|
+
*/
|
|
12
|
+
export type GridSelector = {
|
|
13
|
+
/**
|
|
14
|
+
* Element(s) that wrap one grid. When the selector matches MORE than one,
|
|
15
|
+
* the wrapper holding the most data rows wins — mirroring pickBestTable's
|
|
16
|
+
* "the real data grid is the biggest one" bias, so an upcoming-dividend
|
|
17
|
+
* teaser grid never shadows the full history grid. Omit to search the
|
|
18
|
+
* whole document.
|
|
19
|
+
*/
|
|
20
|
+
container?: string;
|
|
21
|
+
/** Selector (searched under the container) matching every row — header and data alike. */
|
|
22
|
+
row: string;
|
|
23
|
+
/** Selector (searched under a row) matching each cell. */
|
|
24
|
+
cell: string;
|
|
25
|
+
/**
|
|
26
|
+
* Selector identifying the header row among the `row` matches. Rows that
|
|
27
|
+
* match it are treated as headers (never data); the first such row supplies
|
|
28
|
+
* the column headers. Omit to treat the first row as the header.
|
|
29
|
+
*/
|
|
30
|
+
headerRow?: string;
|
|
31
|
+
};
|
|
6
32
|
export type ExtractTableOptions = {
|
|
7
33
|
select?: string;
|
|
34
|
+
/** Read a <div>-based pseudo-table instead of a real <table>. Takes precedence over `select`. */
|
|
35
|
+
grid?: GridSelector;
|
|
8
36
|
};
|
|
9
37
|
export declare function extractTable(html: string, opts?: ExtractTableOptions): ExtractTableResult;
|
|
10
38
|
//# sourceMappingURL=extract_table.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extract_table.d.ts","sourceRoot":"","sources":["../../src/parse/extract_table.ts"],"names":[],"mappings":"AAsBA,MAAM,MAAM,kBAAkB,GAAG;IAC/B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"extract_table.d.ts","sourceRoot":"","sources":["../../src/parse/extract_table.ts"],"names":[],"mappings":"AAsBA,MAAM,MAAM,kBAAkB,GAAG;IAC/B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF;;;;;GAKG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0FAA0F;IAC1F,GAAG,EAAE,MAAM,CAAC;IACZ,0DAA0D;IAC1D,IAAI,EAAE,MAAM,CAAC;IACb;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,iGAAiG;IACjG,IAAI,CAAC,EAAE,YAAY,CAAC;CACrB,CAAC;AAwLF,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,mBAAmB,GAAG,kBAAkB,CA6BzF"}
|
|
@@ -80,17 +80,96 @@ function readTable(tableSel) {
|
|
|
80
80
|
const dataRowsSel = rows.slice(1);
|
|
81
81
|
const dataRows = [];
|
|
82
82
|
for (let i = 0; i < dataRowsSel.length; i++) {
|
|
83
|
-
const
|
|
84
|
-
|
|
85
|
-
|
|
83
|
+
const rowSel = dataRowsSel.eq(i);
|
|
84
|
+
// A genuine data row has at least one <td>; a pure-<th> row (a section
|
|
85
|
+
// divider spanning the table) is skipped — prototype behaviour. But the
|
|
86
|
+
// cells themselves are then read as `th, td`, not `td` alone, so a
|
|
87
|
+
// row-header cell (<th scope="row">, common in accessible IR tables
|
|
88
|
+
// whose first column labels the row) is kept and the row stays
|
|
89
|
+
// column-aligned with the header row (which is likewise read as th,td).
|
|
90
|
+
if (rowSel.children('td').length === 0)
|
|
91
|
+
continue;
|
|
92
|
+
const cellEls = rowSel.children('th, td');
|
|
86
93
|
const cells = [];
|
|
87
|
-
for (let j = 0; j <
|
|
88
|
-
|
|
94
|
+
for (let j = 0; j < cellEls.length; j++) {
|
|
95
|
+
let text = normalizeCell(cellEls.eq(j).text());
|
|
96
|
+
// Responsive-table pattern: some IR pages repeat the column header
|
|
97
|
+
// INSIDE each cell as a (visually-hidden on desktop) label, e.g. NAB's
|
|
98
|
+
// "<div>Payment date</div><div>2 July 2026</div>", whose .text()
|
|
99
|
+
// collapses to "Payment date2 July 2026". Strip a leading copy of this
|
|
100
|
+
// cell's own column header so downstream number/date parsing sees just
|
|
101
|
+
// the value. Guarded to an exact non-empty header prefix that leaves a
|
|
102
|
+
// non-empty remainder, so a legitimate value is never truncated.
|
|
103
|
+
const header = headers[j];
|
|
104
|
+
if (header != null && header !== '' && text.length > header.length && text.startsWith(header)) {
|
|
105
|
+
text = text.slice(header.length).trim();
|
|
106
|
+
}
|
|
107
|
+
cells.push(text);
|
|
89
108
|
}
|
|
90
109
|
dataRows.push(cells);
|
|
91
110
|
}
|
|
92
111
|
return { headers, rows: dataRows };
|
|
93
112
|
}
|
|
113
|
+
// Read one cell's normalized text list from a row selection, using a cell
|
|
114
|
+
// sub-selector (div-grid mode — see GridSelector).
|
|
115
|
+
function gridRowCells(rowSel, cellSelector) {
|
|
116
|
+
const cellEls = rowSel.find(cellSelector);
|
|
117
|
+
const cells = [];
|
|
118
|
+
for (let j = 0; j < cellEls.length; j++)
|
|
119
|
+
cells.push(normalizeCell(cellEls.eq(j).text()));
|
|
120
|
+
return cells;
|
|
121
|
+
}
|
|
122
|
+
// Read a single div-grid wrapper into the same {headers, rows} shape a real
|
|
123
|
+
// <table> yields, so every downstream stage (mapColumns/mapRows) is oblivious
|
|
124
|
+
// to which markup the data came from.
|
|
125
|
+
function readGridContainer(container, grid) {
|
|
126
|
+
const allRows = container.find(grid.row);
|
|
127
|
+
if (allRows.length === 0)
|
|
128
|
+
return { headers: [], rows: [] };
|
|
129
|
+
let headers = [];
|
|
130
|
+
let firstRowIsHeader = false;
|
|
131
|
+
if (grid.headerRow != null) {
|
|
132
|
+
const headerRows = allRows.filter(grid.headerRow);
|
|
133
|
+
if (headerRows.length > 0)
|
|
134
|
+
headers = gridRowCells(headerRows.eq(0), grid.cell);
|
|
135
|
+
}
|
|
136
|
+
if (headers.length === 0) {
|
|
137
|
+
headers = gridRowCells(allRows.eq(0), grid.cell);
|
|
138
|
+
firstRowIsHeader = true;
|
|
139
|
+
}
|
|
140
|
+
const dataRows = [];
|
|
141
|
+
for (let i = 0; i < allRows.length; i++) {
|
|
142
|
+
const rowSel = allRows.eq(i);
|
|
143
|
+
if (grid.headerRow != null && rowSel.is(grid.headerRow))
|
|
144
|
+
continue; // never treat a header row as data
|
|
145
|
+
if (grid.headerRow == null && firstRowIsHeader && i === 0)
|
|
146
|
+
continue; // first row already consumed as header
|
|
147
|
+
const cells = gridRowCells(rowSel, grid.cell);
|
|
148
|
+
if (cells.length === 0)
|
|
149
|
+
continue;
|
|
150
|
+
dataRows.push(cells);
|
|
151
|
+
}
|
|
152
|
+
return { headers, rows: dataRows };
|
|
153
|
+
}
|
|
154
|
+
function readGrid($, grid) {
|
|
155
|
+
const containers = grid.container != null ? $(grid.container) : $.root();
|
|
156
|
+
if (grid.container != null && containers.length === 0) {
|
|
157
|
+
return { headers: [], rows: [], warning: `The selector "${grid.container}" matched no element on the page.` };
|
|
158
|
+
}
|
|
159
|
+
// When the container selector matches several grids, keep the one with the
|
|
160
|
+
// most data rows (same "the real data is the biggest grid" heuristic
|
|
161
|
+
// pickBestTable uses for real tables). Ties keep the first encountered.
|
|
162
|
+
let best = null;
|
|
163
|
+
for (let i = 0; i < containers.length; i++) {
|
|
164
|
+
const parsed = readGridContainer(containers.eq(i), grid);
|
|
165
|
+
if (best === null || parsed.rows.length > best.rows.length)
|
|
166
|
+
best = parsed;
|
|
167
|
+
}
|
|
168
|
+
if (best === null || (best.headers.length === 0 && best.rows.length === 0)) {
|
|
169
|
+
return { headers: [], rows: [], warning: NO_TABLE_WARNING };
|
|
170
|
+
}
|
|
171
|
+
return best;
|
|
172
|
+
}
|
|
94
173
|
function pickBestTable($) {
|
|
95
174
|
const tables = $('table');
|
|
96
175
|
let best = null;
|
|
@@ -111,6 +190,9 @@ function pickBestTable($) {
|
|
|
111
190
|
}
|
|
112
191
|
export function extractTable(html, opts) {
|
|
113
192
|
const $ = load(html);
|
|
193
|
+
if (opts?.grid != null) {
|
|
194
|
+
return readGrid($, opts.grid);
|
|
195
|
+
}
|
|
114
196
|
if (opts?.select != null) {
|
|
115
197
|
const matched = $(opts.select);
|
|
116
198
|
if (matched.length === 0) {
|
package/dist/parse/map_rows.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"map_rows.d.ts","sourceRoot":"","sources":["../../src/parse/map_rows.ts"],"names":[],"mappings":"AA2BA,OAAO,KAAK,EAAQ,cAAc,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAI7E,MAAM,WAAW,cAAc;IAC7B,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"map_rows.d.ts","sourceRoot":"","sources":["../../src/parse/map_rows.ts"],"names":[],"mappings":"AA2BA,OAAO,KAAK,EAAQ,cAAc,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAI7E,MAAM,WAAW,cAAc;IAC7B,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAMvB,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAmBD,wBAAgB,OAAO,CACrB,KAAK,EAAE;IAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAA;CAAE,EAC9C,GAAG,EAAE,SAAS,EACd,QAAQ,EAAE,cAAc,EACxB,IAAI,CAAC,EAAE,cAAc,GACpB,SAAS,EAAE,CAyBb"}
|
package/dist/parse/map_rows.js
CHANGED
|
@@ -26,17 +26,22 @@
|
|
|
26
26
|
// the exact prior behavior — this is purely additive.
|
|
27
27
|
import { parseNumber } from './parse_number.js';
|
|
28
28
|
import { parseDate } from './parse_date.js';
|
|
29
|
-
function cellValue(type, raw, dateFormats) {
|
|
29
|
+
function cellValue(type, raw, dateFormats, dateExtractLeading) {
|
|
30
30
|
if (type === 'number')
|
|
31
31
|
return parseNumber(raw);
|
|
32
|
-
if (type === 'date')
|
|
33
|
-
return parseDate(raw,
|
|
32
|
+
if (type === 'date') {
|
|
33
|
+
return parseDate(raw, {
|
|
34
|
+
...(dateFormats ? { formats: dateFormats } : {}),
|
|
35
|
+
...(dateExtractLeading ? { extractLeading: true } : {}),
|
|
36
|
+
});
|
|
37
|
+
}
|
|
34
38
|
const trimmed = raw.trim();
|
|
35
39
|
return trimmed === '' ? null : trimmed;
|
|
36
40
|
}
|
|
37
41
|
export function mapRows(table, map, keywords, opts) {
|
|
38
42
|
const keys = Object.keys(map.matched);
|
|
39
43
|
const dateFormats = opts?.dateFormats;
|
|
44
|
+
const dateExtractLeading = opts?.dateExtractLeading;
|
|
40
45
|
return table.rows.map((row) => {
|
|
41
46
|
const mappedRow = {};
|
|
42
47
|
for (const key of keys) {
|
|
@@ -48,7 +53,7 @@ export function mapRows(table, map, keywords, opts) {
|
|
|
48
53
|
header: candidate.header,
|
|
49
54
|
index: candidate.index,
|
|
50
55
|
raw,
|
|
51
|
-
value: cellValue(type, raw, dateFormats),
|
|
56
|
+
value: cellValue(type, raw, dateFormats, dateExtractLeading),
|
|
52
57
|
};
|
|
53
58
|
});
|
|
54
59
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parse_date.d.ts","sourceRoot":"","sources":["../../src/parse/parse_date.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"parse_date.d.ts","sourceRoot":"","sources":["../../src/parse/parse_date.ts"],"names":[],"mappings":"AAuFA,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;IAAE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,cAAc,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,MAAM,GAAG,IAAI,CA4C7G"}
|
package/dist/parse/parse_date.js
CHANGED
|
@@ -8,6 +8,16 @@
|
|
|
8
8
|
// blindly assuming day-first. Slash dates are only resolved without a
|
|
9
9
|
// hint when one ordering is calendar-impossible (a component > 12), or
|
|
10
10
|
// when the caller supplies `opts.formats` to disambiguate.
|
|
11
|
+
//
|
|
12
|
+
// opts.extractLeading (added 2026-07-24): OFF by default. When true, a date
|
|
13
|
+
// at the START of the cell is accepted even if trailing text follows it
|
|
14
|
+
// (e.g. "25/06/2024 - special dividend" -> "2024-06-25"). Only the leading
|
|
15
|
+
// date token is read; the trailing text is ignored, never parsed. Leading
|
|
16
|
+
// junk is still rejected — the date must be the first token. This preserves
|
|
17
|
+
// the "never guess" contract (the date itself is still matched exactly and
|
|
18
|
+
// still disambiguated by the same rules) while tolerating real IR pages that
|
|
19
|
+
// staple a label onto a date cell. Omitting it preserves exact prior
|
|
20
|
+
// behaviour (a whole-string anchored match) — this is purely additive.
|
|
11
21
|
const MONTHS = {
|
|
12
22
|
jan: 1,
|
|
13
23
|
feb: 2,
|
|
@@ -80,13 +90,20 @@ export function parseDate(raw, opts) {
|
|
|
80
90
|
const s = String(raw).trim();
|
|
81
91
|
if (s === '')
|
|
82
92
|
return null;
|
|
93
|
+
// End-of-token anchor. Strict mode (default) anchors to end-of-string, so
|
|
94
|
+
// any trailing text fails the match. `extractLeading` relaxes that to "no
|
|
95
|
+
// further word char or slash follows the date" — enough to end the date
|
|
96
|
+
// token cleanly (a following space, hyphen-with-space, punctuation, or EOS)
|
|
97
|
+
// without letting a partial number ("2024" out of "20245") slip through.
|
|
98
|
+
const tail = opts?.extractLeading ? '(?![\\w/])' : '$';
|
|
99
|
+
const re = (body) => new RegExp(`^${body}${tail}`);
|
|
83
100
|
// ISO: yyyy-mm-dd
|
|
84
|
-
let m = s.match(
|
|
101
|
+
let m = s.match(re('(\\d{4})-(\\d{2})-(\\d{2})'));
|
|
85
102
|
if (m) {
|
|
86
103
|
return toIsoIfValid(Number(m[1]), Number(m[2]), Number(m[3]));
|
|
87
104
|
}
|
|
88
105
|
// "D Mon YYYY" / "D Month YYYY", e.g. "5 Mar 2026", "18 November 2025"
|
|
89
|
-
m = s.match(
|
|
106
|
+
m = s.match(re('(\\d{1,2})\\s+([A-Za-z]{3,})\\s+(\\d{4})'));
|
|
90
107
|
if (m) {
|
|
91
108
|
const month = monthFromName(m[2]);
|
|
92
109
|
if (month == null)
|
|
@@ -94,7 +111,7 @@ export function parseDate(raw, opts) {
|
|
|
94
111
|
return toIsoIfValid(Number(m[3]), month, Number(m[1]));
|
|
95
112
|
}
|
|
96
113
|
// "Mon D, YYYY" / "Month D YYYY", comma optional, e.g. "Mar 5, 2026"
|
|
97
|
-
m = s.match(
|
|
114
|
+
m = s.match(re('([A-Za-z]{3,})\\s+(\\d{1,2}),?\\s+(\\d{4})'));
|
|
98
115
|
if (m) {
|
|
99
116
|
const month = monthFromName(m[1]);
|
|
100
117
|
if (month == null)
|
|
@@ -104,7 +121,7 @@ export function parseDate(raw, opts) {
|
|
|
104
121
|
// Slash dates: D/M/Y or M/D/Y — ambiguous without a hint or a proof by
|
|
105
122
|
// calendar impossibility (see resolveSlashDate). Two-digit years are
|
|
106
123
|
// intentionally rejected: the year group below requires exactly 4 digits.
|
|
107
|
-
m = s.match(
|
|
124
|
+
m = s.match(re('(\\d{1,2})\\/(\\d{1,2})\\/(\\d{4})'));
|
|
108
125
|
if (m) {
|
|
109
126
|
return resolveSlashDate(Number(m[1]), Number(m[2]), Number(m[3]), opts?.formats);
|
|
110
127
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parse_number.d.ts","sourceRoot":"","sources":["../../src/parse/parse_number.ts"],"names":[],"mappings":"AAaA,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,
|
|
1
|
+
{"version":3,"file":"parse_number.d.ts","sourceRoot":"","sources":["../../src/parse/parse_number.ts"],"names":[],"mappings":"AAaA,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CA4BtD"}
|
|
@@ -23,6 +23,12 @@ export function parseNumber(raw) {
|
|
|
23
23
|
// Strip thousands separators and percent signs.
|
|
24
24
|
s = s.replace(/,/g, '');
|
|
25
25
|
s = s.replace(/%/g, '');
|
|
26
|
+
// Strip a trailing minor-unit word — some IR pages write an amount as
|
|
27
|
+
// "85 cents". This only TOKENIZES ("85 cents" -> "85"); the cents->dollars
|
|
28
|
+
// SCALING remains the caller's job via a domain amount_unit (see the file
|
|
29
|
+
// header's no-cents-conversion note). Word-boundaried so it never touches
|
|
30
|
+
// the digits or an unrelated word like "recent".
|
|
31
|
+
s = s.replace(/\bcents?\b/gi, '');
|
|
26
32
|
s = s.trim();
|
|
27
33
|
if (s === '')
|
|
28
34
|
return null;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { ExtractTableResult } from '../parse/extract_table.js';
|
|
2
|
+
export type ExtractTablePdfOptions = {
|
|
3
|
+
headerLabels?: string[];
|
|
4
|
+
headerRows?: number;
|
|
5
|
+
columnBands?: number[];
|
|
6
|
+
mergeContinuationRows?: boolean;
|
|
7
|
+
rowTolerance?: number;
|
|
8
|
+
maxPages?: number;
|
|
9
|
+
};
|
|
10
|
+
export declare function extractTableFromPdf(data: Uint8Array, opts?: ExtractTablePdfOptions): Promise<ExtractTableResult>;
|
|
11
|
+
//# sourceMappingURL=extract_table_pdf.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extract_table_pdf.d.ts","sourceRoot":"","sources":["../../src/pdf/extract_table_pdf.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,MAAM,MAAM,sBAAsB,GAAG;IACnC,YAAY,CAAC,EAAE,MAAM,EAAE,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AA4EF,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,UAAU,EAChB,IAAI,GAAE,sBAA2B,GAChC,OAAO,CAAC,kBAAkB,CAAC,CAwD7B"}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
// hazo_scrape/src/pdf/extract_table_pdf.ts — deterministic PDF text-layer → table.
|
|
2
|
+
//
|
|
3
|
+
// Reconstructs a {headers, rows} grid from a PDF's positioned text items by
|
|
4
|
+
// clustering items into visual lines (by y) and columns (by x-band), mirroring
|
|
5
|
+
// the ExtractTableResult shape of extract_table.ts so a caller's
|
|
6
|
+
// mapColumns/mapRows pipeline consumes HTML and PDF tables identically.
|
|
7
|
+
//
|
|
8
|
+
// SERVER-ONLY: pulls in pdfjs-dist. Exported from the package root ('.'),
|
|
9
|
+
// never from the client-safe './parse' entry.
|
|
10
|
+
import { getDocument } from 'pdfjs-dist/legacy/build/pdf.mjs';
|
|
11
|
+
/** Cluster items into visual lines by y within `tol`, ordered top→bottom. */
|
|
12
|
+
function clusterLines(items, tol) {
|
|
13
|
+
const sorted = [...items].sort((a, b) => b.y - a.y);
|
|
14
|
+
const lines = [];
|
|
15
|
+
for (const it of sorted) {
|
|
16
|
+
const line = lines.find((l) => Math.abs(l.y - it.y) <= tol);
|
|
17
|
+
if (line)
|
|
18
|
+
line.items.push(it);
|
|
19
|
+
else
|
|
20
|
+
lines.push({ y: it.y, items: [it] });
|
|
21
|
+
}
|
|
22
|
+
return lines;
|
|
23
|
+
}
|
|
24
|
+
/** Auto-detect column left-edges by cutting the sorted x-starts at the widest gaps. */
|
|
25
|
+
function detectBands(items) {
|
|
26
|
+
const xs = [...new Set(items.map((i) => Math.round(i.x)))].sort((a, b) => a - b);
|
|
27
|
+
if (xs.length === 0)
|
|
28
|
+
return [0];
|
|
29
|
+
const bands = [xs[0]];
|
|
30
|
+
for (let i = 1; i < xs.length; i++) {
|
|
31
|
+
if (xs[i] - xs[i - 1] > 25)
|
|
32
|
+
bands.push(xs[i]); // 25px gap = new column
|
|
33
|
+
}
|
|
34
|
+
return bands;
|
|
35
|
+
}
|
|
36
|
+
/** Assign x to the last band whose left-edge is <= x. */
|
|
37
|
+
function bandIndex(bands, x) {
|
|
38
|
+
let idx = 0;
|
|
39
|
+
for (let i = 0; i < bands.length; i++)
|
|
40
|
+
if (x + 3 >= bands[i])
|
|
41
|
+
idx = i;
|
|
42
|
+
return idx;
|
|
43
|
+
}
|
|
44
|
+
// A word the PDF split across two text runs reports the second run beginning
|
|
45
|
+
// (within GLUE_PX) exactly where the first ended, with NO whitespace at either
|
|
46
|
+
// run's boundary — e.g. "Septemb"|"er". A real inter-word space renders a
|
|
47
|
+
// visible gap (and/or a boundary space in the run text). GLUE_PX is the max
|
|
48
|
+
// horizontal gap still considered "same word".
|
|
49
|
+
const GLUE_PX = 1;
|
|
50
|
+
/**
|
|
51
|
+
* Join a band's items (sorted by x) into one cell string. Consecutive items
|
|
52
|
+
* that are horizontally CONTIGUOUS and carry no whitespace at their shared
|
|
53
|
+
* boundary are glued with no space (they are pieces of one word split across
|
|
54
|
+
* text runs); otherwise a single space separates them. Without this, a
|
|
55
|
+
* mid-word split surfaces as "Septemb er 2020" and fails downstream date/number
|
|
56
|
+
* parsing.
|
|
57
|
+
*/
|
|
58
|
+
function joinCellItems(items) {
|
|
59
|
+
let out = '';
|
|
60
|
+
let prev = null;
|
|
61
|
+
for (const it of items) {
|
|
62
|
+
if (!it.str.trim())
|
|
63
|
+
continue;
|
|
64
|
+
if (prev != null) {
|
|
65
|
+
const contiguous = it.x - (prev.x + prev.w) < GLUE_PX;
|
|
66
|
+
const boundarySpace = /\s$/.test(prev.str) || /^\s/.test(it.str);
|
|
67
|
+
out += contiguous && !boundarySpace ? '' : ' ';
|
|
68
|
+
}
|
|
69
|
+
out += it.str.trim();
|
|
70
|
+
prev = it;
|
|
71
|
+
}
|
|
72
|
+
return out.replace(/\s+/g, ' ').trim();
|
|
73
|
+
}
|
|
74
|
+
/** Concatenate a line's items into one cell per band (items sorted by x). */
|
|
75
|
+
function lineToCells(line, bands) {
|
|
76
|
+
const cells = bands.map(() => []);
|
|
77
|
+
for (const it of [...line.items].sort((a, b) => a.x - b.x)) {
|
|
78
|
+
if (!it.str.trim())
|
|
79
|
+
continue;
|
|
80
|
+
cells[bandIndex(bands, it.x)].push(it);
|
|
81
|
+
}
|
|
82
|
+
return cells.map(joinCellItems);
|
|
83
|
+
}
|
|
84
|
+
export async function extractTableFromPdf(data, opts = {}) {
|
|
85
|
+
const tol = opts.rowTolerance ?? 3;
|
|
86
|
+
const doc = await getDocument({ data, isEvalSupported: false, useSystemFonts: false }).promise;
|
|
87
|
+
const pageCount = opts.maxPages ? Math.min(doc.numPages, opts.maxPages) : doc.numPages;
|
|
88
|
+
const pages = [];
|
|
89
|
+
for (let p = 1; p <= pageCount; p++) {
|
|
90
|
+
const page = await doc.getPage(p);
|
|
91
|
+
const content = await page.getTextContent();
|
|
92
|
+
const items = [];
|
|
93
|
+
for (const it of content.items) {
|
|
94
|
+
if (typeof it.str !== 'string' || !it.str.trim())
|
|
95
|
+
continue;
|
|
96
|
+
items.push({ x: it.transform[4], y: it.transform[5], w: it.width ?? 0, str: it.str });
|
|
97
|
+
}
|
|
98
|
+
pages.push(items);
|
|
99
|
+
}
|
|
100
|
+
// Column geometry is page-independent, so detect bands from every item; but
|
|
101
|
+
// cluster visual lines PER PAGE. Text-item y-coordinates repeat page to page
|
|
102
|
+
// (row N sits at the same y on every page), so clustering the concatenation
|
|
103
|
+
// would merge each page's row N into a single garbled line. Page order is
|
|
104
|
+
// preserved, and a single-page document behaves exactly as before.
|
|
105
|
+
const bands = opts.columnBands ?? detectBands(pages.flat());
|
|
106
|
+
let grid = [];
|
|
107
|
+
for (const items of pages) {
|
|
108
|
+
for (const line of clusterLines(items, tol))
|
|
109
|
+
grid.push(lineToCells(line, bands));
|
|
110
|
+
}
|
|
111
|
+
// Continuation merge: a line whose leftmost cell is empty belongs to the row above.
|
|
112
|
+
if (opts.mergeContinuationRows) {
|
|
113
|
+
const merged = [];
|
|
114
|
+
for (const row of grid) {
|
|
115
|
+
if (merged.length > 0 && row[0].trim() === '') {
|
|
116
|
+
const prev = merged[merged.length - 1];
|
|
117
|
+
for (let i = 0; i < row.length; i++) {
|
|
118
|
+
if (row[i].trim())
|
|
119
|
+
prev[i] = [prev[i], row[i]].filter(Boolean).join(' ').trim();
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
else {
|
|
123
|
+
merged.push([...row]);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
grid = merged;
|
|
127
|
+
}
|
|
128
|
+
let headers;
|
|
129
|
+
let rows;
|
|
130
|
+
if (opts.headerLabels) {
|
|
131
|
+
headers = opts.headerLabels;
|
|
132
|
+
rows = grid;
|
|
133
|
+
}
|
|
134
|
+
else {
|
|
135
|
+
const n = Math.max(1, opts.headerRows ?? 1);
|
|
136
|
+
const headerLines = grid.slice(0, n);
|
|
137
|
+
headers = bands.map((_, c) => headerLines.map((l) => l[c] ?? '').join(' ').replace(/\s+/g, ' ').trim());
|
|
138
|
+
rows = grid.slice(n);
|
|
139
|
+
}
|
|
140
|
+
return { headers, rows };
|
|
141
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/pdf/index.ts"],"names":[],"mappings":"AACA,cAAc,wBAAwB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "hazo_scrape",
|
|
3
|
-
"version": "1.2
|
|
3
|
+
"version": "1.5.2",
|
|
4
4
|
"description": "Generic source-agnostic web scraping engine with a network-free parse core.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"module": "./dist/index.js",
|
|
@@ -62,6 +62,7 @@
|
|
|
62
62
|
"cheerio": "^1.2.0",
|
|
63
63
|
"hazo_config": "^2.4.1",
|
|
64
64
|
"hazo_secure": "^1.4.0",
|
|
65
|
+
"pdfjs-dist": "^4.10.38",
|
|
65
66
|
"robots-parser": "^3.0.1",
|
|
66
67
|
"undici": "^7.28.0"
|
|
67
68
|
}
|