@toolpath/tool-scraper 0.1.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -11
- package/dist/columns.d.ts +64 -0
- package/dist/columns.js +68 -0
- package/dist/conventions.d.ts +107 -15
- package/dist/conventions.js +126 -15
- package/dist/errors.d.ts +30 -0
- package/dist/errors.js +30 -0
- package/dist/families/emuge.d.ts +185 -0
- package/dist/families/emuge.js +163 -0
- package/dist/families/harvey.d.ts +1782 -0
- package/dist/families/harvey.js +1328 -0
- package/dist/families/index.js +5 -2
- package/dist/families/kennametal.d.ts +21 -0
- package/dist/families/kennametal.js +10 -0
- package/dist/families/maritool.d.ts +120 -0
- package/dist/families/maritool.js +175 -0
- package/dist/family.d.ts +35 -1
- package/dist/family.js +28 -0
- package/dist/identity.d.ts +18 -0
- package/dist/identity.js +46 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +16 -0
- package/dist/measure.d.ts +112 -0
- package/dist/measure.js +130 -0
- package/dist/node/cli.d.ts +3 -0
- package/dist/node/cli.js +158 -3
- package/dist/node/main.js +0 -0
- package/dist/records.d.ts +214 -12
- package/dist/records.js +140 -5
- package/dist/registry.d.ts +56 -1
- package/dist/registry.js +79 -2
- package/dist/scrape.d.ts +15 -0
- package/dist/scrape.js +24 -0
- package/dist/vendors/destinytool/records.d.ts +13 -3
- package/dist/vendors/destinytool/records.js +49 -37
- package/dist/vendors/emuge/index.d.ts +17 -0
- package/dist/vendors/emuge/index.js +17 -0
- package/dist/vendors/emuge/records.d.ts +230 -0
- package/dist/vendors/emuge/records.js +543 -0
- package/dist/vendors/emuge/scrape.d.ts +227 -0
- package/dist/vendors/emuge/scrape.js +358 -0
- package/dist/vendors/emuge/value.d.ts +75 -0
- package/dist/vendors/emuge/value.js +116 -0
- package/dist/vendors/harvey/catalog.d.ts +53 -0
- package/dist/vendors/harvey/catalog.js +120 -0
- package/dist/vendors/harvey/header.d.ts +89 -0
- package/dist/vendors/harvey/header.js +185 -0
- package/dist/vendors/harvey/index.d.ts +21 -0
- package/dist/vendors/harvey/index.js +21 -0
- package/dist/vendors/harvey/lexicon.d.ts +73 -0
- package/dist/vendors/harvey/lexicon.js +126 -0
- package/dist/vendors/harvey/literal.d.ts +68 -0
- package/dist/vendors/harvey/literal.js +214 -0
- package/dist/vendors/harvey/records.d.ts +79 -0
- package/dist/vendors/harvey/records.js +163 -0
- package/dist/vendors/harvey/scrape.d.ts +187 -0
- package/dist/vendors/harvey/scrape.js +483 -0
- package/dist/vendors/harvey/value.d.ts +74 -0
- package/dist/vendors/harvey/value.js +119 -0
- package/dist/vendors/kennametal/family.d.ts +119 -0
- package/dist/vendors/kennametal/family.js +155 -0
- package/dist/vendors/kennametal/index.d.ts +1 -0
- package/dist/vendors/kennametal/index.js +1 -0
- package/dist/vendors/kennametal/records.d.ts +29 -4
- package/dist/vendors/kennametal/records.js +93 -26
- package/dist/vendors/kennametal/scrape.d.ts +25 -2
- package/dist/vendors/kennametal/scrape.js +28 -3
- package/dist/vendors/maritool/catalog.d.ts +81 -0
- package/dist/vendors/maritool/catalog.js +132 -0
- package/dist/vendors/maritool/index.d.ts +10 -0
- package/dist/vendors/maritool/index.js +10 -0
- package/dist/vendors/maritool/scrape.d.ts +297 -0
- package/dist/vendors/maritool/scrape.js +593 -0
- package/dist/vendors/regofix/scrape.d.ts +8 -11
- package/dist/vendors/regofix/scrape.js +21 -36
- package/package.json +31 -7
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One EMUGE-FRANKEN property — its label and its value — read.
|
|
3
|
+
*
|
|
4
|
+
* The API answers every measurement as a `{ property, value }` pair of display
|
|
5
|
+
* strings, and both halves need work before they can be a CSV column and a
|
|
6
|
+
* number:
|
|
7
|
+
*
|
|
8
|
+
* ```json
|
|
9
|
+
* { "property": "cutting diameter Ød₁ [inch]", "value": "1 1/2 \"" }
|
|
10
|
+
* { "property": "overall length l₁", "value": "38 mm" }
|
|
11
|
+
* { "property": "point angle", "value": "140 deg" }
|
|
12
|
+
* ```
|
|
13
|
+
*
|
|
14
|
+
* ## The unit is in the value; the label's tag is unreliable
|
|
15
|
+
*
|
|
16
|
+
* `cutting length l₂ [mm]` carries a unit tag and `overall length l₁` and
|
|
17
|
+
* `neck diameter Ød₃` do not, on the same part, in the same response. So a
|
|
18
|
+
* label's tag is stripped for the column name — the CSV states the unit once,
|
|
19
|
+
* in `conventions.UNIT_SUFFIX`'s suffix, and a label that carried `[mm]` as
|
|
20
|
+
* well would state it twice — and the reading of the unit comes from the value,
|
|
21
|
+
* which states one on every measurement seen.
|
|
22
|
+
*
|
|
23
|
+
* A stated unit that disagrees with the family's is converted and warned about
|
|
24
|
+
* rather than refused: the value is right and the column's suffix is right, so
|
|
25
|
+
* dropping the row would lose a part somebody can order. That call is not made
|
|
26
|
+
* here — it is `measure.asLength`'s, which is the one place every vendor makes
|
|
27
|
+
* it. What this module owns is the grammar below and nothing else.
|
|
28
|
+
*
|
|
29
|
+
* ## What is deliberately not read
|
|
30
|
+
*
|
|
31
|
+
* **A range.** `helix angle` is `35-38 deg` and `clamping diameter ØD₁` is
|
|
32
|
+
* `2 - 10 mm`; `measure.fractionValue` would read the first as `73`, because
|
|
33
|
+
* its mixed-number form is `1-1/2` and `35-38` fits it with the denominator
|
|
34
|
+
* absent. {@link NUMBER} admits no hyphenated form at all — EMUGE writes a
|
|
35
|
+
* mixed number space-separated, `1 1/2` — so a range has no reading here, and
|
|
36
|
+
* the columns those two land in are unmapped receipt columns anyway.
|
|
37
|
+
*
|
|
38
|
+
* **A tolerance.** `<=0,003 mm`, `± 0,0008 "`. These carry a German decimal
|
|
39
|
+
* comma where every dimensional value uses a dot, and they lead with a
|
|
40
|
+
* comparator. Guessing at what a comma means in a number this package is about
|
|
41
|
+
* to publish is exactly the kind of quiet wrong answer `measure.ts` refuses to
|
|
42
|
+
* make; they get no reading and stay in the CSV as the vendor's own text.
|
|
43
|
+
*/
|
|
44
|
+
import { asCount, asLength, fractionValue } from '../../measure.js';
|
|
45
|
+
import { consoleWarn } from '../../scrape.js';
|
|
46
|
+
/** A label's trailing unit tag: `[mm]`, `[inch]`, `[in]`. */
|
|
47
|
+
const LABEL_UNIT = /\s*\[(?:mm|in|inch)\]\s*$/i;
|
|
48
|
+
/**
|
|
49
|
+
* A decimal, a simple fraction, or a mixed number written with a **space**,
|
|
50
|
+
* followed by the unit the value states.
|
|
51
|
+
*
|
|
52
|
+
* Ordered longest-first, because `\d*\.?\d+` alone matches the `1` of `1 1/2`
|
|
53
|
+
* and would leave the rest unread — a 1.5 inch tool published as a 1 inch one.
|
|
54
|
+
*
|
|
55
|
+
* No hyphenated mixed number: see the module docstring. `"` is EMUGE's inch
|
|
56
|
+
* mark and `deg` its degree word; both are written with a leading space in
|
|
57
|
+
* every value seen, and the `\s*` tolerates one that is not.
|
|
58
|
+
*/
|
|
59
|
+
const NUMBER = /^(\d+\s+\d+\/\d+|\d*\.?\d+\/\d+|\d*\.?\d+)\s*(mm|"|deg)?$/;
|
|
60
|
+
const NOTHING = { value: null, stated: null };
|
|
61
|
+
const UNITS = {
|
|
62
|
+
mm: 'millimeters',
|
|
63
|
+
'"': 'inches',
|
|
64
|
+
deg: 'degrees',
|
|
65
|
+
};
|
|
66
|
+
/**
|
|
67
|
+
* A property label with its unit tag removed — the CSV's bare label.
|
|
68
|
+
*
|
|
69
|
+
* `'cutting diameter Ød₁ [inch]'` -> `'cutting diameter Ød₁'`. Everything else
|
|
70
|
+
* about the label is left exactly as the vendor wrote it, subscripts and `Ø`
|
|
71
|
+
* included: the CSV is the receipt, and the tag is the one part of it that the
|
|
72
|
+
* column's own `_mm`/`_in` suffix already says.
|
|
73
|
+
*/
|
|
74
|
+
export function bareLabel(property) {
|
|
75
|
+
return property.replace(LABEL_UNIT, '').trim();
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* One value string -> its number and the unit it states.
|
|
79
|
+
*
|
|
80
|
+
* `''`, a range, a tolerance and any other text come back stating nothing —
|
|
81
|
+
* there is no number there and 0 is not a substitute for one.
|
|
82
|
+
*/
|
|
83
|
+
export function parseMeasure(display) {
|
|
84
|
+
const text = display.trim();
|
|
85
|
+
if (text === '')
|
|
86
|
+
return NOTHING;
|
|
87
|
+
const matched = NUMBER.exec(text);
|
|
88
|
+
if (matched === null)
|
|
89
|
+
return NOTHING;
|
|
90
|
+
// A space-separated mixed number is the one shape `fractionValue` does not
|
|
91
|
+
// take: its own form is `1-1/2`. Normalising here rather than widening the
|
|
92
|
+
// core reader keeps the hyphen shape — and the `35-38` range that shape would
|
|
93
|
+
// swallow — out of this vendor entirely.
|
|
94
|
+
const value = fractionValue(matched[1].replace(/\s+/, '-'));
|
|
95
|
+
if (value === null)
|
|
96
|
+
return NOTHING;
|
|
97
|
+
const unit = matched[2];
|
|
98
|
+
return { value, stated: unit === undefined ? null : UNITS[unit] };
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* One value as a length in `unit`, or null where it publishes none.
|
|
102
|
+
*
|
|
103
|
+
* `measure.asLength` makes both calls — convert-and-warn a value whose stated
|
|
104
|
+
* unit disagrees with the family's, refuse an angle in a length column —
|
|
105
|
+
* because they are the same two calls for every vendor. They were a verbatim
|
|
106
|
+
* copy of `vendors/harvey/value.ts`'s, warnings included, until 2026-09-01.
|
|
107
|
+
* What is EMUGE's is above: the grammar, and that the unit is read out of the
|
|
108
|
+
* value rather than off the label.
|
|
109
|
+
*/
|
|
110
|
+
export function measureIn(display, unit, what, warn = consoleWarn) {
|
|
111
|
+
return asLength(parseMeasure(display), display, unit, what, warn);
|
|
112
|
+
}
|
|
113
|
+
/** One value as a whole count — a flute number. Null where there is none. */
|
|
114
|
+
export function wholeCount(display) {
|
|
115
|
+
return asCount(parseMeasure(display));
|
|
116
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The category tree -> the product pages under it.
|
|
3
|
+
*
|
|
4
|
+
* Harvey publishes no sitemap and no flat index: `robots.txt`, `sitemap.xml` and
|
|
5
|
+
* `sitemap_index.xml` all 404, and `/products/all-products` lists the seven top
|
|
6
|
+
* categories and no products. Walking the catalog grid is the only way to find
|
|
7
|
+
* out what exists — see `docs/HARVEY_PRODUCT_TABLE.md` §1.1 and §1.6.
|
|
8
|
+
*
|
|
9
|
+
* Nothing in a scrape needs this: `families/harvey.ts` names all 52 product
|
|
10
|
+
* pages, and a family scrape fetches exactly one. It is here for the maintenance
|
|
11
|
+
* question that table cannot answer — *has Harvey added a page?* — and the CLI
|
|
12
|
+
* exposes it as `harvey --catalog` for that purpose.
|
|
13
|
+
*
|
|
14
|
+
* A page carries subcategory links or product links, never both, so the
|
|
15
|
+
* recursion terminates where products appear. Both selectors were verified
|
|
16
|
+
* against all 33 category pages the four roots reach.
|
|
17
|
+
*/
|
|
18
|
+
import type { Fetcher } from '../../fetch.js';
|
|
19
|
+
import { type Warn } from '../../scrape.js';
|
|
20
|
+
/**
|
|
21
|
+
* The four category trees this adapter covers.
|
|
22
|
+
*
|
|
23
|
+
* Harvey's catalog is far larger; these are the trees asked for, and adding one
|
|
24
|
+
* is a decision rather than a widening of a pattern — see the package's rule on
|
|
25
|
+
* not adding vendor scope without being asked.
|
|
26
|
+
*/
|
|
27
|
+
export declare const CATEGORY_ROOTS: readonly string[];
|
|
28
|
+
/** What one catalog page links to. */
|
|
29
|
+
export interface CategoryLinks {
|
|
30
|
+
subcategories: string[];
|
|
31
|
+
products: string[];
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* The subcategory and product links on one catalog page.
|
|
35
|
+
*
|
|
36
|
+
* Product links are identified by their **wrapper** rather than by the anchor,
|
|
37
|
+
* because the anchor carries no class of its own — the grid renders
|
|
38
|
+
* `<div class="col-md-4 col-12 item-wrapper"><a href="/products/...">`. A rule
|
|
39
|
+
* that took every `/products/` anchor on the page would take the breadcrumb and
|
|
40
|
+
* the footer with it.
|
|
41
|
+
*/
|
|
42
|
+
export declare function parseCategoryPage(html: string): CategoryLinks;
|
|
43
|
+
/**
|
|
44
|
+
* Every product page reachable from `roots`, sorted.
|
|
45
|
+
*
|
|
46
|
+
* Sequential and paced by the package's shared politeness delay. The whole walk
|
|
47
|
+
* is about 33 requests; nothing here needs concurrency, and Cloudflare fronts
|
|
48
|
+
* this site, so raising request volume is the only real risk.
|
|
49
|
+
*/
|
|
50
|
+
export declare function discoverProducts(fetcher: Fetcher, roots?: readonly string[], options?: {
|
|
51
|
+
warn?: Warn;
|
|
52
|
+
delayMs?: number;
|
|
53
|
+
}): Promise<string[]>;
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The category tree -> the product pages under it.
|
|
3
|
+
*
|
|
4
|
+
* Harvey publishes no sitemap and no flat index: `robots.txt`, `sitemap.xml` and
|
|
5
|
+
* `sitemap_index.xml` all 404, and `/products/all-products` lists the seven top
|
|
6
|
+
* categories and no products. Walking the catalog grid is the only way to find
|
|
7
|
+
* out what exists — see `docs/HARVEY_PRODUCT_TABLE.md` §1.1 and §1.6.
|
|
8
|
+
*
|
|
9
|
+
* Nothing in a scrape needs this: `families/harvey.ts` names all 52 product
|
|
10
|
+
* pages, and a family scrape fetches exactly one. It is here for the maintenance
|
|
11
|
+
* question that table cannot answer — *has Harvey added a page?* — and the CLI
|
|
12
|
+
* exposes it as `harvey --catalog` for that purpose.
|
|
13
|
+
*
|
|
14
|
+
* A page carries subcategory links or product links, never both, so the
|
|
15
|
+
* recursion terminates where products appear. Both selectors were verified
|
|
16
|
+
* against all 33 category pages the four roots reach.
|
|
17
|
+
*/
|
|
18
|
+
import { Parser } from 'htmlparser2';
|
|
19
|
+
import { compare } from '../../order.js';
|
|
20
|
+
import { consoleWarn, pause, REQUEST_DELAY_MS } from '../../scrape.js';
|
|
21
|
+
import { BASE } from './scrape.js';
|
|
22
|
+
/**
|
|
23
|
+
* The four category trees this adapter covers.
|
|
24
|
+
*
|
|
25
|
+
* Harvey's catalog is far larger; these are the trees asked for, and adding one
|
|
26
|
+
* is a decision rather than a widening of a pattern — see the package's rule on
|
|
27
|
+
* not adding vendor scope without being asked.
|
|
28
|
+
*/
|
|
29
|
+
export const CATEGORY_ROOTS = [
|
|
30
|
+
'/products/miniature-end-mills/ball',
|
|
31
|
+
'/products/miniature-end-mills/corner-radius',
|
|
32
|
+
'/products/miniature-end-mills/square',
|
|
33
|
+
'/products/specialty-profiles/keyseat-cutters',
|
|
34
|
+
];
|
|
35
|
+
/** The class marking a link to a subcategory in the catalog grid. */
|
|
36
|
+
const SUBCATEGORY_CLASS = 'img-wrapper';
|
|
37
|
+
/** The classes marking the wrapper around a link to one product. */
|
|
38
|
+
const PRODUCT_CLASSES = ['col-md-4', 'col-12', 'item-wrapper'];
|
|
39
|
+
function classes(attribs) {
|
|
40
|
+
return (attribs['class'] ?? '').split(/\s+/).filter(Boolean);
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* The subcategory and product links on one catalog page.
|
|
44
|
+
*
|
|
45
|
+
* Product links are identified by their **wrapper** rather than by the anchor,
|
|
46
|
+
* because the anchor carries no class of its own — the grid renders
|
|
47
|
+
* `<div class="col-md-4 col-12 item-wrapper"><a href="/products/...">`. A rule
|
|
48
|
+
* that took every `/products/` anchor on the page would take the breadcrumb and
|
|
49
|
+
* the footer with it.
|
|
50
|
+
*/
|
|
51
|
+
export function parseCategoryPage(html) {
|
|
52
|
+
const subcategories = new Set();
|
|
53
|
+
const products = new Set();
|
|
54
|
+
// A depth counter rather than a flag, because the wrapper holds nested divs
|
|
55
|
+
// and a flag would clear on the first inner close tag.
|
|
56
|
+
let inProduct = 0;
|
|
57
|
+
const parser = new Parser({
|
|
58
|
+
onopentag: (tag, attribs) => {
|
|
59
|
+
if (inProduct > 0) {
|
|
60
|
+
inProduct++;
|
|
61
|
+
if (tag === 'a') {
|
|
62
|
+
const href = attribs['href'] ?? '';
|
|
63
|
+
if (href.startsWith('/products/'))
|
|
64
|
+
products.add(href);
|
|
65
|
+
}
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
if (tag === 'div' && PRODUCT_CLASSES.every((c) => classes(attribs).includes(c))) {
|
|
69
|
+
inProduct = 1;
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
if (tag === 'a' && classes(attribs).includes(SUBCATEGORY_CLASS)) {
|
|
73
|
+
const href = attribs['href'] ?? '';
|
|
74
|
+
if (href.startsWith('/products/'))
|
|
75
|
+
subcategories.add(href);
|
|
76
|
+
}
|
|
77
|
+
},
|
|
78
|
+
onclosetag: () => {
|
|
79
|
+
if (inProduct > 0)
|
|
80
|
+
inProduct--;
|
|
81
|
+
},
|
|
82
|
+
}, { decodeEntities: true });
|
|
83
|
+
parser.write(html);
|
|
84
|
+
parser.end();
|
|
85
|
+
return {
|
|
86
|
+
subcategories: [...subcategories].sort(compare),
|
|
87
|
+
products: [...products].sort(compare),
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Every product page reachable from `roots`, sorted.
|
|
92
|
+
*
|
|
93
|
+
* Sequential and paced by the package's shared politeness delay. The whole walk
|
|
94
|
+
* is about 33 requests; nothing here needs concurrency, and Cloudflare fronts
|
|
95
|
+
* this site, so raising request volume is the only real risk.
|
|
96
|
+
*/
|
|
97
|
+
export async function discoverProducts(fetcher, roots = CATEGORY_ROOTS, options = {}) {
|
|
98
|
+
const { warn = consoleWarn, delayMs = REQUEST_DELAY_MS } = options;
|
|
99
|
+
const products = new Set();
|
|
100
|
+
const seen = new Set();
|
|
101
|
+
const queue = [...roots];
|
|
102
|
+
while (queue.length > 0) {
|
|
103
|
+
const path = queue.shift();
|
|
104
|
+
if (seen.has(path))
|
|
105
|
+
continue;
|
|
106
|
+
seen.add(path);
|
|
107
|
+
const { subcategories, products: found } = parseCategoryPage(await fetcher.text(BASE + path));
|
|
108
|
+
if (subcategories.length === 0 && found.length === 0) {
|
|
109
|
+
warn(` WARNING: ${path} links to neither a subcategory nor a product`);
|
|
110
|
+
}
|
|
111
|
+
for (const product of found)
|
|
112
|
+
products.add(product);
|
|
113
|
+
for (const child of subcategories) {
|
|
114
|
+
if (child !== path)
|
|
115
|
+
queue.push(child);
|
|
116
|
+
}
|
|
117
|
+
await pause(delayMs);
|
|
118
|
+
}
|
|
119
|
+
return [...products].sort(compare);
|
|
120
|
+
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A product table's two-row `<thead>` -> one flat column per `cols<N>` entry.
|
|
3
|
+
*
|
|
4
|
+
* Harvey publishes a table's header in the DOM and its data in a JavaScript
|
|
5
|
+
* literal, and the only thing joining them is position: `cols1` is
|
|
6
|
+
* `[{data:"a0"}, {data:"a1"}, ...]` and its Nth entry is the Nth column of the
|
|
7
|
+
* flattened header. That alignment is checked on every table — see
|
|
8
|
+
* `scrape.ts` — because a header that has drifted by one column produces a CSV
|
|
9
|
+
* where every dimension is labelled as its neighbour.
|
|
10
|
+
*
|
|
11
|
+
* ## Flattening
|
|
12
|
+
*
|
|
13
|
+
* The header is exactly two `<tr>`s. A top `<th>` with `rowspan >= 2` is its own
|
|
14
|
+
* column and has no sub-label (only "Add to Cart" is like this); otherwise it
|
|
15
|
+
* consumes `colspan` cells from the second row as its sub-labels. Verified
|
|
16
|
+
* against all 80 tables on the 52 product pages.
|
|
17
|
+
*
|
|
18
|
+
* ## Two things about the text
|
|
19
|
+
*
|
|
20
|
+
* **`<br>` is a space.** `CUTTER <br/>DIAMETER` is the same label as
|
|
21
|
+
* `CUTTER DIAMETER`, and collapsing the tag away instead yields
|
|
22
|
+
* `CUTTERDIAMETER` — a 51st header shape that exists only in the parser.
|
|
23
|
+
*
|
|
24
|
+
* **A sub-label's tolerance is excluded, not stripped afterwards.** The
|
|
25
|
+
* tolerance is rendered in its own `div.hpc-inline`, so skipping that subtree
|
|
26
|
+
* gives `D1` where reading the whole cell gives `D1+.0005"-.0005"`. It matters
|
|
27
|
+
* more than tidiness: Harvey's tolerance text carries typos — `D1+0005"-.0005"`,
|
|
28
|
+
* `R+.001"-001"`, `L2.020"-.000"` are all real — and every one of them would be
|
|
29
|
+
* another spelling for a post-hoc stripper to get right.
|
|
30
|
+
*/
|
|
31
|
+
/** The class of the `<div>` holding a sub-label's published tolerance. */
|
|
32
|
+
export declare const TOLERANCE_CLASS = "hpc-inline";
|
|
33
|
+
/** The class Harvey renders an unlabelled annotation column's header with. */
|
|
34
|
+
export declare const HIDDEN_CLASS = "white-text";
|
|
35
|
+
/** True where a header label says nothing about what the column holds. */
|
|
36
|
+
export declare function isJunkLabel(label: string): boolean;
|
|
37
|
+
/** One `<th>` as parsed: its text, its span, and its class attribute. */
|
|
38
|
+
interface HeaderCell {
|
|
39
|
+
text: string;
|
|
40
|
+
colspan: number;
|
|
41
|
+
rowspan: number;
|
|
42
|
+
className: string;
|
|
43
|
+
}
|
|
44
|
+
/** One data column of a table, as the header describes it. */
|
|
45
|
+
export interface HeaderColumn {
|
|
46
|
+
/** The top row's label, `<br>`s collapsed to spaces. */
|
|
47
|
+
top: string;
|
|
48
|
+
/** The second row's label with its tolerance removed, or null under a rowspan. */
|
|
49
|
+
sub: string | null;
|
|
50
|
+
/** The top `<th>`'s class attribute — `white-text` marks a hidden column. */
|
|
51
|
+
topClass: string;
|
|
52
|
+
/** How many columns the top `<th>` spanned. 1 unless it is a coating group. */
|
|
53
|
+
span: number;
|
|
54
|
+
/** This column's index inside that span. 0 on a column of its own. */
|
|
55
|
+
slot: number;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* The `<th>` rows of the `<thead>` of the table with this id.
|
|
59
|
+
*
|
|
60
|
+
* Streams the whole document rather than slicing it: a `</table>` inside a cell
|
|
61
|
+
* would end a sliced segment early, and htmlparser2 is already this package's
|
|
62
|
+
* one parser dependency.
|
|
63
|
+
*
|
|
64
|
+
* `decodeEntities` is on because these headers carry ` ` and `°` —
|
|
65
|
+
* the blank flute sub-label is literally ` `, and it has to reach
|
|
66
|
+
* {@link isJunkLabel} as an empty string rather than as two characters.
|
|
67
|
+
*/
|
|
68
|
+
export declare function parseHeadRows(html: string, tableId: string): HeaderCell[][];
|
|
69
|
+
/**
|
|
70
|
+
* One table's header, flattened to one entry per data column.
|
|
71
|
+
*
|
|
72
|
+
* Throws when the table is not in the document, rather than returning an empty
|
|
73
|
+
* header: a page that stopped serving a table it declares in `cols<N>` is a
|
|
74
|
+
* response that changed shape, and a zero-column table would reach the caller
|
|
75
|
+
* looking like a family the vendor discontinued.
|
|
76
|
+
*/
|
|
77
|
+
export declare function flatHeader(html: string, tableId: string): HeaderColumn[];
|
|
78
|
+
/**
|
|
79
|
+
* The vendor's own symbol for a dimension column — `D1`, `L2`, `R`, `A1`, `#`.
|
|
80
|
+
*
|
|
81
|
+
* Null where the sub-label carries none, which is a real state rather than a
|
|
82
|
+
* fault: `NECK DIA.` publishes no symbol at all, and the ratio columns publish
|
|
83
|
+
* junk.
|
|
84
|
+
*
|
|
85
|
+
* A trailing `(h6)` is part of the shank tolerance class and not of the symbol,
|
|
86
|
+
* so `D2(h6)` and `D2` are the same column.
|
|
87
|
+
*/
|
|
88
|
+
export declare function symbolOf(sub: string | null): string | null;
|
|
89
|
+
export {};
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A product table's two-row `<thead>` -> one flat column per `cols<N>` entry.
|
|
3
|
+
*
|
|
4
|
+
* Harvey publishes a table's header in the DOM and its data in a JavaScript
|
|
5
|
+
* literal, and the only thing joining them is position: `cols1` is
|
|
6
|
+
* `[{data:"a0"}, {data:"a1"}, ...]` and its Nth entry is the Nth column of the
|
|
7
|
+
* flattened header. That alignment is checked on every table — see
|
|
8
|
+
* `scrape.ts` — because a header that has drifted by one column produces a CSV
|
|
9
|
+
* where every dimension is labelled as its neighbour.
|
|
10
|
+
*
|
|
11
|
+
* ## Flattening
|
|
12
|
+
*
|
|
13
|
+
* The header is exactly two `<tr>`s. A top `<th>` with `rowspan >= 2` is its own
|
|
14
|
+
* column and has no sub-label (only "Add to Cart" is like this); otherwise it
|
|
15
|
+
* consumes `colspan` cells from the second row as its sub-labels. Verified
|
|
16
|
+
* against all 80 tables on the 52 product pages.
|
|
17
|
+
*
|
|
18
|
+
* ## Two things about the text
|
|
19
|
+
*
|
|
20
|
+
* **`<br>` is a space.** `CUTTER <br/>DIAMETER` is the same label as
|
|
21
|
+
* `CUTTER DIAMETER`, and collapsing the tag away instead yields
|
|
22
|
+
* `CUTTERDIAMETER` — a 51st header shape that exists only in the parser.
|
|
23
|
+
*
|
|
24
|
+
* **A sub-label's tolerance is excluded, not stripped afterwards.** The
|
|
25
|
+
* tolerance is rendered in its own `div.hpc-inline`, so skipping that subtree
|
|
26
|
+
* gives `D1` where reading the whole cell gives `D1+.0005"-.0005"`. It matters
|
|
27
|
+
* more than tidiness: Harvey's tolerance text carries typos — `D1+0005"-.0005"`,
|
|
28
|
+
* `R+.001"-001"`, `L2.020"-.000"` are all real — and every one of them would be
|
|
29
|
+
* another spelling for a post-hoc stripper to get right.
|
|
30
|
+
*/
|
|
31
|
+
import { Parser } from 'htmlparser2';
|
|
32
|
+
import { VendorResponseError } from '../../errors.js';
|
|
33
|
+
/** The class of the `<div>` holding a sub-label's published tolerance. */
|
|
34
|
+
export const TOLERANCE_CLASS = 'hpc-inline';
|
|
35
|
+
/** The class Harvey renders an unlabelled annotation column's header with. */
|
|
36
|
+
export const HIDDEN_CLASS = 'white-text';
|
|
37
|
+
/** Sub-label text that carries nothing. ` ` collapses to empty here. */
|
|
38
|
+
const JUNK = new Set(['', '.', '"', 'X']);
|
|
39
|
+
/** True where a header label says nothing about what the column holds. */
|
|
40
|
+
export function isJunkLabel(label) {
|
|
41
|
+
return JUNK.has(label);
|
|
42
|
+
}
|
|
43
|
+
/** `" a b \n"` -> `"a b"`. */
|
|
44
|
+
function collapse(text) {
|
|
45
|
+
return text.split(/\s+/).filter(Boolean).join(' ');
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* The `<th>` rows of the `<thead>` of the table with this id.
|
|
49
|
+
*
|
|
50
|
+
* Streams the whole document rather than slicing it: a `</table>` inside a cell
|
|
51
|
+
* would end a sliced segment early, and htmlparser2 is already this package's
|
|
52
|
+
* one parser dependency.
|
|
53
|
+
*
|
|
54
|
+
* `decodeEntities` is on because these headers carry ` ` and `°` —
|
|
55
|
+
* the blank flute sub-label is literally ` `, and it has to reach
|
|
56
|
+
* {@link isJunkLabel} as an empty string rather than as two characters.
|
|
57
|
+
*/
|
|
58
|
+
export function parseHeadRows(html, tableId) {
|
|
59
|
+
const rows = [];
|
|
60
|
+
let inTable = false;
|
|
61
|
+
let inHead = false;
|
|
62
|
+
let row = null;
|
|
63
|
+
let cell = null;
|
|
64
|
+
// A depth counter rather than a flag: the tolerance is a `div` inside a `div`,
|
|
65
|
+
// and a flag would re-enable the text on the inner close tag.
|
|
66
|
+
let tolerance = 0;
|
|
67
|
+
let depth = 0;
|
|
68
|
+
const parser = new Parser({
|
|
69
|
+
onopentag: (tag, attribs) => {
|
|
70
|
+
if (tag === 'table') {
|
|
71
|
+
inTable = attribs['id'] === tableId;
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
if (!inTable)
|
|
75
|
+
return;
|
|
76
|
+
if (tag === 'thead') {
|
|
77
|
+
inHead = true;
|
|
78
|
+
}
|
|
79
|
+
else if (inHead && tag === 'tr') {
|
|
80
|
+
row = [];
|
|
81
|
+
}
|
|
82
|
+
else if (inHead && tag === 'th' && row !== null) {
|
|
83
|
+
cell = {
|
|
84
|
+
text: '',
|
|
85
|
+
colspan: Number.parseInt(attribs['colspan'] ?? '1', 10) || 1,
|
|
86
|
+
rowspan: Number.parseInt(attribs['rowspan'] ?? '1', 10) || 1,
|
|
87
|
+
className: attribs['class'] ?? '',
|
|
88
|
+
};
|
|
89
|
+
tolerance = 0;
|
|
90
|
+
depth = 0;
|
|
91
|
+
}
|
|
92
|
+
else if (cell !== null) {
|
|
93
|
+
depth++;
|
|
94
|
+
if (tolerance === 0 && (attribs['class'] ?? '').split(/\s+/).includes(TOLERANCE_CLASS)) {
|
|
95
|
+
tolerance = depth;
|
|
96
|
+
}
|
|
97
|
+
if (tag === 'br')
|
|
98
|
+
cell.text += ' ';
|
|
99
|
+
}
|
|
100
|
+
},
|
|
101
|
+
ontext: (text) => {
|
|
102
|
+
if (cell !== null && tolerance === 0)
|
|
103
|
+
cell.text += text;
|
|
104
|
+
},
|
|
105
|
+
onclosetag: (tag) => {
|
|
106
|
+
if (!inTable)
|
|
107
|
+
return;
|
|
108
|
+
if (tag === 'th' && cell !== null) {
|
|
109
|
+
row?.push({ ...cell, text: collapse(cell.text) });
|
|
110
|
+
cell = null;
|
|
111
|
+
}
|
|
112
|
+
else if (tag === 'tr' && row !== null) {
|
|
113
|
+
rows.push(row);
|
|
114
|
+
row = null;
|
|
115
|
+
}
|
|
116
|
+
else if (tag === 'thead') {
|
|
117
|
+
inHead = false;
|
|
118
|
+
// Everything after the head is data the literal already carries.
|
|
119
|
+
inTable = false;
|
|
120
|
+
}
|
|
121
|
+
else if (cell !== null) {
|
|
122
|
+
if (tolerance === depth)
|
|
123
|
+
tolerance = 0;
|
|
124
|
+
depth--;
|
|
125
|
+
}
|
|
126
|
+
},
|
|
127
|
+
}, { decodeEntities: true });
|
|
128
|
+
parser.write(html);
|
|
129
|
+
parser.end();
|
|
130
|
+
return rows;
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* One table's header, flattened to one entry per data column.
|
|
134
|
+
*
|
|
135
|
+
* Throws when the table is not in the document, rather than returning an empty
|
|
136
|
+
* header: a page that stopped serving a table it declares in `cols<N>` is a
|
|
137
|
+
* response that changed shape, and a zero-column table would reach the caller
|
|
138
|
+
* looking like a family the vendor discontinued.
|
|
139
|
+
*/
|
|
140
|
+
export function flatHeader(html, tableId) {
|
|
141
|
+
const rows = parseHeadRows(html, tableId);
|
|
142
|
+
if (rows.length === 0) {
|
|
143
|
+
throw new VendorResponseError(tableId, 'has no <thead> on the page — the table changed shape');
|
|
144
|
+
}
|
|
145
|
+
const [top = [], sub = []] = rows;
|
|
146
|
+
const columns = [];
|
|
147
|
+
let next = 0;
|
|
148
|
+
for (const cell of top) {
|
|
149
|
+
if (cell.rowspan >= 2) {
|
|
150
|
+
columns.push({ top: cell.text, sub: null, topClass: cell.className, span: 1, slot: 0 });
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
for (let slot = 0; slot < cell.colspan; slot++) {
|
|
154
|
+
columns.push({
|
|
155
|
+
top: cell.text,
|
|
156
|
+
sub: sub[next]?.text ?? '',
|
|
157
|
+
topClass: cell.className,
|
|
158
|
+
span: cell.colspan,
|
|
159
|
+
slot,
|
|
160
|
+
});
|
|
161
|
+
next++;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
if (next > sub.length) {
|
|
165
|
+
throw new VendorResponseError(tableId, `spans ${next} sub-labels across its top row but the second row has ` +
|
|
166
|
+
`${sub.length} — the header changed shape`);
|
|
167
|
+
}
|
|
168
|
+
return columns;
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* The vendor's own symbol for a dimension column — `D1`, `L2`, `R`, `A1`, `#`.
|
|
172
|
+
*
|
|
173
|
+
* Null where the sub-label carries none, which is a real state rather than a
|
|
174
|
+
* fault: `NECK DIA.` publishes no symbol at all, and the ratio columns publish
|
|
175
|
+
* junk.
|
|
176
|
+
*
|
|
177
|
+
* A trailing `(h6)` is part of the shank tolerance class and not of the symbol,
|
|
178
|
+
* so `D2(h6)` and `D2` are the same column.
|
|
179
|
+
*/
|
|
180
|
+
export function symbolOf(sub) {
|
|
181
|
+
if (sub === null)
|
|
182
|
+
return null;
|
|
183
|
+
const match = /^([A-Z]+\d*)/.exec(sub);
|
|
184
|
+
return match?.[1] ?? null;
|
|
185
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Harvey Tool — miniature end mills and keyseat cutters, from a printed-catalog
|
|
3
|
+
* table rendered as a web page.
|
|
4
|
+
*
|
|
5
|
+
* No JSON API and no XHR: a product page inlines its entire variant table as a
|
|
6
|
+
* JavaScript literal beside a `<thead>` that names the columns, and the two are
|
|
7
|
+
* joined by position. `literal.ts`, `header.ts`, `lexicon.ts` and `value.ts` are
|
|
8
|
+
* pure and hold the parsing risk; `scrape.ts` and `catalog.ts` are the only
|
|
9
|
+
* modules that read through a `Fetcher`.
|
|
10
|
+
*
|
|
11
|
+
* The one structural thing to know before reading any of it: **one HTML row is
|
|
12
|
+
* up to nine orderable parts.** See `scrape.ts`, and
|
|
13
|
+
* `docs/HARVEY_PRODUCT_TABLE.md` for the transport.
|
|
14
|
+
*/
|
|
15
|
+
export * from './catalog.js';
|
|
16
|
+
export * from './header.js';
|
|
17
|
+
export * from './lexicon.js';
|
|
18
|
+
export * from './literal.js';
|
|
19
|
+
export * from './records.js';
|
|
20
|
+
export * from './scrape.js';
|
|
21
|
+
export * from './value.js';
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Harvey Tool — miniature end mills and keyseat cutters, from a printed-catalog
|
|
3
|
+
* table rendered as a web page.
|
|
4
|
+
*
|
|
5
|
+
* No JSON API and no XHR: a product page inlines its entire variant table as a
|
|
6
|
+
* JavaScript literal beside a `<thead>` that names the columns, and the two are
|
|
7
|
+
* joined by position. `literal.ts`, `header.ts`, `lexicon.ts` and `value.ts` are
|
|
8
|
+
* pure and hold the parsing risk; `scrape.ts` and `catalog.ts` are the only
|
|
9
|
+
* modules that read through a `Fetcher`.
|
|
10
|
+
*
|
|
11
|
+
* The one structural thing to know before reading any of it: **one HTML row is
|
|
12
|
+
* up to nine orderable parts.** See `scrape.ts`, and
|
|
13
|
+
* `docs/HARVEY_PRODUCT_TABLE.md` for the transport.
|
|
14
|
+
*/
|
|
15
|
+
export * from './catalog.js';
|
|
16
|
+
export * from './header.js';
|
|
17
|
+
export * from './lexicon.js';
|
|
18
|
+
export * from './literal.js';
|
|
19
|
+
export * from './records.js';
|
|
20
|
+
export * from './scrape.js';
|
|
21
|
+
export * from './value.js';
|