@toolpath/tool-scraper 0.1.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/README.md +41 -11
  2. package/dist/columns.d.ts +62 -0
  3. package/dist/columns.js +62 -0
  4. package/dist/conventions.d.ts +89 -15
  5. package/dist/conventions.js +108 -15
  6. package/dist/families/emuge.d.ts +183 -0
  7. package/dist/families/emuge.js +159 -0
  8. package/dist/families/harvey.d.ts +1782 -0
  9. package/dist/families/harvey.js +1328 -0
  10. package/dist/families/index.js +5 -2
  11. package/dist/families/kennametal.d.ts +21 -0
  12. package/dist/families/kennametal.js +10 -0
  13. package/dist/families/maritool.d.ts +120 -0
  14. package/dist/families/maritool.js +175 -0
  15. package/dist/family.d.ts +35 -1
  16. package/dist/family.js +28 -0
  17. package/dist/identity.d.ts +18 -0
  18. package/dist/identity.js +46 -0
  19. package/dist/index.d.ts +16 -0
  20. package/dist/index.js +16 -0
  21. package/dist/measure.d.ts +112 -0
  22. package/dist/measure.js +130 -0
  23. package/dist/node/cli.d.ts +3 -0
  24. package/dist/node/cli.js +152 -2
  25. package/dist/node/main.js +0 -0
  26. package/dist/records.d.ts +169 -12
  27. package/dist/records.js +116 -5
  28. package/dist/registry.d.ts +31 -1
  29. package/dist/registry.js +40 -1
  30. package/dist/scrape.d.ts +15 -0
  31. package/dist/scrape.js +24 -0
  32. package/dist/vendors/destinytool/records.d.ts +13 -3
  33. package/dist/vendors/destinytool/records.js +41 -37
  34. package/dist/vendors/emuge/index.d.ts +17 -0
  35. package/dist/vendors/emuge/index.js +17 -0
  36. package/dist/vendors/emuge/records.d.ts +150 -0
  37. package/dist/vendors/emuge/records.js +375 -0
  38. package/dist/vendors/emuge/scrape.d.ts +227 -0
  39. package/dist/vendors/emuge/scrape.js +358 -0
  40. package/dist/vendors/emuge/value.d.ts +75 -0
  41. package/dist/vendors/emuge/value.js +116 -0
  42. package/dist/vendors/harvey/catalog.d.ts +53 -0
  43. package/dist/vendors/harvey/catalog.js +120 -0
  44. package/dist/vendors/harvey/header.d.ts +89 -0
  45. package/dist/vendors/harvey/header.js +185 -0
  46. package/dist/vendors/harvey/index.d.ts +21 -0
  47. package/dist/vendors/harvey/index.js +21 -0
  48. package/dist/vendors/harvey/lexicon.d.ts +73 -0
  49. package/dist/vendors/harvey/lexicon.js +126 -0
  50. package/dist/vendors/harvey/literal.d.ts +68 -0
  51. package/dist/vendors/harvey/literal.js +214 -0
  52. package/dist/vendors/harvey/records.d.ts +79 -0
  53. package/dist/vendors/harvey/records.js +163 -0
  54. package/dist/vendors/harvey/scrape.d.ts +187 -0
  55. package/dist/vendors/harvey/scrape.js +483 -0
  56. package/dist/vendors/harvey/value.d.ts +74 -0
  57. package/dist/vendors/harvey/value.js +119 -0
  58. package/dist/vendors/kennametal/records.d.ts +13 -4
  59. package/dist/vendors/kennametal/records.js +61 -26
  60. package/dist/vendors/maritool/catalog.d.ts +81 -0
  61. package/dist/vendors/maritool/catalog.js +132 -0
  62. package/dist/vendors/maritool/index.d.ts +10 -0
  63. package/dist/vendors/maritool/index.js +10 -0
  64. package/dist/vendors/maritool/scrape.d.ts +297 -0
  65. package/dist/vendors/maritool/scrape.js +593 -0
  66. package/dist/vendors/regofix/scrape.d.ts +8 -11
  67. package/dist/vendors/regofix/scrape.js +21 -36
  68. package/package.json +31 -7
@@ -0,0 +1,120 @@
1
+ /**
2
+ * The category tree -> the product pages under it.
3
+ *
4
+ * Harvey publishes no sitemap and no flat index: `robots.txt`, `sitemap.xml` and
5
+ * `sitemap_index.xml` all 404, and `/products/all-products` lists the seven top
6
+ * categories and no products. Walking the catalog grid is the only way to find
7
+ * out what exists — see `docs/HARVEY_PRODUCT_TABLE.md` §1.1 and §1.6.
8
+ *
9
+ * Nothing in a scrape needs this: `families/harvey.ts` names all 52 product
10
+ * pages, and a family scrape fetches exactly one. It is here for the maintenance
11
+ * question that table cannot answer — *has Harvey added a page?* — and the CLI
12
+ * exposes it as `harvey --catalog` for that purpose.
13
+ *
14
+ * A page carries subcategory links or product links, never both, so the
15
+ * recursion terminates where products appear. Both selectors were verified
16
+ * against all 33 category pages the four roots reach.
17
+ */
18
+ import { Parser } from 'htmlparser2';
19
+ import { compare } from '../../order.js';
20
+ import { consoleWarn, pause, REQUEST_DELAY_MS } from '../../scrape.js';
21
+ import { BASE } from './scrape.js';
22
+ /**
23
+ * The four category trees this adapter covers.
24
+ *
25
+ * Harvey's catalog is far larger; these are the trees asked for, and adding one
26
+ * is a decision rather than a widening of a pattern — see the package's rule on
27
+ * not adding vendor scope without being asked.
28
+ */
29
+ export const CATEGORY_ROOTS = [
30
+ '/products/miniature-end-mills/ball',
31
+ '/products/miniature-end-mills/corner-radius',
32
+ '/products/miniature-end-mills/square',
33
+ '/products/specialty-profiles/keyseat-cutters',
34
+ ];
35
+ /** The class marking a link to a subcategory in the catalog grid. */
36
+ const SUBCATEGORY_CLASS = 'img-wrapper';
37
+ /** The classes marking the wrapper around a link to one product. */
38
+ const PRODUCT_CLASSES = ['col-md-4', 'col-12', 'item-wrapper'];
39
+ function classes(attribs) {
40
+ return (attribs['class'] ?? '').split(/\s+/).filter(Boolean);
41
+ }
42
+ /**
43
+ * The subcategory and product links on one catalog page.
44
+ *
45
+ * Product links are identified by their **wrapper** rather than by the anchor,
46
+ * because the anchor carries no class of its own — the grid renders
47
+ * `<div class="col-md-4 col-12 item-wrapper"><a href="/products/...">`. A rule
48
+ * that took every `/products/` anchor on the page would take the breadcrumb and
49
+ * the footer with it.
50
+ */
51
+ export function parseCategoryPage(html) {
52
+ const subcategories = new Set();
53
+ const products = new Set();
54
+ // A depth counter rather than a flag, because the wrapper holds nested divs
55
+ // and a flag would clear on the first inner close tag.
56
+ let inProduct = 0;
57
+ const parser = new Parser({
58
+ onopentag: (tag, attribs) => {
59
+ if (inProduct > 0) {
60
+ inProduct++;
61
+ if (tag === 'a') {
62
+ const href = attribs['href'] ?? '';
63
+ if (href.startsWith('/products/'))
64
+ products.add(href);
65
+ }
66
+ return;
67
+ }
68
+ if (tag === 'div' && PRODUCT_CLASSES.every((c) => classes(attribs).includes(c))) {
69
+ inProduct = 1;
70
+ return;
71
+ }
72
+ if (tag === 'a' && classes(attribs).includes(SUBCATEGORY_CLASS)) {
73
+ const href = attribs['href'] ?? '';
74
+ if (href.startsWith('/products/'))
75
+ subcategories.add(href);
76
+ }
77
+ },
78
+ onclosetag: () => {
79
+ if (inProduct > 0)
80
+ inProduct--;
81
+ },
82
+ }, { decodeEntities: true });
83
+ parser.write(html);
84
+ parser.end();
85
+ return {
86
+ subcategories: [...subcategories].sort(compare),
87
+ products: [...products].sort(compare),
88
+ };
89
+ }
90
+ /**
91
+ * Every product page reachable from `roots`, sorted.
92
+ *
93
+ * Sequential and paced by the package's shared politeness delay. The whole walk
94
+ * is about 33 requests; nothing here needs concurrency, and Cloudflare fronts
95
+ * this site, so raising request volume is the only real risk.
96
+ */
97
+ export async function discoverProducts(fetcher, roots = CATEGORY_ROOTS, options = {}) {
98
+ const { warn = consoleWarn, delayMs = REQUEST_DELAY_MS } = options;
99
+ const products = new Set();
100
+ const seen = new Set();
101
+ const queue = [...roots];
102
+ while (queue.length > 0) {
103
+ const path = queue.shift();
104
+ if (seen.has(path))
105
+ continue;
106
+ seen.add(path);
107
+ const { subcategories, products: found } = parseCategoryPage(await fetcher.text(BASE + path));
108
+ if (subcategories.length === 0 && found.length === 0) {
109
+ warn(` WARNING: ${path} links to neither a subcategory nor a product`);
110
+ }
111
+ for (const product of found)
112
+ products.add(product);
113
+ for (const child of subcategories) {
114
+ if (child !== path)
115
+ queue.push(child);
116
+ }
117
+ await pause(delayMs);
118
+ }
119
+ return [...products].sort(compare);
120
+ }
@@ -0,0 +1,89 @@
1
+ /**
2
+ * A product table's two-row `<thead>` -> one flat column per `cols<N>` entry.
3
+ *
4
+ * Harvey publishes a table's header in the DOM and its data in a JavaScript
5
+ * literal, and the only thing joining them is position: `cols1` is
6
+ * `[{data:"a0"}, {data:"a1"}, ...]` and its Nth entry is the Nth column of the
7
+ * flattened header. That alignment is checked on every table — see
8
+ * `scrape.ts` — because a header that has drifted by one column produces a CSV
9
+ * where every dimension is labelled as its neighbour.
10
+ *
11
+ * ## Flattening
12
+ *
13
+ * The header is exactly two `<tr>`s. A top `<th>` with `rowspan >= 2` is its own
14
+ * column and has no sub-label (only "Add to Cart" is like this); otherwise it
15
+ * consumes `colspan` cells from the second row as its sub-labels. Verified
16
+ * against all 80 tables on the 52 product pages.
17
+ *
18
+ * ## Two things about the text
19
+ *
20
+ * **`<br>` is a space.** `CUTTER <br/>DIAMETER` is the same label as
21
+ * `CUTTER DIAMETER`, and collapsing the tag away instead yields
22
+ * `CUTTERDIAMETER` — a 51st header shape that exists only in the parser.
23
+ *
24
+ * **A sub-label's tolerance is excluded, not stripped afterwards.** The
25
+ * tolerance is rendered in its own `div.hpc-inline`, so skipping that subtree
26
+ * gives `D1` where reading the whole cell gives `D1+.0005"-.0005"`. It matters
27
+ * more than tidiness: Harvey's tolerance text carries typos — `D1+0005"-.0005"`,
28
+ * `R+.001"-001"`, `L2.020"-.000"` are all real — and every one of them would be
29
+ * another spelling for a post-hoc stripper to get right.
30
+ */
31
+ /** The class of the `<div>` holding a sub-label's published tolerance. */
32
+ export declare const TOLERANCE_CLASS = "hpc-inline";
33
+ /** The class Harvey renders an unlabelled annotation column's header with. */
34
+ export declare const HIDDEN_CLASS = "white-text";
35
+ /** True where a header label says nothing about what the column holds. */
36
+ export declare function isJunkLabel(label: string): boolean;
37
+ /** One `<th>` as parsed: its text, its span, and its class attribute. */
38
+ interface HeaderCell {
39
+ text: string;
40
+ colspan: number;
41
+ rowspan: number;
42
+ className: string;
43
+ }
44
+ /** One data column of a table, as the header describes it. */
45
+ export interface HeaderColumn {
46
+ /** The top row's label, `<br>`s collapsed to spaces. */
47
+ top: string;
48
+ /** The second row's label with its tolerance removed, or null under a rowspan. */
49
+ sub: string | null;
50
+ /** The top `<th>`'s class attribute — `white-text` marks a hidden column. */
51
+ topClass: string;
52
+ /** How many columns the top `<th>` spanned. 1 unless it is a coating group. */
53
+ span: number;
54
+ /** This column's index inside that span. 0 on a column of its own. */
55
+ slot: number;
56
+ }
57
+ /**
58
+ * The `<th>` rows of the `<thead>` of the table with this id.
59
+ *
60
+ * Streams the whole document rather than slicing it: a `</table>` inside a cell
61
+ * would end a sliced segment early, and htmlparser2 is already this package's
62
+ * one parser dependency.
63
+ *
64
+ * `decodeEntities` is on because these headers carry `&nbsp;` and `&deg;` —
65
+ * the blank flute sub-label is literally `&nbsp;&nbsp;`, and it has to reach
66
+ * {@link isJunkLabel} as an empty string rather than as two characters.
67
+ */
68
+ export declare function parseHeadRows(html: string, tableId: string): HeaderCell[][];
69
+ /**
70
+ * One table's header, flattened to one entry per data column.
71
+ *
72
+ * Throws when the table is not in the document, rather than returning an empty
73
+ * header: a page that stopped serving a table it declares in `cols<N>` is a
74
+ * response that changed shape, and a zero-column table would reach the caller
75
+ * looking like a family the vendor discontinued.
76
+ */
77
+ export declare function flatHeader(html: string, tableId: string): HeaderColumn[];
78
+ /**
79
+ * The vendor's own symbol for a dimension column — `D1`, `L2`, `R`, `A1`, `#`.
80
+ *
81
+ * Null where the sub-label carries none, which is a real state rather than a
82
+ * fault: `NECK DIA.` publishes no symbol at all, and the ratio columns publish
83
+ * junk.
84
+ *
85
+ * A trailing `(h6)` is part of the shank tolerance class and not of the symbol,
86
+ * so `D2(h6)` and `D2` are the same column.
87
+ */
88
+ export declare function symbolOf(sub: string | null): string | null;
89
+ export {};
@@ -0,0 +1,185 @@
1
+ /**
2
+ * A product table's two-row `<thead>` -> one flat column per `cols<N>` entry.
3
+ *
4
+ * Harvey publishes a table's header in the DOM and its data in a JavaScript
5
+ * literal, and the only thing joining them is position: `cols1` is
6
+ * `[{data:"a0"}, {data:"a1"}, ...]` and its Nth entry is the Nth column of the
7
+ * flattened header. That alignment is checked on every table — see
8
+ * `scrape.ts` — because a header that has drifted by one column produces a CSV
9
+ * where every dimension is labelled as its neighbour.
10
+ *
11
+ * ## Flattening
12
+ *
13
+ * The header is exactly two `<tr>`s. A top `<th>` with `rowspan >= 2` is its own
14
+ * column and has no sub-label (only "Add to Cart" is like this); otherwise it
15
+ * consumes `colspan` cells from the second row as its sub-labels. Verified
16
+ * against all 80 tables on the 52 product pages.
17
+ *
18
+ * ## Two things about the text
19
+ *
20
+ * **`<br>` is a space.** `CUTTER <br/>DIAMETER` is the same label as
21
+ * `CUTTER DIAMETER`, and collapsing the tag away instead yields
22
+ * `CUTTERDIAMETER` — a 51st header shape that exists only in the parser.
23
+ *
24
+ * **A sub-label's tolerance is excluded, not stripped afterwards.** The
25
+ * tolerance is rendered in its own `div.hpc-inline`, so skipping that subtree
26
+ * gives `D1` where reading the whole cell gives `D1+.0005"-.0005"`. It matters
27
+ * more than tidiness: Harvey's tolerance text carries typos — `D1+0005"-.0005"`,
28
+ * `R+.001"-001"`, `L2.020"-.000"` are all real — and every one of them would be
29
+ * another spelling for a post-hoc stripper to get right.
30
+ */
31
+ import { Parser } from 'htmlparser2';
32
+ import { VendorResponseError } from '../../errors.js';
33
+ /** The class of the `<div>` holding a sub-label's published tolerance. */
34
+ export const TOLERANCE_CLASS = 'hpc-inline';
35
+ /** The class Harvey renders an unlabelled annotation column's header with. */
36
+ export const HIDDEN_CLASS = 'white-text';
37
+ /** Sub-label text that carries nothing. `&nbsp;` collapses to empty here. */
38
+ const JUNK = new Set(['', '.', '"', 'X']);
39
+ /** True where a header label says nothing about what the column holds. */
40
+ export function isJunkLabel(label) {
41
+ return JUNK.has(label);
42
+ }
43
+ /** `" a b \n"` -> `"a b"`. */
44
+ function collapse(text) {
45
+ return text.split(/\s+/).filter(Boolean).join(' ');
46
+ }
47
+ /**
48
+ * The `<th>` rows of the `<thead>` of the table with this id.
49
+ *
50
+ * Streams the whole document rather than slicing it: a `</table>` inside a cell
51
+ * would end a sliced segment early, and htmlparser2 is already this package's
52
+ * one parser dependency.
53
+ *
54
+ * `decodeEntities` is on because these headers carry `&nbsp;` and `&deg;` —
55
+ * the blank flute sub-label is literally `&nbsp;&nbsp;`, and it has to reach
56
+ * {@link isJunkLabel} as an empty string rather than as two characters.
57
+ */
58
+ export function parseHeadRows(html, tableId) {
59
+ const rows = [];
60
+ let inTable = false;
61
+ let inHead = false;
62
+ let row = null;
63
+ let cell = null;
64
+ // A depth counter rather than a flag: the tolerance is a `div` inside a `div`,
65
+ // and a flag would re-enable the text on the inner close tag.
66
+ let tolerance = 0;
67
+ let depth = 0;
68
+ const parser = new Parser({
69
+ onopentag: (tag, attribs) => {
70
+ if (tag === 'table') {
71
+ inTable = attribs['id'] === tableId;
72
+ return;
73
+ }
74
+ if (!inTable)
75
+ return;
76
+ if (tag === 'thead') {
77
+ inHead = true;
78
+ }
79
+ else if (inHead && tag === 'tr') {
80
+ row = [];
81
+ }
82
+ else if (inHead && tag === 'th' && row !== null) {
83
+ cell = {
84
+ text: '',
85
+ colspan: Number.parseInt(attribs['colspan'] ?? '1', 10) || 1,
86
+ rowspan: Number.parseInt(attribs['rowspan'] ?? '1', 10) || 1,
87
+ className: attribs['class'] ?? '',
88
+ };
89
+ tolerance = 0;
90
+ depth = 0;
91
+ }
92
+ else if (cell !== null) {
93
+ depth++;
94
+ if (tolerance === 0 && (attribs['class'] ?? '').split(/\s+/).includes(TOLERANCE_CLASS)) {
95
+ tolerance = depth;
96
+ }
97
+ if (tag === 'br')
98
+ cell.text += ' ';
99
+ }
100
+ },
101
+ ontext: (text) => {
102
+ if (cell !== null && tolerance === 0)
103
+ cell.text += text;
104
+ },
105
+ onclosetag: (tag) => {
106
+ if (!inTable)
107
+ return;
108
+ if (tag === 'th' && cell !== null) {
109
+ row?.push({ ...cell, text: collapse(cell.text) });
110
+ cell = null;
111
+ }
112
+ else if (tag === 'tr' && row !== null) {
113
+ rows.push(row);
114
+ row = null;
115
+ }
116
+ else if (tag === 'thead') {
117
+ inHead = false;
118
+ // Everything after the head is data the literal already carries.
119
+ inTable = false;
120
+ }
121
+ else if (cell !== null) {
122
+ if (tolerance === depth)
123
+ tolerance = 0;
124
+ depth--;
125
+ }
126
+ },
127
+ }, { decodeEntities: true });
128
+ parser.write(html);
129
+ parser.end();
130
+ return rows;
131
+ }
132
+ /**
133
+ * One table's header, flattened to one entry per data column.
134
+ *
135
+ * Throws when the table is not in the document, rather than returning an empty
136
+ * header: a page that stopped serving a table it declares in `cols<N>` is a
137
+ * response that changed shape, and a zero-column table would reach the caller
138
+ * looking like a family the vendor discontinued.
139
+ */
140
+ export function flatHeader(html, tableId) {
141
+ const rows = parseHeadRows(html, tableId);
142
+ if (rows.length === 0) {
143
+ throw new VendorResponseError(tableId, 'has no <thead> on the page — the table changed shape');
144
+ }
145
+ const [top = [], sub = []] = rows;
146
+ const columns = [];
147
+ let next = 0;
148
+ for (const cell of top) {
149
+ if (cell.rowspan >= 2) {
150
+ columns.push({ top: cell.text, sub: null, topClass: cell.className, span: 1, slot: 0 });
151
+ continue;
152
+ }
153
+ for (let slot = 0; slot < cell.colspan; slot++) {
154
+ columns.push({
155
+ top: cell.text,
156
+ sub: sub[next]?.text ?? '',
157
+ topClass: cell.className,
158
+ span: cell.colspan,
159
+ slot,
160
+ });
161
+ next++;
162
+ }
163
+ }
164
+ if (next > sub.length) {
165
+ throw new VendorResponseError(tableId, `spans ${next} sub-labels across its top row but the second row has ` +
166
+ `${sub.length} — the header changed shape`);
167
+ }
168
+ return columns;
169
+ }
170
+ /**
171
+ * The vendor's own symbol for a dimension column — `D1`, `L2`, `R`, `A1`, `#`.
172
+ *
173
+ * Null where the sub-label carries none, which is a real state rather than a
174
+ * fault: `NECK DIA.` publishes no symbol at all, and the ratio columns publish
175
+ * junk.
176
+ *
177
+ * A trailing `(h6)` is part of the shank tolerance class and not of the symbol,
178
+ * so `D2(h6)` and `D2` are the same column.
179
+ */
180
+ export function symbolOf(sub) {
181
+ if (sub === null)
182
+ return null;
183
+ const match = /^([A-Z]+\d*)/.exec(sub);
184
+ return match?.[1] ?? null;
185
+ }
@@ -0,0 +1,21 @@
1
+ /**
2
+ * Harvey Tool — miniature end mills and keyseat cutters, from a printed-catalog
3
+ * table rendered as a web page.
4
+ *
5
+ * No JSON API and no XHR: a product page inlines its entire variant table as a
6
+ * JavaScript literal beside a `<thead>` that names the columns, and the two are
7
+ * joined by position. `literal.ts`, `header.ts`, `lexicon.ts` and `value.ts` are
8
+ * pure and hold the parsing risk; `scrape.ts` and `catalog.ts` are the only
9
+ * modules that read through a `Fetcher`.
10
+ *
11
+ * The one structural thing to know before reading any of it: **one HTML row is
12
+ * up to nine orderable parts.** See `scrape.ts`, and
13
+ * `docs/HARVEY_PRODUCT_TABLE.md` for the transport.
14
+ */
15
+ export * from './catalog.js';
16
+ export * from './header.js';
17
+ export * from './lexicon.js';
18
+ export * from './literal.js';
19
+ export * from './records.js';
20
+ export * from './scrape.js';
21
+ export * from './value.js';
@@ -0,0 +1,21 @@
1
+ /**
2
+ * Harvey Tool — miniature end mills and keyseat cutters, from a printed-catalog
3
+ * table rendered as a web page.
4
+ *
5
+ * No JSON API and no XHR: a product page inlines its entire variant table as a
6
+ * JavaScript literal beside a `<thead>` that names the columns, and the two are
7
+ * joined by position. `literal.ts`, `header.ts`, `lexicon.ts` and `value.ts` are
8
+ * pure and hold the parsing risk; `scrape.ts` and `catalog.ts` are the only
9
+ * modules that read through a `Fetcher`.
10
+ *
11
+ * The one structural thing to know before reading any of it: **one HTML row is
12
+ * up to nine orderable parts.** See `scrape.ts`, and
13
+ * `docs/HARVEY_PRODUCT_TABLE.md` for the transport.
14
+ */
15
+ export * from './catalog.js';
16
+ export * from './header.js';
17
+ export * from './lexicon.js';
18
+ export * from './literal.js';
19
+ export * from './records.js';
20
+ export * from './scrape.js';
21
+ export * from './value.js';
@@ -0,0 +1,73 @@
1
+ /**
2
+ * What Harvey's own column labels mean, and which of them carry a unit.
3
+ *
4
+ * 50 distinct header shapes across the 52 product pages collapse to the table
5
+ * below, because the variance is in *spelling* rather than in vocabulary:
6
+ * `CUTTER DIA.` and `CUTTER DIAMETER` are the same column, and the keyseat
7
+ * families call the length of cut `CUTTER WIDTH`.
8
+ *
9
+ * **This module does not map a label to a canonical ISO name.** That mapping is
10
+ * per family and lives in `families/harvey.ts`, where it belongs: a family
11
+ * declares which of *its* labels is `LCF`, and the core appends the unit suffix.
12
+ * What is left over — is this label one this package has seen, and does its
13
+ * column carry a length — is the same on every page, and is here.
14
+ *
15
+ * ## An unknown label is a hard failure
16
+ *
17
+ * A label nothing here recognises stops the scrape naming the family. The
18
+ * alternative is a column that reaches the CSV under whatever Harvey renamed it
19
+ * to, silently no longer matching the family's column map, and a family whose
20
+ * `DC` is quietly absent from every record. The cost of the strict rule is that
21
+ * a vendor's copy edit breaks a scrape; the cost of the loose one is that it
22
+ * does not.
23
+ */
24
+ /**
25
+ * What a column holds, which is only ever asked so that a length gets a `_mm` /
26
+ * `_in` suffix and a tooth count does not.
27
+ *
28
+ * `code` is for a column whose values are not numbers at all — the keyseat
29
+ * `TYPE`, whose `I`/`II`/`III` nobody at Harvey has yet explained.
30
+ */
31
+ export type LabelKind = 'dimension' | 'count' | 'angle' | 'code';
32
+ /**
33
+ * Every geometry top label the 52 product pages publish, and what it holds.
34
+ *
35
+ * Written out rather than pattern-matched. A regex over "anything containing
36
+ * DIA" would have swallowed `RADIAL DOC` and `NECK DIA.` into the same rule and
37
+ * quietly given one of them the other's meaning, and the whole point of the
38
+ * strict check below is that a label this package has not seen gets looked at
39
+ * by a person.
40
+ */
41
+ export declare const LABELS: Readonly<Record<string, LabelKind>>;
42
+ /** The top label of the column holding the add-to-cart control. Dropped. */
43
+ export declare const CART_LABEL = "Add to Cart";
44
+ /** The top label of the column stating a row's flute count, where there is one. */
45
+ export declare const FLUTES_LABEL = "FLUTES";
46
+ /** The sub-label a coating group uses when the row states the flute count. */
47
+ export declare const TOOL_NUMBER_LABEL = "TOOL #";
48
+ /** The sub-label ending every coating group. */
49
+ export declare const PRICE_LABEL = "PRICE";
50
+ /**
51
+ * The eight coating names Harvey publishes across the whole catalog.
52
+ *
53
+ * A closed list, checked, because the coating is *synthesised* from a column's
54
+ * position rather than lifted from a cell — nothing in a row says which coating
55
+ * a tool number belongs to. A ninth coating is Harvey extending its catalog and
56
+ * is worth a person looking; a coating group whose header stopped being a
57
+ * coating is the grid having shifted, and that is the failure this catches.
58
+ */
59
+ export declare const COATINGS: readonly string[];
60
+ /** The flute count a group sub-label states, or null where it states none. */
61
+ export declare function flutesInLabel(sub: string): number | null;
62
+ /**
63
+ * What the column under `label` holds, refusing a label nothing here knows.
64
+ *
65
+ * `family` is in the message because that is the one thing the person reading
66
+ * the failure needs and cannot work out from the label: 52 pages publish these
67
+ * labels, and only one of them changed.
68
+ */
69
+ export declare function labelKind(family: string, label: string): LabelKind;
70
+ /** Whether the column under `label` gets a `_mm` / `_in` suffix. */
71
+ export declare function isDimensional(family: string, label: string): boolean;
72
+ /** Refuse a coating group header that is not one of the eight. */
73
+ export declare function checkCoating(family: string, coating: string): string;
@@ -0,0 +1,126 @@
1
+ /**
2
+ * What Harvey's own column labels mean, and which of them carry a unit.
3
+ *
4
+ * 50 distinct header shapes across the 52 product pages collapse to the table
5
+ * below, because the variance is in *spelling* rather than in vocabulary:
6
+ * `CUTTER DIA.` and `CUTTER DIAMETER` are the same column, and the keyseat
7
+ * families call the length of cut `CUTTER WIDTH`.
8
+ *
9
+ * **This module does not map a label to a canonical ISO name.** That mapping is
10
+ * per family and lives in `families/harvey.ts`, where it belongs: a family
11
+ * declares which of *its* labels is `LCF`, and the core appends the unit suffix.
12
+ * What is left over — is this label one this package has seen, and does its
13
+ * column carry a length — is the same on every page, and is here.
14
+ *
15
+ * ## An unknown label is a hard failure
16
+ *
17
+ * A label nothing here recognises stops the scrape naming the family. The
18
+ * alternative is a column that reaches the CSV under whatever Harvey renamed it
19
+ * to, silently no longer matching the family's column map, and a family whose
20
+ * `DC` is quietly absent from every record. The cost of the strict rule is that
21
+ * a vendor's copy edit breaks a scrape; the cost of the loose one is that it
22
+ * does not.
23
+ */
24
+ import { ScraperConfigError } from '../../errors.js';
25
+ /**
26
+ * Every geometry top label the 52 product pages publish, and what it holds.
27
+ *
28
+ * Written out rather than pattern-matched. A regex over "anything containing
29
+ * DIA" would have swallowed `RADIAL DOC` and `NECK DIA.` into the same rule and
30
+ * quietly given one of them the other's meaning, and the whole point of the
31
+ * strict check below is that a label this package has not seen gets looked at
32
+ * by a person.
33
+ */
34
+ export const LABELS = {
35
+ // Mapped to canonical names by each family's own column map.
36
+ 'CUTTER DIA.': 'dimension',
37
+ 'CUTTER DIAMETER': 'dimension',
38
+ LOC: 'dimension',
39
+ 'LENGTH OF CUT': 'dimension',
40
+ 'CUTTER WIDTH': 'dimension',
41
+ 'SHANK DIA.': 'dimension',
42
+ 'SHANK DIAMETER': 'dimension',
43
+ OAL: 'dimension',
44
+ 'OVERALL LENGTH': 'dimension',
45
+ 'CORNER RADIUS': 'dimension',
46
+ RADIUS: 'dimension',
47
+ 'NECK DIA.': 'dimension',
48
+ 'NECK LENGTH': 'dimension',
49
+ 'OVERALL REACH': 'dimension',
50
+ FLUTES: 'count',
51
+ // Kept verbatim and mapped to nothing. Each is either a measurement this
52
+ // package has no canonical name for, or one whose meaning Harvey has not
53
+ // stated — see `docs/HARVEY_PRODUCT_TABLE.md` §5.1 and §6.
54
+ 'RADIAL DOC*': 'dimension',
55
+ 'Radial DOC*': 'dimension',
56
+ 'Radial DOC**': 'dimension',
57
+ 'Interference Depth At Wall Angle*': 'dimension',
58
+ 'RIGHT HAND TEETH': 'count',
59
+ 'LEFT HAND TEETH': 'count',
60
+ 'ANGLE PER SIDE': 'angle',
61
+ 'EFFECTIVE WALL ANGLE*': 'angle',
62
+ 'EFF WALL ANGLE': 'angle',
63
+ TYPE: 'code',
64
+ };
65
+ /** The top label of the column holding the add-to-cart control. Dropped. */
66
+ export const CART_LABEL = 'Add to Cart';
67
+ /** The top label of the column stating a row's flute count, where there is one. */
68
+ export const FLUTES_LABEL = 'FLUTES';
69
+ /** The sub-label a coating group uses when the row states the flute count. */
70
+ export const TOOL_NUMBER_LABEL = 'TOOL #';
71
+ /** The sub-label ending every coating group. */
72
+ export const PRICE_LABEL = 'PRICE';
73
+ /** `2 FL`, `4FL`, `5 FL` — a per-part flute count in a group's sub-label. */
74
+ const FLUTE_LABEL = /^(\d+)\s*FL$/i;
75
+ /**
76
+ * The eight coating names Harvey publishes across the whole catalog.
77
+ *
78
+ * A closed list, checked, because the coating is *synthesised* from a column's
79
+ * position rather than lifted from a cell — nothing in a row says which coating
80
+ * a tool number belongs to. A ninth coating is Harvey extending its catalog and
81
+ * is worth a person looking; a coating group whose header stopped being a
82
+ * coating is the grid having shifted, and that is the failure this catches.
83
+ */
84
+ export const COATINGS = [
85
+ 'UNCOATED',
86
+ 'AlTiN COATED',
87
+ 'AMORPHOUS DIAMOND COATED',
88
+ 'AlTiN NANO COATED',
89
+ 'Ti NANO COATED',
90
+ 'TiB2 COATED',
91
+ 'BALL END UNCOATED',
92
+ 'AlTiN NANO BALL END COATED',
93
+ ];
94
+ /** The flute count a group sub-label states, or null where it states none. */
95
+ export function flutesInLabel(sub) {
96
+ const match = FLUTE_LABEL.exec(sub);
97
+ return match ? Number.parseInt(match[1], 10) : null;
98
+ }
99
+ /**
100
+ * What the column under `label` holds, refusing a label nothing here knows.
101
+ *
102
+ * `family` is in the message because that is the one thing the person reading
103
+ * the failure needs and cannot work out from the label: 52 pages publish these
104
+ * labels, and only one of them changed.
105
+ */
106
+ export function labelKind(family, label) {
107
+ const kind = LABELS[label];
108
+ if (kind === undefined) {
109
+ throw new ScraperConfigError(family, `publishes a column headed ${JSON.stringify(label)}, which this adapter ` +
110
+ `does not know — add it to vendors/harvey/lexicon.ts with what it ` +
111
+ `holds, and to the family's column map if it is a canonical field`);
112
+ }
113
+ return kind;
114
+ }
115
+ /** Whether the column under `label` gets a `_mm` / `_in` suffix. */
116
+ export function isDimensional(family, label) {
117
+ return labelKind(family, label) === 'dimension';
118
+ }
119
+ /** Refuse a coating group header that is not one of the eight. */
120
+ export function checkCoating(family, coating) {
121
+ if (!COATINGS.includes(coating)) {
122
+ throw new ScraperConfigError(family, `has a tool-number group headed ${JSON.stringify(coating)}, which is not ` +
123
+ `one of the coatings this adapter knows (${COATINGS.join(', ')})`);
124
+ }
125
+ return coating;
126
+ }