@toolpath/tool-scraper 0.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -11
- package/dist/columns.d.ts +62 -0
- package/dist/columns.js +62 -0
- package/dist/conventions.d.ts +89 -15
- package/dist/conventions.js +108 -15
- package/dist/families/emuge.d.ts +183 -0
- package/dist/families/emuge.js +159 -0
- package/dist/families/harvey.d.ts +1782 -0
- package/dist/families/harvey.js +1328 -0
- package/dist/families/index.js +5 -2
- package/dist/families/kennametal.d.ts +21 -0
- package/dist/families/kennametal.js +10 -0
- package/dist/families/maritool.d.ts +120 -0
- package/dist/families/maritool.js +175 -0
- package/dist/family.d.ts +35 -1
- package/dist/family.js +28 -0
- package/dist/identity.d.ts +18 -0
- package/dist/identity.js +46 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +16 -0
- package/dist/measure.d.ts +112 -0
- package/dist/measure.js +130 -0
- package/dist/node/cli.d.ts +3 -0
- package/dist/node/cli.js +152 -2
- package/dist/node/main.js +0 -0
- package/dist/records.d.ts +169 -12
- package/dist/records.js +116 -5
- package/dist/registry.d.ts +31 -1
- package/dist/registry.js +40 -1
- package/dist/scrape.d.ts +15 -0
- package/dist/scrape.js +24 -0
- package/dist/vendors/destinytool/records.d.ts +13 -3
- package/dist/vendors/destinytool/records.js +41 -37
- package/dist/vendors/emuge/index.d.ts +17 -0
- package/dist/vendors/emuge/index.js +17 -0
- package/dist/vendors/emuge/records.d.ts +150 -0
- package/dist/vendors/emuge/records.js +375 -0
- package/dist/vendors/emuge/scrape.d.ts +227 -0
- package/dist/vendors/emuge/scrape.js +358 -0
- package/dist/vendors/emuge/value.d.ts +75 -0
- package/dist/vendors/emuge/value.js +116 -0
- package/dist/vendors/harvey/catalog.d.ts +53 -0
- package/dist/vendors/harvey/catalog.js +120 -0
- package/dist/vendors/harvey/header.d.ts +89 -0
- package/dist/vendors/harvey/header.js +185 -0
- package/dist/vendors/harvey/index.d.ts +21 -0
- package/dist/vendors/harvey/index.js +21 -0
- package/dist/vendors/harvey/lexicon.d.ts +73 -0
- package/dist/vendors/harvey/lexicon.js +126 -0
- package/dist/vendors/harvey/literal.d.ts +68 -0
- package/dist/vendors/harvey/literal.js +214 -0
- package/dist/vendors/harvey/records.d.ts +79 -0
- package/dist/vendors/harvey/records.js +163 -0
- package/dist/vendors/harvey/scrape.d.ts +187 -0
- package/dist/vendors/harvey/scrape.js +483 -0
- package/dist/vendors/harvey/value.d.ts +74 -0
- package/dist/vendors/harvey/value.js +119 -0
- package/dist/vendors/kennametal/records.d.ts +13 -4
- package/dist/vendors/kennametal/records.js +61 -26
- package/dist/vendors/maritool/catalog.d.ts +81 -0
- package/dist/vendors/maritool/catalog.js +132 -0
- package/dist/vendors/maritool/index.d.ts +10 -0
- package/dist/vendors/maritool/index.js +10 -0
- package/dist/vendors/maritool/scrape.d.ts +297 -0
- package/dist/vendors/maritool/scrape.js +593 -0
- package/dist/vendors/regofix/scrape.d.ts +8 -11
- package/dist/vendors/regofix/scrape.js +21 -36
- package/package.json +31 -7
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One display cell -> a number, or an honest reason there is none.
|
|
3
|
+
*
|
|
4
|
+
* Harvey publishes values the way a printed catalog does, because the table is
|
|
5
|
+
* a printed catalog: `.1250 (1/8)` is a decimal with the fraction a machinist
|
|
6
|
+
* orders by beside it, `1-1/2` is a mixed number, `3 mm` is a metric shank on
|
|
7
|
+
* an imperial page, and `-` means the column does not apply to that row.
|
|
8
|
+
*
|
|
9
|
+
* **One rule covers the whole grammar: the value is the leading token, and
|
|
10
|
+
* anything in parentheses is Harvey's own equivalent annotation.** Every shape
|
|
11
|
+
* measured across all 12,799 parts is in `docs/HARVEY_PRODUCT_TABLE.md` §4;
|
|
12
|
+
* this module is that section as code.
|
|
13
|
+
*
|
|
14
|
+
* ## Read `d`, never `v`
|
|
15
|
+
*
|
|
16
|
+
* Each cell also carries `v`, which looks pre-parsed and is the wrong field: it
|
|
17
|
+
* changes unit basis *within a single row*. On the metric ball page, `v` is the
|
|
18
|
+
* inch equivalent for the cutter diameter and the raw millimetre number for the
|
|
19
|
+
* length of cut. Anything reading it gets a clean conversion with wrong numbers
|
|
20
|
+
* in it — the exact failure `conventions.UNIT_SUFFIX` exists to prevent.
|
|
21
|
+
*
|
|
22
|
+
* ## A stated unit wins over the column's
|
|
23
|
+
*
|
|
24
|
+
* 46 cells across six otherwise-imperial families state a metric value outright
|
|
25
|
+
* — `3 mm` in a `SHANK DIAMETER` column on an inch page, a metric-shank tool
|
|
26
|
+
* listed among imperial ones. {@link dimension} converts those into the family's
|
|
27
|
+
* declared unit and warns, naming the part. It does **not** treat a
|
|
28
|
+
* parenthesised `(3 mm)` that way: that is an annotation on an inch value, and
|
|
29
|
+
* converting it would turn `.1181 in` into `3 in`.
|
|
30
|
+
*/
|
|
31
|
+
import type { UnitSystem } from '../../conventions.js';
|
|
32
|
+
import { type Warn } from '../../scrape.js';
|
|
33
|
+
/** What one display cell says. */
|
|
34
|
+
export interface HarveyValue {
|
|
35
|
+
/** The leading numeric token, in whatever {@link stated} says. */
|
|
36
|
+
readonly value: number | null;
|
|
37
|
+
/** The unit the token states outright. Null where it states none. */
|
|
38
|
+
readonly stated: UnitSystem | null;
|
|
39
|
+
/** True where the token is an angle in degrees rather than a length. */
|
|
40
|
+
readonly degrees: boolean;
|
|
41
|
+
/** Harvey's parenthesised equivalent, verbatim — `1/8`, `3 mm`, `N.P.T.`. */
|
|
42
|
+
readonly annotation: string | null;
|
|
43
|
+
/** The multiplier a `(1.5x)` cell states, and nothing else. */
|
|
44
|
+
readonly ratio: number | null;
|
|
45
|
+
/** Text with no numeric reading at all: `I`, `II`, `III`, `LONG`. */
|
|
46
|
+
readonly code: string | null;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* One cell's display text, read.
|
|
50
|
+
*
|
|
51
|
+
* An empty cell and a `-` both come back as {@link NOTHING}. They mean slightly
|
|
52
|
+
* different things to a reader of the catalog — "nothing published" and "does
|
|
53
|
+
* not apply" — and the same thing to anything building a record, which is that
|
|
54
|
+
* there is no number here and 0 is not a substitute for one.
|
|
55
|
+
*/
|
|
56
|
+
export declare function parseValue(display: string): HarveyValue;
|
|
57
|
+
/**
|
|
58
|
+
* One cell as a length in `unit`, or null where it publishes none.
|
|
59
|
+
*
|
|
60
|
+
* `measure.asLength` makes both calls — convert-and-warn a cell whose stated
|
|
61
|
+
* unit disagrees with the family's, refuse an angle in a dimensional column —
|
|
62
|
+
* because they are the same two calls for every vendor and were a verbatim copy
|
|
63
|
+
* here and in `vendors/emuge/value.ts` until 2026-09-01. What is Harvey's is
|
|
64
|
+
* above: that `degrees` is a field of its own rather than a member of
|
|
65
|
+
* `stated`, and that a parenthesised `(3 mm)` is an annotation on an inch value
|
|
66
|
+
* and never the value itself.
|
|
67
|
+
*
|
|
68
|
+
* The 46 cells this converts are metric-shank tools listed among imperial ones
|
|
69
|
+
* — real parts somebody can order, which is why they are converted rather than
|
|
70
|
+
* dropped.
|
|
71
|
+
*/
|
|
72
|
+
export declare function dimension(display: string, unit: UnitSystem, what: string, warn?: Warn): number | null;
|
|
73
|
+
/** One cell as a whole count — a flute or tooth number. Null where blank. */
|
|
74
|
+
export declare function count(display: string): number | null;
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One display cell -> a number, or an honest reason there is none.
|
|
3
|
+
*
|
|
4
|
+
* Harvey publishes values the way a printed catalog does, because the table is
|
|
5
|
+
* a printed catalog: `.1250 (1/8)` is a decimal with the fraction a machinist
|
|
6
|
+
* orders by beside it, `1-1/2` is a mixed number, `3 mm` is a metric shank on
|
|
7
|
+
* an imperial page, and `-` means the column does not apply to that row.
|
|
8
|
+
*
|
|
9
|
+
* **One rule covers the whole grammar: the value is the leading token, and
|
|
10
|
+
* anything in parentheses is Harvey's own equivalent annotation.** Every shape
|
|
11
|
+
* measured across all 12,799 parts is in `docs/HARVEY_PRODUCT_TABLE.md` §4;
|
|
12
|
+
* this module is that section as code.
|
|
13
|
+
*
|
|
14
|
+
* ## Read `d`, never `v`
|
|
15
|
+
*
|
|
16
|
+
* Each cell also carries `v`, which looks pre-parsed and is the wrong field: it
|
|
17
|
+
* changes unit basis *within a single row*. On the metric ball page, `v` is the
|
|
18
|
+
* inch equivalent for the cutter diameter and the raw millimetre number for the
|
|
19
|
+
* length of cut. Anything reading it gets a clean conversion with wrong numbers
|
|
20
|
+
* in it — the exact failure `conventions.UNIT_SUFFIX` exists to prevent.
|
|
21
|
+
*
|
|
22
|
+
* ## A stated unit wins over the column's
|
|
23
|
+
*
|
|
24
|
+
* 46 cells across six otherwise-imperial families state a metric value outright
|
|
25
|
+
* — `3 mm` in a `SHANK DIAMETER` column on an inch page, a metric-shank tool
|
|
26
|
+
* listed among imperial ones. {@link dimension} converts those into the family's
|
|
27
|
+
* declared unit and warns, naming the part. It does **not** treat a
|
|
28
|
+
* parenthesised `(3 mm)` that way: that is an annotation on an inch value, and
|
|
29
|
+
* converting it would turn `.1181 in` into `3 in`.
|
|
30
|
+
*/
|
|
31
|
+
import { asCount, asLength, fractionValue } from '../../measure.js';
|
|
32
|
+
import { consoleWarn } from '../../scrape.js';
|
|
33
|
+
/**
|
|
34
|
+
* Footnote references Harvey appends to a value or a tool number. 62 cells
|
|
35
|
+
* carry one; each points at a note printed under the table, and none of them is
|
|
36
|
+
* part of the number.
|
|
37
|
+
*/
|
|
38
|
+
const FOOTNOTE = /[*!†]+$/;
|
|
39
|
+
/** Harvey's own equivalent, in parentheses at the end: `(1/8)`, `(3 mm)`. */
|
|
40
|
+
const ANNOTATION = /\s*\(([^)]*)\)\s*$/;
|
|
41
|
+
/** `(1.5x)`, `(30x)` — the vendor's reach- or length-to-diameter ratio. */
|
|
42
|
+
const RATIO = /^(\d*\.?\d+)x$/;
|
|
43
|
+
/**
|
|
44
|
+
* A mixed number, a simple fraction or a decimal, with an optional unit.
|
|
45
|
+
*
|
|
46
|
+
* The alternation is ordered longest-first on purpose: `\d*\.?\d+` alone
|
|
47
|
+
* matches the `3` of `3-3/4` and would leave the rest unread, which parses a
|
|
48
|
+
* 3.75-inch tool as a 3-inch one.
|
|
49
|
+
*/
|
|
50
|
+
const NUMBER = /^(\d+-\d+\/\d+|\d*\.?\d+\/\d+|\d*\.?\d+)\s*(mm|°)?$/;
|
|
51
|
+
const NOTHING = {
|
|
52
|
+
value: null,
|
|
53
|
+
stated: null,
|
|
54
|
+
degrees: false,
|
|
55
|
+
annotation: null,
|
|
56
|
+
ratio: null,
|
|
57
|
+
code: null,
|
|
58
|
+
};
|
|
59
|
+
/**
|
|
60
|
+
* One cell's display text, read.
|
|
61
|
+
*
|
|
62
|
+
* An empty cell and a `-` both come back as {@link NOTHING}. They mean slightly
|
|
63
|
+
* different things to a reader of the catalog — "nothing published" and "does
|
|
64
|
+
* not apply" — and the same thing to anything building a record, which is that
|
|
65
|
+
* there is no number here and 0 is not a substitute for one.
|
|
66
|
+
*/
|
|
67
|
+
export function parseValue(display) {
|
|
68
|
+
const text = display.trim().replace(FOOTNOTE, '').trim();
|
|
69
|
+
if (text === '' || text === '-')
|
|
70
|
+
return NOTHING;
|
|
71
|
+
const annotated = ANNOTATION.exec(text);
|
|
72
|
+
const annotation = annotated?.[1]?.trim() ?? null;
|
|
73
|
+
const head = annotated ? text.slice(0, annotated.index).trim() : text;
|
|
74
|
+
if (head === '') {
|
|
75
|
+
const ratio = annotation === null ? null : RATIO.exec(annotation);
|
|
76
|
+
if (ratio)
|
|
77
|
+
return { ...NOTHING, ratio: Number(ratio[1]) };
|
|
78
|
+
return { ...NOTHING, annotation };
|
|
79
|
+
}
|
|
80
|
+
const number = NUMBER.exec(head);
|
|
81
|
+
// `measure.fractionValue` is stricter than {@link NUMBER} in one place — it
|
|
82
|
+
// refuses a division by zero — so a token that matched here can still have no
|
|
83
|
+
// reading, and that is a code cell rather than a number.
|
|
84
|
+
const value = number === null ? null : fractionValue(number[1]);
|
|
85
|
+
if (number === null || value === null)
|
|
86
|
+
return { ...NOTHING, annotation, code: head };
|
|
87
|
+
const unit = number[2];
|
|
88
|
+
return {
|
|
89
|
+
value,
|
|
90
|
+
stated: unit === 'mm' ? 'millimeters' : null,
|
|
91
|
+
degrees: unit === '°',
|
|
92
|
+
annotation,
|
|
93
|
+
ratio: null,
|
|
94
|
+
code: null,
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* One cell as a length in `unit`, or null where it publishes none.
|
|
99
|
+
*
|
|
100
|
+
* `measure.asLength` makes both calls — convert-and-warn a cell whose stated
|
|
101
|
+
* unit disagrees with the family's, refuse an angle in a dimensional column —
|
|
102
|
+
* because they are the same two calls for every vendor and were a verbatim copy
|
|
103
|
+
* here and in `vendors/emuge/value.ts` until 2026-09-01. What is Harvey's is
|
|
104
|
+
* above: that `degrees` is a field of its own rather than a member of
|
|
105
|
+
* `stated`, and that a parenthesised `(3 mm)` is an annotation on an inch value
|
|
106
|
+
* and never the value itself.
|
|
107
|
+
*
|
|
108
|
+
* The 46 cells this converts are metric-shank tools listed among imperial ones
|
|
109
|
+
* — real parts somebody can order, which is why they are converted rather than
|
|
110
|
+
* dropped.
|
|
111
|
+
*/
|
|
112
|
+
export function dimension(display, unit, what, warn = consoleWarn) {
|
|
113
|
+
const { value, stated, degrees } = parseValue(display);
|
|
114
|
+
return asLength({ value, stated: degrees ? 'degrees' : stated }, display, unit, what, warn);
|
|
115
|
+
}
|
|
116
|
+
/** One cell as a whole count — a flute or tooth number. Null where blank. */
|
|
117
|
+
export function count(display) {
|
|
118
|
+
return asCount(parseValue(display));
|
|
119
|
+
}
|
|
@@ -15,10 +15,11 @@
|
|
|
15
15
|
* Number` are Kennametal's header text — the labels
|
|
16
16
|
* `conventions.IDENTITY_COLUMNS` took as the convention because this vendor
|
|
17
17
|
* was first, and which Destiny Tool then did not follow.
|
|
18
|
-
* - **Which column is the
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
18
|
+
* - **Which column is the coating.** `Coating` on every table that publishes
|
|
19
|
+
* one, which is every tap table and some of the others. The carbide `Grade`
|
|
20
|
+
* a drill and an end mill table also carry reaches no record: it is what the
|
|
21
|
+
* tool is made of in Kennametal's own vocabulary, nothing downstream reads
|
|
22
|
+
* it, and `substrate` already carries the cutting material as a fact.
|
|
22
23
|
* - **That a tap's unit system is per row.** `Thread System` is a constant tag
|
|
23
24
|
* column this package appends at scrape time, and a metric and an inch tap
|
|
24
25
|
* can sit in one family — so a tap's `unit` is read per row where a drill's
|
|
@@ -43,6 +44,14 @@ import type { ScrapedRow } from '../../scrape.js';
|
|
|
43
44
|
*/
|
|
44
45
|
export declare const MATERIAL_NUMBER = "Material Number";
|
|
45
46
|
export declare const CATALOG_NUMBER = "ISO Catalog Number";
|
|
47
|
+
/**
|
|
48
|
+
* The surface-treatment column, on the tables that publish one.
|
|
49
|
+
*
|
|
50
|
+
* Every tap table does; a drill or an end mill table may not, which is why the
|
|
51
|
+
* mappers read it with a `?? ''` rather than through a required column — an
|
|
52
|
+
* absent coating column is a table that states no coating, not a scrape fault.
|
|
53
|
+
*/
|
|
54
|
+
export declare const COATING = "Coating";
|
|
46
55
|
/**
|
|
47
56
|
* A drill, in the family's native unit system per its `unit` fact.
|
|
48
57
|
*
|
|
@@ -15,10 +15,11 @@
|
|
|
15
15
|
* Number` are Kennametal's header text — the labels
|
|
16
16
|
* `conventions.IDENTITY_COLUMNS` took as the convention because this vendor
|
|
17
17
|
* was first, and which Destiny Tool then did not follow.
|
|
18
|
-
* - **Which column is the
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
18
|
+
* - **Which column is the coating.** `Coating` on every table that publishes
|
|
19
|
+
* one, which is every tap table and some of the others. The carbide `Grade`
|
|
20
|
+
* a drill and an end mill table also carry reaches no record: it is what the
|
|
21
|
+
* tool is made of in Kennametal's own vocabulary, nothing downstream reads
|
|
22
|
+
* it, and `substrate` already carries the cutting material as a fact.
|
|
22
23
|
* - **That a tap's unit system is per row.** `Thread System` is a constant tag
|
|
23
24
|
* column this package appends at scrape time, and a metric and an inch tap
|
|
24
25
|
* can sit in one family — so a tap's `unit` is read per row where a drill's
|
|
@@ -34,10 +35,10 @@
|
|
|
34
35
|
* nose, `LB` and `assemblyGaugeLength` being `OAL` on a bare tool, and every
|
|
35
36
|
* cutting preset. Those are true of a tool, not of a table.
|
|
36
37
|
*/
|
|
37
|
-
import {
|
|
38
|
-
import { familyBrand } from '../../family.js';
|
|
38
|
+
import { VendorResponseError } from '../../errors.js';
|
|
39
|
+
import { fact, familyBrand } from '../../family.js';
|
|
39
40
|
import { BRANDS } from '../../identity.js';
|
|
40
|
-
import { toolRecord } from '../../records.js';
|
|
41
|
+
import { toolRecord, UNSPECIFIED, } from '../../records.js';
|
|
41
42
|
import { threadMajorDiameter } from '../../thread.js';
|
|
42
43
|
import { MATERIALS_COLUMN, materialClasses } from './materials.js';
|
|
43
44
|
/**
|
|
@@ -46,6 +47,14 @@ import { MATERIALS_COLUMN, materialClasses } from './materials.js';
|
|
|
46
47
|
*/
|
|
47
48
|
export const MATERIAL_NUMBER = 'Material Number';
|
|
48
49
|
export const CATALOG_NUMBER = 'ISO Catalog Number';
|
|
50
|
+
/**
|
|
51
|
+
* The surface-treatment column, on the tables that publish one.
|
|
52
|
+
*
|
|
53
|
+
* Every tap table does; a drill or an end mill table may not, which is why the
|
|
54
|
+
* mappers read it with a `?? ''` rather than through a required column — an
|
|
55
|
+
* absent coating column is a table that states no coating, not a scrape fault.
|
|
56
|
+
*/
|
|
57
|
+
export const COATING = 'Coating';
|
|
49
58
|
/**
|
|
50
59
|
* One canonical dimension, or null when this family maps or publishes none.
|
|
51
60
|
*
|
|
@@ -79,12 +88,30 @@ function require_(row, columns, canonical, unit, what) {
|
|
|
79
88
|
}
|
|
80
89
|
return value;
|
|
81
90
|
}
|
|
82
|
-
/**
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
91
|
+
/**
|
|
92
|
+
* The workpiece-material groups, and how they were arrived at.
|
|
93
|
+
*
|
|
94
|
+
* **The distinction the column cannot make on its own is present versus
|
|
95
|
+
* blank.** The material groups are not scraped with the variant table: they
|
|
96
|
+
* come from a second CLI step (`toolpath-scrape materials`) that sweeps the
|
|
97
|
+
* `workpieceMaterialDetail` facet and writes {@link MATERIALS_COLUMN} into the
|
|
98
|
+
* CSV. A family that step never ran over has **no such column**, and
|
|
99
|
+
* `node/csv.parseCsv` fills `''` only for cells under a column that is in the
|
|
100
|
+
* header — so the cell is `undefined` on an unswept CSV and `''` on a swept
|
|
101
|
+
* one whose part the vendor rates for nothing.
|
|
102
|
+
*
|
|
103
|
+
* Those are different claims and the record now keeps them apart: an
|
|
104
|
+
* `unspecified` label says this package has no evidence, `[]` says the vendor's
|
|
105
|
+
* own index has none. All
|
|
106
|
+
* 129 taps land on the second after a sweep — Kennametal indexes no tap by
|
|
107
|
+
* workpiece material at all — and reading that as "unconstrained" would put
|
|
108
|
+
* every tap under every material.
|
|
109
|
+
*/
|
|
110
|
+
function materialGroups(row) {
|
|
111
|
+
const cell = row[MATERIALS_COLUMN];
|
|
112
|
+
if (cell === undefined)
|
|
113
|
+
return { materialGroups: null, materialGroupsSource: UNSPECIFIED };
|
|
114
|
+
return { materialGroups: materialClasses(cell), materialGroupsSource: 'vendor-stated' };
|
|
88
115
|
}
|
|
89
116
|
/**
|
|
90
117
|
* The `Thread System` tag, refusing anything that is not one of the two.
|
|
@@ -129,15 +156,19 @@ export function drillRecord(row, family, columns) {
|
|
|
129
156
|
const unit = fact(family, 'unit', family.unit);
|
|
130
157
|
const what = row[MATERIAL_NUMBER] ?? '';
|
|
131
158
|
return toolRecord({
|
|
159
|
+
brand: familyBrand(family),
|
|
132
160
|
vendor: BRANDS[familyBrand(family)].vendor,
|
|
133
161
|
materialNumber: what,
|
|
134
162
|
catalogNumber: row[CATALOG_NUMBER] ?? '',
|
|
135
|
-
description
|
|
163
|
+
// Kennametal publishes no description column on any table. The catalog
|
|
164
|
+
// number is already on the record, and repeating it here said nothing —
|
|
165
|
+
// see `records.ToolRecord.description`.
|
|
166
|
+
description: '',
|
|
136
167
|
kind: 'drill',
|
|
137
168
|
unit,
|
|
138
169
|
substrate: fact(family, 'bmc', family.bmc),
|
|
139
|
-
|
|
140
|
-
materialGroups
|
|
170
|
+
coating: row[COATING] ?? '',
|
|
171
|
+
...materialGroups(row),
|
|
141
172
|
coolantThrough: fact(family, 'coolantThrough', family.coolantThrough),
|
|
142
173
|
nonFerrous: fact(family, 'nonFerrous', family.nonFerrous),
|
|
143
174
|
geometry: {
|
|
@@ -170,19 +201,21 @@ export function tapRecord(row, family, columns) {
|
|
|
170
201
|
const tdz = row['D1-TDZ'] ?? '';
|
|
171
202
|
const what = row[MATERIAL_NUMBER] ?? '';
|
|
172
203
|
return toolRecord({
|
|
204
|
+
brand: familyBrand(family),
|
|
173
205
|
vendor: BRANDS[familyBrand(family)].vendor,
|
|
174
206
|
materialNumber: what,
|
|
175
207
|
catalogNumber: row[CATALOG_NUMBER] ?? '',
|
|
176
|
-
// The
|
|
177
|
-
//
|
|
178
|
-
|
|
208
|
+
// The one Kennametal table that publishes per-part text: the thread
|
|
209
|
+
// designation is part of what a tap *is*, and the catalog number does not
|
|
210
|
+
// carry the size. The catalog number no longer leads it — it is already on
|
|
211
|
+
// the record, and a description that restates one is not a description.
|
|
212
|
+
description: tdz,
|
|
179
213
|
kind: 'tap',
|
|
180
214
|
unit,
|
|
181
215
|
substrate: fact(family, 'bmc', family.bmc),
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
coolantThrough: false,
|
|
216
|
+
coating: row[COATING] ?? '',
|
|
217
|
+
...materialGroups(row),
|
|
218
|
+
coolantThrough: fact(family, 'coolantThrough', family.coolantThrough),
|
|
186
219
|
geometry: {
|
|
187
220
|
DC: threadMajorDiameter(tdz, system),
|
|
188
221
|
TP: require_(row, columns, 'TP', unit, what),
|
|
@@ -212,15 +245,17 @@ export function endmillRecord(row, family, columns) {
|
|
|
212
245
|
const dc = require_(row, columns, 'DC', unit, what);
|
|
213
246
|
const fluteLength = require_(row, columns, 'LCF', unit, what);
|
|
214
247
|
return toolRecord({
|
|
248
|
+
brand: familyBrand(family),
|
|
215
249
|
vendor: BRANDS[familyBrand(family)].vendor,
|
|
216
250
|
materialNumber: what,
|
|
217
251
|
catalogNumber: row[CATALOG_NUMBER] ?? '',
|
|
218
|
-
description
|
|
252
|
+
// No description column here either. See the drill mapper.
|
|
253
|
+
description: '',
|
|
219
254
|
kind: 'endmill',
|
|
220
255
|
unit,
|
|
221
256
|
substrate: fact(family, 'bmc', family.bmc),
|
|
222
|
-
|
|
223
|
-
materialGroups
|
|
257
|
+
coating: row[COATING] ?? '',
|
|
258
|
+
...materialGroups(row),
|
|
224
259
|
coolantThrough: fact(family, 'coolantThrough', family.coolantThrough),
|
|
225
260
|
geometry: {
|
|
226
261
|
DC: dc,
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The category tree -> the leaves under it.
|
|
3
|
+
*
|
|
4
|
+
* MariTool publishes no sitemap: `/sitemap.xml` answers with a meta-refresh
|
|
5
|
+
* shell rather than a document, and there is no flat index anywhere on the
|
|
6
|
+
* site. Walking the category tree is the only way to find out what exists —
|
|
7
|
+
* see `docs/MARITOOL_CATALOG.md` §2.
|
|
8
|
+
*
|
|
9
|
+
* **Nothing in a scrape needs this.** `families/maritool.ts` names all 41 leaf
|
|
10
|
+
* cPaths in scope, and a scrape fetches exactly those; re-deriving them at
|
|
11
|
+
* scrape time would put a 199-page crawl in front of every run to rediscover
|
|
12
|
+
* a list that is already written down. It is here for the maintenance question
|
|
13
|
+
* that table cannot answer — *has MariTool added or moved a leaf?* — and the
|
|
14
|
+
* CLI exposes it as `maritool --catalog` for that purpose. It is how the table
|
|
15
|
+
* was built and how it gets rechecked.
|
|
16
|
+
*
|
|
17
|
+
* ## The tree is deeper than it looks
|
|
18
|
+
*
|
|
19
|
+
* Two to four levels, and **no root carries products itself**. `Dual Contact
|
|
20
|
+
* CAT40` (`c23_25_432`) and each of the nine HSK sizes are intermediate nodes
|
|
21
|
+
* with children, so a walk that stopped at depth 2 would find zero HSK holders
|
|
22
|
+
* and zero dual-contact parts — 187 of the 529 in scope. That is why this
|
|
23
|
+
* recurses on the child links rather than reading one level of the sidebar.
|
|
24
|
+
*
|
|
25
|
+
* A category page carries its children as links whose cPath extends its own by
|
|
26
|
+
* exactly one segment, and the sidebar renders unrelated branches beside them,
|
|
27
|
+
* so the prefix test is what separates a child from a sibling of an ancestor.
|
|
28
|
+
*/
|
|
29
|
+
import type { Fetcher } from '../../fetch.js';
|
|
30
|
+
import { type Warn } from '../../scrape.js';
|
|
31
|
+
/**
|
|
32
|
+
* The five taper trees this adapter covers.
|
|
33
|
+
*
|
|
34
|
+
* MariTool's catalog is larger — `c23` holds twenty more roots, and the store
|
|
35
|
+
* sells cutting tools besides. These are the trees asked for, and adding one
|
|
36
|
+
* is a decision rather than a widening of a pattern; see the package's rule on
|
|
37
|
+
* not adding vendor scope without being asked.
|
|
38
|
+
*
|
|
39
|
+
* **There is no CAT30.** MariTool's 30-taper is BT30 and ISO30, confirmed
|
|
40
|
+
* against the full sidebar rather than against a landing page. ISO30 (`c23_54`)
|
|
41
|
+
* is absent for a different reason: its three ER parts publish no spec table,
|
|
42
|
+
* so the family would be a receipt of nothing.
|
|
43
|
+
*/
|
|
44
|
+
export declare const CATEGORY_ROOTS: readonly string[];
|
|
45
|
+
/** One node of the tree, as the walk found it. */
|
|
46
|
+
export interface Category {
|
|
47
|
+
/** MariTool's own category path, e.g. `23_25_42`. */
|
|
48
|
+
readonly cPath: string;
|
|
49
|
+
/** The vendor's own name for it, off the page title. */
|
|
50
|
+
readonly name: string;
|
|
51
|
+
/** Child cPaths, sorted. Empty on a leaf. */
|
|
52
|
+
readonly children: readonly string[];
|
|
53
|
+
/** What `(of N products)` said, or 0 where the page states no count. */
|
|
54
|
+
readonly products: number;
|
|
55
|
+
}
|
|
56
|
+
/** The category name a page states, stripped of the store's boilerplate. */
|
|
57
|
+
export declare function categoryName(html: string): string;
|
|
58
|
+
/**
|
|
59
|
+
* One category page: its children, its name and its own product count.
|
|
60
|
+
*
|
|
61
|
+
* A child is a link whose cPath extends this one by exactly one segment. The
|
|
62
|
+
* page also links every sibling of every ancestor — the sidebar renders the
|
|
63
|
+
* whole open branch — so taking every `cPath` link would walk the catalog from
|
|
64
|
+
* any starting point and take the twenty out-of-scope roots with it.
|
|
65
|
+
*/
|
|
66
|
+
export declare function parseCategory(html: string, cPath: string): Category;
|
|
67
|
+
/**
|
|
68
|
+
* Every category reachable from `roots`, breadth first and paced.
|
|
69
|
+
*
|
|
70
|
+
* Sequential and paced by the package's shared politeness delay. The five
|
|
71
|
+
* roots in scope reach 199 categories, which is a three-minute walk and is run
|
|
72
|
+
* by hand rather than by a scrape.
|
|
73
|
+
*/
|
|
74
|
+
export declare function discoverCategories(fetcher: Fetcher, roots?: readonly string[], options?: {
|
|
75
|
+
warn?: Warn;
|
|
76
|
+
delayMs?: number;
|
|
77
|
+
}): Promise<Category[]>;
|
|
78
|
+
/** The leaves of a walk: the categories that carry products themselves. */
|
|
79
|
+
export declare function leavesOf(categories: readonly Category[]): Category[];
|
|
80
|
+
/** One line per category, for a human reading a `--catalog` run. */
|
|
81
|
+
export declare function describe(category: Category): string;
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The category tree -> the leaves under it.
|
|
3
|
+
*
|
|
4
|
+
* MariTool publishes no sitemap: `/sitemap.xml` answers with a meta-refresh
|
|
5
|
+
* shell rather than a document, and there is no flat index anywhere on the
|
|
6
|
+
* site. Walking the category tree is the only way to find out what exists —
|
|
7
|
+
* see `docs/MARITOOL_CATALOG.md` §2.
|
|
8
|
+
*
|
|
9
|
+
* **Nothing in a scrape needs this.** `families/maritool.ts` names all 41 leaf
|
|
10
|
+
* cPaths in scope, and a scrape fetches exactly those; re-deriving them at
|
|
11
|
+
* scrape time would put a 199-page crawl in front of every run to rediscover
|
|
12
|
+
* a list that is already written down. It is here for the maintenance question
|
|
13
|
+
* that table cannot answer — *has MariTool added or moved a leaf?* — and the
|
|
14
|
+
* CLI exposes it as `maritool --catalog` for that purpose. It is how the table
|
|
15
|
+
* was built and how it gets rechecked.
|
|
16
|
+
*
|
|
17
|
+
* ## The tree is deeper than it looks
|
|
18
|
+
*
|
|
19
|
+
* Two to four levels, and **no root carries products itself**. `Dual Contact
|
|
20
|
+
* CAT40` (`c23_25_432`) and each of the nine HSK sizes are intermediate nodes
|
|
21
|
+
* with children, so a walk that stopped at depth 2 would find zero HSK holders
|
|
22
|
+
* and zero dual-contact parts — 187 of the 529 in scope. That is why this
|
|
23
|
+
* recurses on the child links rather than reading one level of the sidebar.
|
|
24
|
+
*
|
|
25
|
+
* A category page carries its children as links whose cPath extends its own by
|
|
26
|
+
* exactly one segment, and the sidebar renders unrelated branches beside them,
|
|
27
|
+
* so the prefix test is what separates a child from a sibling of an ancestor.
|
|
28
|
+
*/
|
|
29
|
+
import { Parser } from 'htmlparser2';
|
|
30
|
+
import { compare } from '../../order.js';
|
|
31
|
+
import { consoleWarn, pause, REQUEST_DELAY_MS } from '../../scrape.js';
|
|
32
|
+
import { BASE, categoryUrl } from './scrape.js';
|
|
33
|
+
/**
|
|
34
|
+
* The five taper trees this adapter covers.
|
|
35
|
+
*
|
|
36
|
+
* MariTool's catalog is larger — `c23` holds twenty more roots, and the store
|
|
37
|
+
* sells cutting tools besides. These are the trees asked for, and adding one
|
|
38
|
+
* is a decision rather than a widening of a pattern; see the package's rule on
|
|
39
|
+
* not adding vendor scope without being asked.
|
|
40
|
+
*
|
|
41
|
+
* **There is no CAT30.** MariTool's 30-taper is BT30 and ISO30, confirmed
|
|
42
|
+
* against the full sidebar rather than against a landing page. ISO30 (`c23_54`)
|
|
43
|
+
* is absent for a different reason: its three ER parts publish no spec table,
|
|
44
|
+
* so the family would be a receipt of nothing.
|
|
45
|
+
*/
|
|
46
|
+
export const CATEGORY_ROOTS = [
|
|
47
|
+
'23_25', // CAT40
|
|
48
|
+
'23_24', // CAT50
|
|
49
|
+
'23_33', // BT30
|
|
50
|
+
'23_26', // BT40
|
|
51
|
+
'23_46', // HSK
|
|
52
|
+
];
|
|
53
|
+
/** `.../c23_25_42/index.html` and `index.php?cPath=23_25_42` both give `23_25_42`. */
|
|
54
|
+
const CATEGORY_HREF = /(?:\/c|[?&]cPath=)(\d+(?:_\d+)*)(?:\/index\.html)?(?:[&#]|$)/;
|
|
55
|
+
/** `<title>… Accessories ER Collet Chucks - MariTool</title>` -> the middle. */
|
|
56
|
+
const TITLE = /<title>([^<]*)<\/title>/i;
|
|
57
|
+
/** The vendor's own boilerplate around every category name. */
|
|
58
|
+
const TITLE_NOISE = [/^Tool Holders, Collets and Machine Accessories\s*/i, /\s*-\s*MariTool$/i];
|
|
59
|
+
/** `(of 51 products)`, or 0 — an empty category renders no count at all. */
|
|
60
|
+
const PRODUCT_COUNT = /\(of\s*<b>\s*(\d+)\s*<\/b>\s*products\)/i;
|
|
61
|
+
/** The category name a page states, stripped of the store's boilerplate. */
|
|
62
|
+
export function categoryName(html) {
|
|
63
|
+
let name = TITLE.exec(html)?.[1] ?? '';
|
|
64
|
+
for (const noise of TITLE_NOISE)
|
|
65
|
+
name = name.replace(noise, '');
|
|
66
|
+
return name.replace(/\s+/g, ' ').trim();
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* One category page: its children, its name and its own product count.
|
|
70
|
+
*
|
|
71
|
+
* A child is a link whose cPath extends this one by exactly one segment. The
|
|
72
|
+
* page also links every sibling of every ancestor — the sidebar renders the
|
|
73
|
+
* whole open branch — so taking every `cPath` link would walk the catalog from
|
|
74
|
+
* any starting point and take the twenty out-of-scope roots with it.
|
|
75
|
+
*/
|
|
76
|
+
export function parseCategory(html, cPath) {
|
|
77
|
+
const children = new Set();
|
|
78
|
+
const parser = new Parser({
|
|
79
|
+
onopentag: (tag, attribs) => {
|
|
80
|
+
if (tag !== 'a')
|
|
81
|
+
return;
|
|
82
|
+
const found = CATEGORY_HREF.exec(attribs['href'] ?? '')?.[1];
|
|
83
|
+
if (found === undefined)
|
|
84
|
+
return;
|
|
85
|
+
if (found.startsWith(`${cPath}_`) &&
|
|
86
|
+
found.split('_').length === cPath.split('_').length + 1)
|
|
87
|
+
children.add(found);
|
|
88
|
+
},
|
|
89
|
+
}, { decodeEntities: true });
|
|
90
|
+
parser.write(html);
|
|
91
|
+
parser.end();
|
|
92
|
+
return {
|
|
93
|
+
cPath,
|
|
94
|
+
name: categoryName(html),
|
|
95
|
+
children: [...children].sort(compare),
|
|
96
|
+
products: Number(PRODUCT_COUNT.exec(html)?.[1] ?? 0),
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Every category reachable from `roots`, breadth first and paced.
|
|
101
|
+
*
|
|
102
|
+
* Sequential and paced by the package's shared politeness delay. The five
|
|
103
|
+
* roots in scope reach 199 categories, which is a three-minute walk and is run
|
|
104
|
+
* by hand rather than by a scrape.
|
|
105
|
+
*/
|
|
106
|
+
export async function discoverCategories(fetcher, roots = CATEGORY_ROOTS, options = {}) {
|
|
107
|
+
const { warn = consoleWarn, delayMs = REQUEST_DELAY_MS } = options;
|
|
108
|
+
const found = new Map();
|
|
109
|
+
const queue = [...roots];
|
|
110
|
+
while (queue.length > 0) {
|
|
111
|
+
const cPath = queue.shift();
|
|
112
|
+
if (found.has(cPath))
|
|
113
|
+
continue;
|
|
114
|
+
const category = parseCategory(await fetcher.text(categoryUrl(cPath)), cPath);
|
|
115
|
+
found.set(cPath, category);
|
|
116
|
+
if (category.children.length === 0 && category.products === 0) {
|
|
117
|
+
warn(` WARNING: c${cPath} (${category.name}) holds neither a subcategory nor a product`);
|
|
118
|
+
}
|
|
119
|
+
queue.push(...category.children);
|
|
120
|
+
await pause(delayMs);
|
|
121
|
+
}
|
|
122
|
+
return [...found.values()].sort((a, b) => compare(a.cPath, b.cPath));
|
|
123
|
+
}
|
|
124
|
+
/** The leaves of a walk: the categories that carry products themselves. */
|
|
125
|
+
export function leavesOf(categories) {
|
|
126
|
+
return categories.filter((category) => category.children.length === 0);
|
|
127
|
+
}
|
|
128
|
+
/** One line per category, for a human reading a `--catalog` run. */
|
|
129
|
+
export function describe(category) {
|
|
130
|
+
const shape = category.children.length > 0 ? `${category.children.length} subcategories` : 'leaf';
|
|
131
|
+
return `${BASE}/index.php?cPath=${category.cPath}\t${category.products}\t${shape}\t${category.name}`;
|
|
132
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MariTool — toolholding only, and the one vendor here with no transport to
|
|
3
|
+
* discover.
|
|
4
|
+
*
|
|
5
|
+
* An osCommerce-family storefront: server-rendered HTML, a paced `GET`, and a
|
|
6
|
+
* roster on the category page with the geometry one request further on. See
|
|
7
|
+
* `docs/MARITOOL_CATALOG.md`.
|
|
8
|
+
*/
|
|
9
|
+
export * from './catalog.js';
|
|
10
|
+
export * from './scrape.js';
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MariTool — toolholding only, and the one vendor here with no transport to
|
|
3
|
+
* discover.
|
|
4
|
+
*
|
|
5
|
+
* An osCommerce-family storefront: server-rendered HTML, a paced `GET`, and a
|
|
6
|
+
* roster on the category page with the geometry one request further on. See
|
|
7
|
+
* `docs/MARITOOL_CATALOG.md`.
|
|
8
|
+
*/
|
|
9
|
+
export * from './catalog.js';
|
|
10
|
+
export * from './scrape.js';
|