@toolpath/tool-scraper 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/columns.d.ts +5 -3
- package/dist/columns.js +11 -5
- package/dist/conventions.d.ts +18 -0
- package/dist/conventions.js +18 -0
- package/dist/errors.d.ts +30 -0
- package/dist/errors.js +30 -0
- package/dist/families/emuge.d.ts +3 -1
- package/dist/families/emuge.js +8 -4
- package/dist/node/cli.js +6 -1
- package/dist/records.d.ts +57 -12
- package/dist/records.js +37 -13
- package/dist/registry.d.ts +26 -1
- package/dist/registry.js +40 -2
- package/dist/vendors/destinytool/records.js +8 -0
- package/dist/vendors/emuge/records.d.ts +83 -3
- package/dist/vendors/emuge/records.js +184 -16
- package/dist/vendors/kennametal/family.d.ts +119 -0
- package/dist/vendors/kennametal/family.js +155 -0
- package/dist/vendors/kennametal/index.d.ts +1 -0
- package/dist/vendors/kennametal/index.js +1 -0
- package/dist/vendors/kennametal/records.d.ts +16 -0
- package/dist/vendors/kennametal/records.js +32 -0
- package/dist/vendors/kennametal/scrape.d.ts +25 -2
- package/dist/vendors/kennametal/scrape.js +28 -3
- package/package.json +1 -1
|
@@ -14,7 +14,12 @@
|
|
|
14
14
|
* article code, per part, so there is no `conventions.IDENTITY_DEVIATIONS`
|
|
15
15
|
* entry — the first vendor since Kennametal that needs none.
|
|
16
16
|
* - **A point angle, per drill.** Kennametal's drill families assume theirs; the
|
|
17
|
-
* detail record states it, so `SIG` is a mapped column here and no fact.
|
|
17
|
+
* detail record states it, so `SIG` is a mapped column here and no fact. On
|
|
18
|
+
* all but one part: 2,669 of 2,670 drill variants state one, and part
|
|
19
|
+
* `000000000010727800` publishes a single classification feature and no
|
|
20
|
+
* dimensional properties at all — so its row carries no `SIG` **key**, not
|
|
21
|
+
* an empty one. That is why `records.RECORD_GEOMETRY.drill` lists `SIG`
|
|
22
|
+
* under `sometimes`, and why {@link angle} reads the row rather than `cell`.
|
|
18
23
|
* - **A per-part ISO 513 index.** `applicationMaterials` returns the vendor's
|
|
19
24
|
* own P/M/K/N/S/H rating for each part, which fills `materialGroups` as
|
|
20
25
|
* `vendor-stated`.
|
|
@@ -132,6 +137,125 @@ export const SUBSTRATES = {
|
|
|
132
137
|
CBN: 'cbn',
|
|
133
138
|
ceramic: 'ceramic',
|
|
134
139
|
};
|
|
140
|
+
/**
|
|
141
|
+
* The column each EMUGE-FRANKEN category states its product line in, keyed by
|
|
142
|
+
* the vendor's own category code.
|
|
143
|
+
*
|
|
144
|
+
* **One column per category, and each is a facet that partitions its category
|
|
145
|
+
* exactly.** Checked against the vendor's own index on 2026-09-01, at group
|
|
146
|
+
* level rather than by summing counts:
|
|
147
|
+
*
|
|
148
|
+
* | Category | Facet | Values | Groups it covers | Groups in two values |
|
|
149
|
+
* | -------- | -------------------- | ------ | ---------------- | -------------------- |
|
|
150
|
+
* | `FF01` | `AMM_PROG_LINIE` | 15 | 554 of 554 | 2 |
|
|
151
|
+
* | `FB01` | `HYB_BAM_SB_GT` | 4 | 17 of 17 | 0 |
|
|
152
|
+
* | `FG01` | `HYB_BAM_SB_GT` | 17 | 414 of 414 | 0 |
|
|
153
|
+
*
|
|
154
|
+
* That is what makes a product line here a **read and not an arbitration**.
|
|
155
|
+
* EMUGE's marketing publishes 43 overlapping product-family pages — a tap is
|
|
156
|
+
* simultaneously "Rekord B-Z Taps", "Enorm Z Taps" and "Left Hand Taps" — and
|
|
157
|
+
* choosing between those would be this package inventing a rule the vendor
|
|
158
|
+
* never stated. The facets above are the vendor's own partition of the same
|
|
159
|
+
* catalog, so there is nothing to choose.
|
|
160
|
+
*
|
|
161
|
+
* **The two milling groups that fall in two values cost nothing**, because the
|
|
162
|
+
* milling column is read per part: `H300024` and `H300025` each hold both
|
|
163
|
+
* `FRANKEN TiNox-Cut` and `FRANKEN TiNox-Cut VAR` variants, and the per-part
|
|
164
|
+
* detail record states which is which. A rule that tagged a whole group would
|
|
165
|
+
* have had to pick.
|
|
166
|
+
*
|
|
167
|
+
* **No request is added for any of this.** `Geometry` is on the grouped product
|
|
168
|
+
* and `product line` on the per-part detail, so `scrape.ts` already writes both
|
|
169
|
+
* into every row — see its three-call note.
|
|
170
|
+
*/
|
|
171
|
+
export const PRODUCT_LINE_COLUMNS = {
|
|
172
|
+
FF01: 'product line',
|
|
173
|
+
FB01: 'Geometry',
|
|
174
|
+
FG01: 'Geometry',
|
|
175
|
+
};
|
|
176
|
+
/**
|
|
177
|
+
* A category's product-line codes onto the vendor's own name for each.
|
|
178
|
+
*
|
|
179
|
+
* **Both sides are EMUGE's.** The key is the value its `HYB_BAM_SB_GT` facet
|
|
180
|
+
* publishes and the name is the title of the vendor's own product-family
|
|
181
|
+
* article page for it, read on 2026-09-01 and cited per entry. Nothing here is
|
|
182
|
+
* this package's wording, which is the whole condition on a table like this —
|
|
183
|
+
* the same rule {@link SUBSTRATES} keeps, one level less strict because that
|
|
184
|
+
* one maps onto a vocabulary this package owns and this one does not.
|
|
185
|
+
*
|
|
186
|
+
* **A category whose facet already names the line has no entry.** `FF01`'s
|
|
187
|
+
* values are `FRANKEN TOP-Cut`, `FRANKEN Hard-Cut`, `FRANKEN Alu-Cut` — the
|
|
188
|
+
* vendor's marketing names already — so milling passes through verbatim and
|
|
189
|
+
* appears nowhere below. Drilling and tapping index by a geometry code
|
|
190
|
+
* (`MULTI`, `Z`, `VA`) that matches nothing EMUGE sells under that name, which
|
|
191
|
+
* is the only reason this table exists.
|
|
192
|
+
*
|
|
193
|
+
* **A code with no article page keeps the code**, and that is a gap in the
|
|
194
|
+
* vendor's marketing rather than a hole here: `SPEED`, `FK`, `GAL`, `GG` and
|
|
195
|
+
* `TILEG` are real lines with no `/a/` page on the US storefront, so the
|
|
196
|
+
* honest answer is the vendor's own code until one appears.
|
|
197
|
+
*/
|
|
198
|
+
export const PRODUCT_LINES = {
|
|
199
|
+
// `/us/en/multi-drill/a/MultiDRILL`, `/us/en/steeldrill/a/SteelDrill`,
|
|
200
|
+
// `/us/en/inox/a/Inox`, and
|
|
201
|
+
// `/us/en/ef-va---carbide-drills---stainless-steel-titanium-alloys`.
|
|
202
|
+
//
|
|
203
|
+
// `STEEL` has two article pages — "SteelDrill" and "EF / CARBIDE DRILLS" —
|
|
204
|
+
// that resolve to the identical 8 groups. `SteelDrill` is the one the
|
|
205
|
+
// vendor's own part names use (`SteelDrill SD102-5xD-HA`), which is the
|
|
206
|
+
// tiebreak: a part that calls itself one of the two is evidence and a
|
|
207
|
+
// preference between two pages is not.
|
|
208
|
+
FB01: {
|
|
209
|
+
MULTI: 'MultiDRILL',
|
|
210
|
+
STEEL: 'SteelDrill',
|
|
211
|
+
INOX: 'Inox',
|
|
212
|
+
VA: 'EF-VA / CARBIDE DRILLS',
|
|
213
|
+
},
|
|
214
|
+
// One article page each, all under `/us/en/<slug>/a/<name>`: multitap,
|
|
215
|
+
// va-taps, ti-taps, ni-taps, aero-taps, rekord-b-z-taps, a-h-taps, al-taps,
|
|
216
|
+
// a-gjv-taps, a-hcut-taps, steel-taps.
|
|
217
|
+
//
|
|
218
|
+
// `FK`, `GAL`, `GG`, `SPEED` and `TILEG` are absent on purpose — see above.
|
|
219
|
+
FG01: {
|
|
220
|
+
MULTI: 'MultiTAP™',
|
|
221
|
+
VA: 'VA Taps',
|
|
222
|
+
TI: 'TI-Taps',
|
|
223
|
+
NI: 'Ni Taps',
|
|
224
|
+
AERO: 'AERO Taps',
|
|
225
|
+
Z: 'Rekord B-Z Taps',
|
|
226
|
+
H: 'A-H Taps',
|
|
227
|
+
AL: 'Al Taps',
|
|
228
|
+
GJV: 'A-GJV Taps',
|
|
229
|
+
HCUT: 'A-HCUT Taps',
|
|
230
|
+
STEEL: 'Steel Taps',
|
|
231
|
+
},
|
|
232
|
+
};
|
|
233
|
+
/**
|
|
234
|
+
* The product line this row states, or null where the vendor states none.
|
|
235
|
+
*
|
|
236
|
+
* Three answers and they are different things. A family whose category has no
|
|
237
|
+
* entry in {@link PRODUCT_LINE_COLUMNS} is a category this adapter has not
|
|
238
|
+
* looked at, and it answers null rather than guessing at which of the row's
|
|
239
|
+
* columns is the line. A column the vendor left empty is null too — the same
|
|
240
|
+
* silence `angle` treats as an omission rather than a fault. A *value* is
|
|
241
|
+
* mapped through {@link PRODUCT_LINES} where its category has a table and
|
|
242
|
+
* passed through verbatim where it does not, which is the milling case.
|
|
243
|
+
*
|
|
244
|
+
* It does not throw on an unknown code, and that is the difference between
|
|
245
|
+
* this and `substrate`: a cutting material this package cannot map would put a
|
|
246
|
+
* wrong word in `ToolRecord.substrate`, where an unmapped product-line code is
|
|
247
|
+
* the vendor's own code and readable as one. A new EMUGE geometry showing up
|
|
248
|
+
* as `SPEED` is a name to improve, not a record to refuse.
|
|
249
|
+
*/
|
|
250
|
+
export function productLine(row, family) {
|
|
251
|
+
const column = PRODUCT_LINE_COLUMNS[family.familyCode ?? ''];
|
|
252
|
+
if (column === undefined)
|
|
253
|
+
return null;
|
|
254
|
+
const stated = (row[column] ?? '').trim();
|
|
255
|
+
if (stated === '')
|
|
256
|
+
return null;
|
|
257
|
+
return PRODUCT_LINES[family.familyCode ?? '']?.[stated] ?? stated;
|
|
258
|
+
}
|
|
135
259
|
/**
|
|
136
260
|
* The flute count EMUGE publishes where it has none: 64 end mill variants, on
|
|
137
261
|
* 2026-09-01. A sentinel and not a number, so it is refused rather than read.
|
|
@@ -146,17 +270,55 @@ export const NO_FLUTE_COUNT = 999;
|
|
|
146
270
|
*/
|
|
147
271
|
const { cell, required, optional } = columnReaders(measureIn);
|
|
148
272
|
/**
|
|
149
|
-
* An angle in degrees — the drill's point angle
|
|
273
|
+
* An angle in degrees — the drill's point angle — or null where the vendor
|
|
274
|
+
* left the cell empty.
|
|
150
275
|
*
|
|
151
276
|
* Read through {@link parseMeasure} rather than {@link measureIn}, which
|
|
152
277
|
* refuses degrees on purpose: a length column stating an angle is a property
|
|
153
278
|
* that has moved. Here degrees are what the column is for.
|
|
279
|
+
*
|
|
280
|
+
* **An empty cell and an unreadable one are different answers**, the same
|
|
281
|
+
* split {@link coolantThrough} makes one level down. EMUGE fills this column
|
|
282
|
+
* on 2,669 of its 2,670 drill variants and leaves it blank on one, so a blank
|
|
283
|
+
* is the vendor publishing nothing and the row is still a part somebody can
|
|
284
|
+
* order — `records.RECORD_GEOMETRY.drill` lists `SIG` under `sometimes` for
|
|
285
|
+
* it. A cell holding a *value* this cannot read is the other case: a length
|
|
286
|
+
* where an angle belongs is the property having moved, a range has no single
|
|
287
|
+
* reading, and either one is refused rather than dropped quietly.
|
|
288
|
+
*
|
|
289
|
+
* A column the family maps to nothing refuses too, and is a third thing again
|
|
290
|
+
* — this adapter's drill family maps `point angle`, so its absence is that map
|
|
291
|
+
* having changed rather than anything the vendor did. **That is a fact about
|
|
292
|
+
* the map and not about a row**, which is why it is asked of `columns` and not
|
|
293
|
+
* inferred from a missing cell: a part publishing no dimensional properties
|
|
294
|
+
* has no such key either, and reading the two as one refused every drill in
|
|
295
|
+
* the family over one incomplete part.
|
|
296
|
+
* `records.REQUIRED_GEOMETRY` cannot catch it: `SIG` is not listed under its
|
|
297
|
+
* `drill` entry, because Kennametal's drills supply theirs as a fact and map
|
|
298
|
+
* no column at all.
|
|
154
299
|
*/
|
|
155
|
-
function angle(row, columns, unit, what) {
|
|
156
|
-
|
|
157
|
-
|
|
300
|
+
function angle(row, columns, unit, what, warn) {
|
|
301
|
+
// `cell` cannot be used here. It answers `undefined` for two different
|
|
302
|
+
// things — the family mapping no `SIG` column, and *this row* carrying no
|
|
303
|
+
// such key — and those are the regression and the vendor's silence
|
|
304
|
+
// respectively. Asking the map directly separates them: `column` is null
|
|
305
|
+
// only when the map has nothing, so an absent key is left to be read off the
|
|
306
|
+
// row like any other blank.
|
|
307
|
+
const column = columns.column('SIG', unit);
|
|
308
|
+
if (column === null) {
|
|
309
|
+
throw new VendorResponseError(what, `is a drill whose family maps no point angle column — EMUGE states one ` +
|
|
310
|
+
`per part and this adapter reads it, so a map without it is a regression`);
|
|
311
|
+
}
|
|
312
|
+
const raw = row[column];
|
|
313
|
+
if (raw === undefined || raw.trim() === '') {
|
|
314
|
+
warn(` WARNING: ${what}: the vendor publishes no point angle — omitted`);
|
|
315
|
+
return null;
|
|
316
|
+
}
|
|
317
|
+
const { value, stated } = parseMeasure(raw);
|
|
158
318
|
if (value === null || stated === 'inches' || stated === 'millimeters') {
|
|
159
|
-
throw new VendorResponseError(what, `
|
|
319
|
+
throw new VendorResponseError(what, `states a point angle of ${JSON.stringify(raw)}, which is not an angle — ` +
|
|
320
|
+
`an empty cell is the vendor's silence and is omitted, but a value ` +
|
|
321
|
+
`this cannot read is a property that has moved`);
|
|
160
322
|
}
|
|
161
323
|
return value;
|
|
162
324
|
}
|
|
@@ -248,6 +410,7 @@ function common(row, family, what, warn) {
|
|
|
248
410
|
materialNumber: what,
|
|
249
411
|
catalogNumber: row[CATALOG_NUMBER_COLUMN] ?? '',
|
|
250
412
|
description: row[DESCRIPTION_COLUMN] ?? '',
|
|
413
|
+
productLine: productLine(row, family),
|
|
251
414
|
substrate: substrate(row, what),
|
|
252
415
|
coating: coating(row),
|
|
253
416
|
coolantThrough: coolantThrough(row, what, warn),
|
|
@@ -308,27 +471,32 @@ export function endmillRecord(row, family, columns, options = {}) {
|
|
|
308
471
|
* `SIG` is a **mapped column**, which no other drill family in this package
|
|
309
472
|
* manages: the per-part detail record states the point angle outright, so
|
|
310
473
|
* nothing here is derived from a point length or assumed from a product line.
|
|
311
|
-
*
|
|
312
|
-
*
|
|
474
|
+
* It is also the one geometry key this record may not carry — one variant's
|
|
475
|
+
* cell is empty, and {@link angle} says what that costs. `NOF` is the one
|
|
476
|
+
* thing that is a fact, and `nonFerrous` with it — neither has a default
|
|
477
|
+
* anywhere, by design.
|
|
313
478
|
*/
|
|
314
479
|
export function drillRecord(row, family, columns, options = {}) {
|
|
315
480
|
const warn = options.warn ?? consoleWarn;
|
|
316
481
|
const opts = { warn };
|
|
317
482
|
const unit = fact(family, 'unit', family.unit);
|
|
318
483
|
const what = partNumber(row, family);
|
|
484
|
+
const geometry = {
|
|
485
|
+
DC: required(row, columns, 'DC', unit, what, opts),
|
|
486
|
+
SFDM: required(row, columns, 'SFDM', unit, what, opts),
|
|
487
|
+
OAL: required(row, columns, 'OAL', unit, what, opts),
|
|
488
|
+
LCF: required(row, columns, 'LCF', unit, what, opts),
|
|
489
|
+
NOF: fact(family, 'flutes', family.flutes),
|
|
490
|
+
};
|
|
491
|
+
const pointAngle = angle(row, columns, unit, what, warn);
|
|
492
|
+
if (pointAngle !== null)
|
|
493
|
+
geometry.SIG = pointAngle;
|
|
319
494
|
return toolRecord({
|
|
320
495
|
...common(row, family, what, warn),
|
|
321
496
|
kind: 'drill',
|
|
322
497
|
unit,
|
|
323
498
|
nonFerrous: fact(family, 'nonFerrous', family.nonFerrous),
|
|
324
|
-
geometry
|
|
325
|
-
DC: required(row, columns, 'DC', unit, what, opts),
|
|
326
|
-
SFDM: required(row, columns, 'SFDM', unit, what, opts),
|
|
327
|
-
OAL: required(row, columns, 'OAL', unit, what, opts),
|
|
328
|
-
LCF: required(row, columns, 'LCF', unit, what, opts),
|
|
329
|
-
NOF: fact(family, 'flutes', family.flutes),
|
|
330
|
-
SIG: angle(row, columns, unit, what),
|
|
331
|
-
},
|
|
499
|
+
geometry,
|
|
332
500
|
});
|
|
333
501
|
}
|
|
334
502
|
/**
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A family page's own title, for every brand on Kennametal's AEM platform.
|
|
3
|
+
*
|
|
4
|
+
* `scrape.ts` reads the variant *table* that a family page renders; this reads
|
|
5
|
+
* the heading above it. They are two requests against the same family because
|
|
6
|
+
* they are two different AEM resources — the table is a component node's
|
|
7
|
+
* `.variants.<code>.html` selector and the title is the page — and the second
|
|
8
|
+
* is what nothing else in this package publishes:
|
|
9
|
+
*
|
|
10
|
+
* ```html
|
|
11
|
+
* <h1>KenCut™ FF • HPFT • Square End • 6 Flutes • Plain Shank • Inch</h1>
|
|
12
|
+
* ```
|
|
13
|
+
*
|
|
14
|
+
* ## The slug does not matter
|
|
15
|
+
*
|
|
16
|
+
* The vendor's own links are `fam.<slug>.<code>.html`, and AEM resolves the
|
|
17
|
+
* page off the numeric code alone — `fam.x.100003783.html` answers 200 with
|
|
18
|
+
* exactly the page above. That is what makes this reachable from a
|
|
19
|
+
* `familyCode` and nothing else: `families/kennametal.ts` records the code
|
|
20
|
+
* because the variants endpoint needs it, and no family in it carries a slug.
|
|
21
|
+
*
|
|
22
|
+
* **`fam.<code>.html` — the code with no slug segment at all — 301s**, so the
|
|
23
|
+
* placeholder is load-bearing rather than decoration. {@link FAMILY_SLUG} is
|
|
24
|
+
* what fills it.
|
|
25
|
+
*
|
|
26
|
+
* ## Why the page is trusted to be the right one
|
|
27
|
+
*
|
|
28
|
+
* A URL built from a code that no longer names a family would answer *some*
|
|
29
|
+
* page, and a title read off it would be silently wrong for a whole table. So
|
|
30
|
+
* the page's own `data-product-code` is checked against the code that was
|
|
31
|
+
* asked for, and a mismatch throws. It is the same argument
|
|
32
|
+
* `scrape.parseVariantTable` makes for telling the vendor's no-results notice
|
|
33
|
+
* apart from a response that changed shape: a scrape that reports the wrong
|
|
34
|
+
* answer confidently is worse than one that stops.
|
|
35
|
+
*
|
|
36
|
+
* ## What the title is, and what only part of it is
|
|
37
|
+
*
|
|
38
|
+
* The whole `h1` is the family's name and reaches the CSV whole, under
|
|
39
|
+
* `conventions.FAMILY_TITLE_COLUMN`. Its **leading `•` segment** is the
|
|
40
|
+
* product line — `KenCut™ FF`, `HARVI™ I TE`, `VariMill™ Chip Splitters` —
|
|
41
|
+
* and that is the part `records.ToolRecord.productLine` keeps. Everything
|
|
42
|
+
* after it describes the shape, the flute count, the shank and the unit, which
|
|
43
|
+
* are already canonical geometry on the record.
|
|
44
|
+
*
|
|
45
|
+
* The split is the vendor's own punctuation and not a guess at where a name
|
|
46
|
+
* ends: every family title on both hosts is `•`-separated, and a title with no
|
|
47
|
+
* separator is its own product line.
|
|
48
|
+
*/
|
|
49
|
+
import type { Fetcher } from '../../fetch.js';
|
|
50
|
+
import { type AemBrandName } from '../../identity.js';
|
|
51
|
+
/**
|
|
52
|
+
* The URL shape of a family page.
|
|
53
|
+
*
|
|
54
|
+
* `{host}` and `{code}` are filled; `{slug}` is {@link FAMILY_SLUG}. Stated as
|
|
55
|
+
* one template beside `scrape.BASE` so that the two URLs this adapter builds
|
|
56
|
+
* are readable together.
|
|
57
|
+
*/
|
|
58
|
+
export declare const FAMILY_PAGE = "https://www.{host}/us/en/products/fam.{slug}.{code}.html";
|
|
59
|
+
/**
|
|
60
|
+
* What goes where the vendor writes a human-readable slug.
|
|
61
|
+
*
|
|
62
|
+
* AEM ignores it — see the module note — and this package has no slug to put
|
|
63
|
+
* there, because a `familyCode` is the only handle `families/kennametal.ts`
|
|
64
|
+
* records. A word rather than a single letter so that a request showing up in
|
|
65
|
+
* somebody's log says what it is.
|
|
66
|
+
*/
|
|
67
|
+
export declare const FAMILY_SLUG = "family";
|
|
68
|
+
/** The vendor's own separator between the parts of a family title. */
|
|
69
|
+
export declare const TITLE_SEPARATOR = "\u2022";
|
|
70
|
+
/** One family page's URL. */
|
|
71
|
+
export declare function familyPageUrl(code: string, brand?: AemBrandName): string;
|
|
72
|
+
/** What {@link parseFamilyPage} reads off a family page. */
|
|
73
|
+
export interface FamilyPage {
|
|
74
|
+
/** The `h1`, collapsed. */
|
|
75
|
+
readonly title: string;
|
|
76
|
+
/** The code the page states for itself, or null where it states none. */
|
|
77
|
+
readonly code: string | null;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* The family title and the page's own code.
|
|
81
|
+
*
|
|
82
|
+
* `htmlparser2` rather than a regex for the same reason `scrape.TableParser`
|
|
83
|
+
* uses it: these pages carry `°`, `Ø` and `™` in their text,
|
|
84
|
+
* and a `KenCut™ FF` reaching the CSV as eleven characters is the bug
|
|
85
|
+
* `decodeEntities` exists to prevent. The `™` in every title above is exactly
|
|
86
|
+
* that entity.
|
|
87
|
+
*
|
|
88
|
+
* The **first** `h1` is taken. The pages carry one, and reading the first is
|
|
89
|
+
* what makes a second one added below the fold somebody else's problem rather
|
|
90
|
+
* than a title that changes under a re-scrape.
|
|
91
|
+
*/
|
|
92
|
+
export declare function parseFamilyPage(html: string): FamilyPage;
|
|
93
|
+
/**
|
|
94
|
+
* A family title's leading segment — the product line.
|
|
95
|
+
*
|
|
96
|
+
* `''` in and `''` out, which is the one case the caller has to keep: a page
|
|
97
|
+
* with no `h1` states no line, and {@link fetchFamily} answers null for it
|
|
98
|
+
* rather than an empty name. See `records.ToolRecord.productLine`.
|
|
99
|
+
*/
|
|
100
|
+
export declare function productLineOf(title: string): string;
|
|
101
|
+
/** A family's title and product line, or nulls where the page states none. */
|
|
102
|
+
export interface Family {
|
|
103
|
+
readonly title: string;
|
|
104
|
+
readonly productLine: string | null;
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* One family page, read.
|
|
108
|
+
*
|
|
109
|
+
* The one network call in this module — the seam a test replaces, exactly as
|
|
110
|
+
* `scrape.fetchVariants` is.
|
|
111
|
+
*
|
|
112
|
+
* A page that answers with no `h1` is **not** an error: it is a family the
|
|
113
|
+
* vendor publishes without a heading, and the table below it is still a table
|
|
114
|
+
* of real parts. It comes back with an empty title and a null line, and the
|
|
115
|
+
* rows are written without either column rather than the scrape stopping. A
|
|
116
|
+
* page whose `data-product-code` names a *different* family is the other case
|
|
117
|
+
* and throws — see the module note.
|
|
118
|
+
*/
|
|
119
|
+
export declare function fetchFamily(fetcher: Fetcher, code: string, brand?: AemBrandName): Promise<Family>;
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A family page's own title, for every brand on Kennametal's AEM platform.
|
|
3
|
+
*
|
|
4
|
+
* `scrape.ts` reads the variant *table* that a family page renders; this reads
|
|
5
|
+
* the heading above it. They are two requests against the same family because
|
|
6
|
+
* they are two different AEM resources — the table is a component node's
|
|
7
|
+
* `.variants.<code>.html` selector and the title is the page — and the second
|
|
8
|
+
* is what nothing else in this package publishes:
|
|
9
|
+
*
|
|
10
|
+
* ```html
|
|
11
|
+
* <h1>KenCut™ FF • HPFT • Square End • 6 Flutes • Plain Shank • Inch</h1>
|
|
12
|
+
* ```
|
|
13
|
+
*
|
|
14
|
+
* ## The slug does not matter
|
|
15
|
+
*
|
|
16
|
+
* The vendor's own links are `fam.<slug>.<code>.html`, and AEM resolves the
|
|
17
|
+
* page off the numeric code alone — `fam.x.100003783.html` answers 200 with
|
|
18
|
+
* exactly the page above. That is what makes this reachable from a
|
|
19
|
+
* `familyCode` and nothing else: `families/kennametal.ts` records the code
|
|
20
|
+
* because the variants endpoint needs it, and no family in it carries a slug.
|
|
21
|
+
*
|
|
22
|
+
* **`fam.<code>.html` — the code with no slug segment at all — 301s**, so the
|
|
23
|
+
* placeholder is load-bearing rather than decoration. {@link FAMILY_SLUG} is
|
|
24
|
+
* what fills it.
|
|
25
|
+
*
|
|
26
|
+
* ## Why the page is trusted to be the right one
|
|
27
|
+
*
|
|
28
|
+
* A URL built from a code that no longer names a family would answer *some*
|
|
29
|
+
* page, and a title read off it would be silently wrong for a whole table. So
|
|
30
|
+
* the page's own `data-product-code` is checked against the code that was
|
|
31
|
+
* asked for, and a mismatch throws. It is the same argument
|
|
32
|
+
* `scrape.parseVariantTable` makes for telling the vendor's no-results notice
|
|
33
|
+
* apart from a response that changed shape: a scrape that reports the wrong
|
|
34
|
+
* answer confidently is worse than one that stops.
|
|
35
|
+
*
|
|
36
|
+
* ## What the title is, and what only part of it is
|
|
37
|
+
*
|
|
38
|
+
* The whole `h1` is the family's name and reaches the CSV whole, under
|
|
39
|
+
* `conventions.FAMILY_TITLE_COLUMN`. Its **leading `•` segment** is the
|
|
40
|
+
* product line — `KenCut™ FF`, `HARVI™ I TE`, `VariMill™ Chip Splitters` —
|
|
41
|
+
* and that is the part `records.ToolRecord.productLine` keeps. Everything
|
|
42
|
+
* after it describes the shape, the flute count, the shank and the unit, which
|
|
43
|
+
* are already canonical geometry on the record.
|
|
44
|
+
*
|
|
45
|
+
* The split is the vendor's own punctuation and not a guess at where a name
|
|
46
|
+
* ends: every family title on both hosts is `•`-separated, and a title with no
|
|
47
|
+
* separator is its own product line.
|
|
48
|
+
*/
|
|
49
|
+
import { Parser } from 'htmlparser2';
|
|
50
|
+
import { VendorResponseError } from '../../errors.js';
|
|
51
|
+
import { BRANDS } from '../../identity.js';
|
|
52
|
+
/**
|
|
53
|
+
* The URL shape of a family page.
|
|
54
|
+
*
|
|
55
|
+
* `{host}` and `{code}` are filled; `{slug}` is {@link FAMILY_SLUG}. Stated as
|
|
56
|
+
* one template beside `scrape.BASE` so that the two URLs this adapter builds
|
|
57
|
+
* are readable together.
|
|
58
|
+
*/
|
|
59
|
+
export const FAMILY_PAGE = 'https://www.{host}/us/en/products/fam.{slug}.{code}.html';
|
|
60
|
+
/**
|
|
61
|
+
* What goes where the vendor writes a human-readable slug.
|
|
62
|
+
*
|
|
63
|
+
* AEM ignores it — see the module note — and this package has no slug to put
|
|
64
|
+
* there, because a `familyCode` is the only handle `families/kennametal.ts`
|
|
65
|
+
* records. A word rather than a single letter so that a request showing up in
|
|
66
|
+
* somebody's log says what it is.
|
|
67
|
+
*/
|
|
68
|
+
export const FAMILY_SLUG = 'family';
|
|
69
|
+
/** The vendor's own separator between the parts of a family title. */
|
|
70
|
+
export const TITLE_SEPARATOR = '•';
|
|
71
|
+
/** The attribute a family page states its own code in. */
|
|
72
|
+
const PRODUCT_CODE_ATTR = 'data-product-code';
|
|
73
|
+
/** One family page's URL. */
|
|
74
|
+
export function familyPageUrl(code, brand = 'kennametal') {
|
|
75
|
+
const { host } = BRANDS[brand];
|
|
76
|
+
return FAMILY_PAGE.replace('{host}', host).replace('{slug}', FAMILY_SLUG).replace('{code}', code);
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* The family title and the page's own code.
|
|
80
|
+
*
|
|
81
|
+
* `htmlparser2` rather than a regex for the same reason `scrape.TableParser`
|
|
82
|
+
* uses it: these pages carry `°`, `Ø` and `™` in their text,
|
|
83
|
+
* and a `KenCut™ FF` reaching the CSV as eleven characters is the bug
|
|
84
|
+
* `decodeEntities` exists to prevent. The `™` in every title above is exactly
|
|
85
|
+
* that entity.
|
|
86
|
+
*
|
|
87
|
+
* The **first** `h1` is taken. The pages carry one, and reading the first is
|
|
88
|
+
* what makes a second one added below the fold somebody else's problem rather
|
|
89
|
+
* than a title that changes under a re-scrape.
|
|
90
|
+
*/
|
|
91
|
+
export function parseFamilyPage(html) {
|
|
92
|
+
let title = null;
|
|
93
|
+
let depth = 0;
|
|
94
|
+
let text = '';
|
|
95
|
+
let code = null;
|
|
96
|
+
const parser = new Parser({
|
|
97
|
+
onopentag: (tag, attribs) => {
|
|
98
|
+
if (tag === 'h1' && title === null)
|
|
99
|
+
depth += 1;
|
|
100
|
+
const stated = attribs[PRODUCT_CODE_ATTR];
|
|
101
|
+
if (code === null && stated !== undefined && stated !== '')
|
|
102
|
+
code = stated;
|
|
103
|
+
},
|
|
104
|
+
ontext: (chunk) => {
|
|
105
|
+
if (depth > 0)
|
|
106
|
+
text += chunk;
|
|
107
|
+
},
|
|
108
|
+
onclosetag: (tag) => {
|
|
109
|
+
if (tag === 'h1' && depth > 0) {
|
|
110
|
+
depth -= 1;
|
|
111
|
+
if (depth === 0) {
|
|
112
|
+
title = text.split(/\s+/).filter(Boolean).join(' ');
|
|
113
|
+
text = '';
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
},
|
|
117
|
+
}, { decodeEntities: true });
|
|
118
|
+
parser.write(html);
|
|
119
|
+
parser.end();
|
|
120
|
+
return { title: title ?? '', code };
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* A family title's leading segment — the product line.
|
|
124
|
+
*
|
|
125
|
+
* `''` in and `''` out, which is the one case the caller has to keep: a page
|
|
126
|
+
* with no `h1` states no line, and {@link fetchFamily} answers null for it
|
|
127
|
+
* rather than an empty name. See `records.ToolRecord.productLine`.
|
|
128
|
+
*/
|
|
129
|
+
export function productLineOf(title) {
|
|
130
|
+
return (title.split(TITLE_SEPARATOR)[0] ?? '').trim();
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* One family page, read.
|
|
134
|
+
*
|
|
135
|
+
* The one network call in this module — the seam a test replaces, exactly as
|
|
136
|
+
* `scrape.fetchVariants` is.
|
|
137
|
+
*
|
|
138
|
+
* A page that answers with no `h1` is **not** an error: it is a family the
|
|
139
|
+
* vendor publishes without a heading, and the table below it is still a table
|
|
140
|
+
* of real parts. It comes back with an empty title and a null line, and the
|
|
141
|
+
* rows are written without either column rather than the scrape stopping. A
|
|
142
|
+
* page whose `data-product-code` names a *different* family is the other case
|
|
143
|
+
* and throws — see the module note.
|
|
144
|
+
*/
|
|
145
|
+
export async function fetchFamily(fetcher, code, brand = 'kennametal') {
|
|
146
|
+
const url = familyPageUrl(code, brand);
|
|
147
|
+
const { title, code: stated } = parseFamilyPage(await fetcher.text(url));
|
|
148
|
+
if (stated !== null && stated !== code) {
|
|
149
|
+
throw new VendorResponseError(url, `is the page for ${JSON.stringify(stated)} and not ${JSON.stringify(code)} — ` +
|
|
150
|
+
`the family code was retired or redirected, and its title would name the ` +
|
|
151
|
+
`wrong product line for every row of the table`);
|
|
152
|
+
}
|
|
153
|
+
const line = productLineOf(title);
|
|
154
|
+
return { title, productLine: line === '' ? null : line };
|
|
155
|
+
}
|
|
@@ -52,6 +52,22 @@ export declare const CATALOG_NUMBER = "ISO Catalog Number";
|
|
|
52
52
|
* absent coating column is a table that states no coating, not a scrape fault.
|
|
53
53
|
*/
|
|
54
54
|
export declare const COATING = "Coating";
|
|
55
|
+
/**
|
|
56
|
+
* The column {@link productLine} reads, and the one this adapter names rather
|
|
57
|
+
* than the vendor.
|
|
58
|
+
*
|
|
59
|
+
* `conventions.FAMILY_TITLE_COLUMN` is vendor-neutral because a second vendor
|
|
60
|
+
* publishing a family title should write the same column; this one is not,
|
|
61
|
+
* because it holds a *segment* of that title that this adapter chose to split
|
|
62
|
+
* out. A vendor whose product line arrives some other way has no business
|
|
63
|
+
* writing here — EMUGE reads its own `product line` and `Geometry` columns and
|
|
64
|
+
* never sees this one.
|
|
65
|
+
*
|
|
66
|
+
* Both columns are written by `scrape.scrapeFamily` under `familyTitle`, and
|
|
67
|
+
* a CSV scraped before that option existed has neither — which is why
|
|
68
|
+
* {@link productLine} reads it as absent rather than required.
|
|
69
|
+
*/
|
|
70
|
+
export declare const PRODUCT_LINE_COLUMN = "Product Line";
|
|
55
71
|
/**
|
|
56
72
|
* A drill, in the family's native unit system per its `unit` fact.
|
|
57
73
|
*
|
|
@@ -55,6 +55,35 @@ export const CATALOG_NUMBER = 'ISO Catalog Number';
|
|
|
55
55
|
* absent coating column is a table that states no coating, not a scrape fault.
|
|
56
56
|
*/
|
|
57
57
|
export const COATING = 'Coating';
|
|
58
|
+
/**
|
|
59
|
+
* The column {@link productLine} reads, and the one this adapter names rather
|
|
60
|
+
* than the vendor.
|
|
61
|
+
*
|
|
62
|
+
* `conventions.FAMILY_TITLE_COLUMN` is vendor-neutral because a second vendor
|
|
63
|
+
* publishing a family title should write the same column; this one is not,
|
|
64
|
+
* because it holds a *segment* of that title that this adapter chose to split
|
|
65
|
+
* out. A vendor whose product line arrives some other way has no business
|
|
66
|
+
* writing here — EMUGE reads its own `product line` and `Geometry` columns and
|
|
67
|
+
* never sees this one.
|
|
68
|
+
*
|
|
69
|
+
* Both columns are written by `scrape.scrapeFamily` under `familyTitle`, and
|
|
70
|
+
* a CSV scraped before that option existed has neither — which is why
|
|
71
|
+
* {@link productLine} reads it as absent rather than required.
|
|
72
|
+
*/
|
|
73
|
+
export const PRODUCT_LINE_COLUMN = 'Product Line';
|
|
74
|
+
/**
|
|
75
|
+
* The vendor's own product line for this row, or null where the CSV has none.
|
|
76
|
+
*
|
|
77
|
+
* **An absent column is not a fault.** It is a table scraped without
|
|
78
|
+
* `scrape.FamilyOptions.familyTitle`, or a family page the vendor publishes
|
|
79
|
+
* with no heading, and both are rows that are otherwise complete. `null` is
|
|
80
|
+
* the answer `records.ToolRecord.productLine` defines for exactly this — see
|
|
81
|
+
* its docstring on why that is not `''`.
|
|
82
|
+
*/
|
|
83
|
+
function productLine(row) {
|
|
84
|
+
const stated = (row[PRODUCT_LINE_COLUMN] ?? '').trim();
|
|
85
|
+
return stated === '' ? null : stated;
|
|
86
|
+
}
|
|
58
87
|
/**
|
|
59
88
|
* One canonical dimension, or null when this family maps or publishes none.
|
|
60
89
|
*
|
|
@@ -160,6 +189,7 @@ export function drillRecord(row, family, columns) {
|
|
|
160
189
|
vendor: BRANDS[familyBrand(family)].vendor,
|
|
161
190
|
materialNumber: what,
|
|
162
191
|
catalogNumber: row[CATALOG_NUMBER] ?? '',
|
|
192
|
+
productLine: productLine(row),
|
|
163
193
|
// Kennametal publishes no description column on any table. The catalog
|
|
164
194
|
// number is already on the record, and repeating it here said nothing —
|
|
165
195
|
// see `records.ToolRecord.description`.
|
|
@@ -210,6 +240,7 @@ export function tapRecord(row, family, columns) {
|
|
|
210
240
|
// carry the size. The catalog number no longer leads it — it is already on
|
|
211
241
|
// the record, and a description that restates one is not a description.
|
|
212
242
|
description: tdz,
|
|
243
|
+
productLine: productLine(row),
|
|
213
244
|
kind: 'tap',
|
|
214
245
|
unit,
|
|
215
246
|
substrate: fact(family, 'bmc', family.bmc),
|
|
@@ -251,6 +282,7 @@ export function endmillRecord(row, family, columns) {
|
|
|
251
282
|
catalogNumber: row[CATALOG_NUMBER] ?? '',
|
|
252
283
|
// No description column here either. See the drill mapper.
|
|
253
284
|
description: '',
|
|
285
|
+
productLine: productLine(row),
|
|
254
286
|
kind: 'endmill',
|
|
255
287
|
unit,
|
|
256
288
|
substrate: fact(family, 'bmc', family.bmc),
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
*/
|
|
26
26
|
import type { Fetcher } from '../../fetch.js';
|
|
27
27
|
import { type AemBrandName } from '../../identity.js';
|
|
28
|
-
import type
|
|
28
|
+
import { type ScrapeResult } from '../../scrape.js';
|
|
29
29
|
export declare const BASE: string;
|
|
30
30
|
/**
|
|
31
31
|
* The Hybris/Solr facet string that scopes a request to a family's active
|
|
@@ -101,11 +101,34 @@ export declare function parseVariantTable(html: string): VariantTable;
|
|
|
101
101
|
* data row.
|
|
102
102
|
*/
|
|
103
103
|
export declare function columnNames(header: readonly Cell[]): (string | null)[];
|
|
104
|
+
/** What {@link scrapeFamily} accepts beyond its tags. */
|
|
105
|
+
export interface FamilyOptions {
|
|
106
|
+
/**
|
|
107
|
+
* Also read the family page's own title, and tag every row with it.
|
|
108
|
+
*
|
|
109
|
+
* **Opt-in, and a second request.** The variants table states no product
|
|
110
|
+
* line — the vendor puts it in the `h1` above the table, which is a
|
|
111
|
+
* different AEM resource — so this is the one thing here that costs a
|
|
112
|
+
* request the table did not. A caller that only wants dimensions should not
|
|
113
|
+
* pay for it, and every existing caller keeps the transport it had.
|
|
114
|
+
*
|
|
115
|
+
* See `family.ts` for what the title is and which part of it becomes
|
|
116
|
+
* `records.ToolRecord.productLine`.
|
|
117
|
+
*/
|
|
118
|
+
readonly familyTitle?: boolean;
|
|
119
|
+
/** Milliseconds between the two requests. Zero in tests. */
|
|
120
|
+
readonly delayMs?: number;
|
|
121
|
+
}
|
|
104
122
|
/**
|
|
105
123
|
* Scrape one family into rows.
|
|
106
124
|
*
|
|
107
125
|
* `tags` is a sequence of `[name, value]` pairs appended to every row as
|
|
108
126
|
* constant columns — used to tag facts the table doesn't state, e.g. the
|
|
109
127
|
* thread system on a tap family.
|
|
128
|
+
*
|
|
129
|
+
* `options.familyTitle` adds two more of exactly that kind. They are tags
|
|
130
|
+
* rather than parsed columns because that is what they are: one string per
|
|
131
|
+
* family, constant down its whole table, which is the case the `tags` seam was
|
|
132
|
+
* built for.
|
|
110
133
|
*/
|
|
111
|
-
export declare function scrapeFamily(fetcher: Fetcher, code: string, brand?: AemBrandName, tags?: readonly Tag[]): Promise<ScrapeResult>;
|
|
134
|
+
export declare function scrapeFamily(fetcher: Fetcher, code: string, brand?: AemBrandName, tags?: readonly Tag[], options?: FamilyOptions): Promise<ScrapeResult>;
|