@toolpath/tool-scraper 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +98 -0
- package/dist/conventions.d.ts +124 -0
- package/dist/conventions.js +143 -0
- package/dist/errors.d.ts +46 -0
- package/dist/errors.js +53 -0
- package/dist/families/destinytool.d.ts +49 -0
- package/dist/families/destinytool.js +55 -0
- package/dist/families/index.d.ts +59 -0
- package/dist/families/index.js +91 -0
- package/dist/families/kennametal.d.ts +757 -0
- package/dist/families/kennametal.js +660 -0
- package/dist/families/regofix.d.ts +185 -0
- package/dist/families/regofix.js +250 -0
- package/dist/family.d.ts +130 -0
- package/dist/family.js +38 -0
- package/dist/fetch.d.ts +98 -0
- package/dist/fetch.js +116 -0
- package/dist/identity.d.ts +133 -0
- package/dist/identity.js +118 -0
- package/dist/index.d.ts +31 -0
- package/dist/index.js +31 -0
- package/dist/node/cad-mirror.d.ts +55 -0
- package/dist/node/cad-mirror.js +89 -0
- package/dist/node/cli.d.ts +35 -0
- package/dist/node/cli.js +340 -0
- package/dist/node/csv.d.ts +47 -0
- package/dist/node/csv.js +123 -0
- package/dist/node/index.d.ts +16 -0
- package/dist/node/index.js +16 -0
- package/dist/node/main.d.ts +13 -0
- package/dist/node/main.js +14 -0
- package/dist/node/paths.d.ts +60 -0
- package/dist/node/paths.js +80 -0
- package/dist/node/receipts.d.ts +100 -0
- package/dist/node/receipts.js +107 -0
- package/dist/order.d.ts +10 -0
- package/dist/order.js +12 -0
- package/dist/provenance.d.ts +125 -0
- package/dist/provenance.js +133 -0
- package/dist/records.d.ts +305 -0
- package/dist/records.js +297 -0
- package/dist/registry.d.ts +63 -0
- package/dist/registry.js +145 -0
- package/dist/scrape.d.ts +70 -0
- package/dist/scrape.js +37 -0
- package/dist/thread.d.ts +48 -0
- package/dist/thread.js +98 -0
- package/dist/uuid5.d.ts +31 -0
- package/dist/uuid5.js +64 -0
- package/dist/vendors/destinytool/index.d.ts +11 -0
- package/dist/vendors/destinytool/index.js +11 -0
- package/dist/vendors/destinytool/records.d.ts +118 -0
- package/dist/vendors/destinytool/records.js +266 -0
- package/dist/vendors/destinytool/scrape.d.ts +108 -0
- package/dist/vendors/destinytool/scrape.js +192 -0
- package/dist/vendors/kennametal/cad.d.ts +87 -0
- package/dist/vendors/kennametal/cad.js +119 -0
- package/dist/vendors/kennametal/index.d.ts +21 -0
- package/dist/vendors/kennametal/index.js +21 -0
- package/dist/vendors/kennametal/materials.d.ts +143 -0
- package/dist/vendors/kennametal/materials.js +200 -0
- package/dist/vendors/kennametal/records.d.ts +88 -0
- package/dist/vendors/kennametal/records.js +241 -0
- package/dist/vendors/kennametal/scrape.d.ts +111 -0
- package/dist/vendors/kennametal/scrape.js +226 -0
- package/dist/vendors/kennametal/thread-column.d.ts +28 -0
- package/dist/vendors/kennametal/thread-column.js +41 -0
- package/dist/vendors/regofix/index.d.ts +8 -0
- package/dist/vendors/regofix/index.js +8 -0
- package/dist/vendors/regofix/scrape.d.ts +237 -0
- package/dist/vendors/regofix/scrape.js +521 -0
- package/package.json +76 -0
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Kennametal rows -> {@link ToolRecord}. The adapter half of the record seam.
|
|
3
|
+
*
|
|
4
|
+
* Everything here knows Kennametal's column vocabulary and nothing here knows
|
|
5
|
+
* what a record becomes. The reverse is true of whatever consumes one: it
|
|
6
|
+
* knows no vendor at all.
|
|
7
|
+
*
|
|
8
|
+
* What stayed on this side of the line, and why each is a *vendor* fact rather
|
|
9
|
+
* than a domain one:
|
|
10
|
+
*
|
|
11
|
+
* - **Which column holds a canonical field.** Declared per family as
|
|
12
|
+
* `columns: { DC: 'D1', … }` and resolved through `ColumnMap`, which appends
|
|
13
|
+
* the unit suffix. A vendor with different labels declares different labels.
|
|
14
|
+
* - **Where the identity strings live.** `Material Number` and `ISO Catalog
|
|
15
|
+
* Number` are Kennametal's header text — the labels
|
|
16
|
+
* `conventions.IDENTITY_COLUMNS` took as the convention because this vendor
|
|
17
|
+
* was first, and which Destiny Tool then did not follow.
|
|
18
|
+
* - **Which column is the grade.** A drill and an end mill carry a carbide
|
|
19
|
+
* `Grade`; a tap has no carbide grade and carries `Coating`, the surface
|
|
20
|
+
* treatment, in the record's `grade` field. That is Kennametal's table
|
|
21
|
+
* shape, not a rule about taps everywhere.
|
|
22
|
+
* - **That a tap's unit system is per row.** `Thread System` is a constant tag
|
|
23
|
+
* column this package appends at scrape time, and a metric and an inch tap
|
|
24
|
+
* can sit in one family — so a tap's `unit` is read per row where a drill's
|
|
25
|
+
* and an end mill's come from config.
|
|
26
|
+
* - **The optional columns.** `Re`, `L3` and `D3` are present on some families
|
|
27
|
+
* and absent on others, and the *absence* means something specific: no
|
|
28
|
+
* corner radius is a square end (RE 0); no `L3` on a plain-shank tool means
|
|
29
|
+
* AP1MAX is the shoulder length too; no `D3` means the shoulder is the
|
|
30
|
+
* cutting diameter. Those three fallbacks are Kennametal-table facts and
|
|
31
|
+
* they stay here.
|
|
32
|
+
*
|
|
33
|
+
* What deliberately stays out: whether a corner radius makes a tool a bull
|
|
34
|
+
* nose, `LB` and `assemblyGaugeLength` being `OAL` on a bare tool, and every
|
|
35
|
+
* cutting preset. Those are true of a tool, not of a table.
|
|
36
|
+
*/
|
|
37
|
+
import { ScraperConfigError, VendorResponseError } from '../../errors.js';
|
|
38
|
+
import { familyBrand } from '../../family.js';
|
|
39
|
+
import { BRANDS } from '../../identity.js';
|
|
40
|
+
import { toolRecord } from '../../records.js';
|
|
41
|
+
import { threadMajorDiameter } from '../../thread.js';
|
|
42
|
+
import { MATERIALS_COLUMN, materialClasses } from './materials.js';
|
|
43
|
+
/**
|
|
44
|
+
* Kennametal's identity columns. Named here rather than inline so a table that
|
|
45
|
+
* renames one fails in a single place.
|
|
46
|
+
*/
|
|
47
|
+
export const MATERIAL_NUMBER = 'Material Number';
|
|
48
|
+
export const CATALOG_NUMBER = 'ISO Catalog Number';
|
|
49
|
+
/**
|
|
50
|
+
* One canonical dimension, or null when this family maps or publishes none.
|
|
51
|
+
*
|
|
52
|
+
* Null is a real state and the callers distinguish it: an absent `Re` is a
|
|
53
|
+
* square-end tool, an absent `L3` is a plain shank. It is never a silent zero,
|
|
54
|
+
* because a zero corner radius and an unpublished one are the same number and
|
|
55
|
+
* different facts.
|
|
56
|
+
*/
|
|
57
|
+
function dim(row, columns, canonical, unit) {
|
|
58
|
+
const column = columns.column(canonical, unit);
|
|
59
|
+
if (column === null)
|
|
60
|
+
return null;
|
|
61
|
+
const raw = row[column];
|
|
62
|
+
if (raw === undefined || raw.trim() === '')
|
|
63
|
+
return null;
|
|
64
|
+
const value = Number(raw);
|
|
65
|
+
if (!Number.isFinite(value)) {
|
|
66
|
+
throw new VendorResponseError(column, `${JSON.stringify(raw)} is not a number`);
|
|
67
|
+
}
|
|
68
|
+
return value;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* A dimension the kind requires. `checkColumnMap` has already refused a family
|
|
72
|
+
* that maps none, so what this catches is a *row* the vendor left empty —
|
|
73
|
+
* which is a scrape problem, not a config one, and says so.
|
|
74
|
+
*/
|
|
75
|
+
function require_(row, columns, canonical, unit, what) {
|
|
76
|
+
const value = dim(row, columns, canonical, unit);
|
|
77
|
+
if (value === null) {
|
|
78
|
+
throw new VendorResponseError(what, `no value for ${canonical} in column ` + `${JSON.stringify(columns.column(canonical, unit))}`);
|
|
79
|
+
}
|
|
80
|
+
return value;
|
|
81
|
+
}
|
|
82
|
+
/** A per-family constant a mapper cannot proceed without. */
|
|
83
|
+
function fact(family, key, value) {
|
|
84
|
+
if (value === undefined) {
|
|
85
|
+
throw new ScraperConfigError(family.id, `a ${family.kind} family must state ${key} as a fact`);
|
|
86
|
+
}
|
|
87
|
+
return value;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* The `Thread System` tag, refusing anything that is not one of the two.
|
|
91
|
+
*
|
|
92
|
+
* Read here and nowhere else, because the two readers of a cast tag defaulted
|
|
93
|
+
* in *opposite* directions: this module took anything that was not `'inch'` as
|
|
94
|
+
* metric, and `thread.threadMajorDiameter` took anything that was not
|
|
95
|
+
* `'metric'` as inch. A missing, empty or capitalised tag then produced a
|
|
96
|
+
* record whose `DC` was parsed in inches and whose `TP`/`SFDM`/`OAL`/`LCF`
|
|
97
|
+
* came from the `_mm` columns — the silent unit mix `conventions` exists to
|
|
98
|
+
* make impossible.
|
|
99
|
+
*/
|
|
100
|
+
function threadSystem(row, what) {
|
|
101
|
+
const value = row['Thread System'] ?? '';
|
|
102
|
+
if (value !== 'metric' && value !== 'inch') {
|
|
103
|
+
throw new VendorResponseError(what, `Thread System is ${JSON.stringify(value)}, not "metric" or "inch" — ` +
|
|
104
|
+
`a tap family states it as a constant column on the scrape`);
|
|
105
|
+
}
|
|
106
|
+
return value;
|
|
107
|
+
}
|
|
108
|
+
/** An integer column the vendor always publishes — the flute count. */
|
|
109
|
+
function count(row, column, what) {
|
|
110
|
+
const value = Number.parseInt(row[column] ?? '', 10);
|
|
111
|
+
if (!Number.isInteger(value)) {
|
|
112
|
+
throw new VendorResponseError(what, `no integer in column ${JSON.stringify(column)}`);
|
|
113
|
+
}
|
|
114
|
+
return value;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* A drill, in the family's native unit system per its `unit` fact.
|
|
118
|
+
*
|
|
119
|
+
* Every drill table publishes both unit columns, so `unit` is config and never
|
|
120
|
+
* inferred: it decides which column is read and what a machinist is shown.
|
|
121
|
+
* Getting it wrong converts cleanly and prints 5.9531 mm where the part
|
|
122
|
+
* ordered is a 15/64 in KenDrill TXD.
|
|
123
|
+
*
|
|
124
|
+
* L4 (max drilling depth) and L5 (point length) are mapped by no family and
|
|
125
|
+
* reach no record — but L5 is not inert, because it is what pins `pointAngle`
|
|
126
|
+
* on a family whose table states none. See `families/kennametal.ts`.
|
|
127
|
+
*/
|
|
128
|
+
export function drillRecord(row, family, columns) {
|
|
129
|
+
const unit = fact(family, 'unit', family.unit);
|
|
130
|
+
const what = row[MATERIAL_NUMBER] ?? '';
|
|
131
|
+
return toolRecord({
|
|
132
|
+
vendor: BRANDS[familyBrand(family)].vendor,
|
|
133
|
+
materialNumber: what,
|
|
134
|
+
catalogNumber: row[CATALOG_NUMBER] ?? '',
|
|
135
|
+
description: row[CATALOG_NUMBER] ?? '',
|
|
136
|
+
kind: 'drill',
|
|
137
|
+
unit,
|
|
138
|
+
substrate: fact(family, 'bmc', family.bmc),
|
|
139
|
+
grade: row['Grade'] ?? '',
|
|
140
|
+
materialGroups: materialClasses(row[MATERIALS_COLUMN]),
|
|
141
|
+
coolantThrough: fact(family, 'coolantThrough', family.coolantThrough),
|
|
142
|
+
nonFerrous: fact(family, 'nonFerrous', family.nonFerrous),
|
|
143
|
+
geometry: {
|
|
144
|
+
DC: require_(row, columns, 'DC', unit, what),
|
|
145
|
+
SFDM: require_(row, columns, 'SFDM', unit, what),
|
|
146
|
+
OAL: require_(row, columns, 'OAL', unit, what),
|
|
147
|
+
LCF: require_(row, columns, 'LCF', unit, what),
|
|
148
|
+
NOF: fact(family, 'flutes', family.flutes),
|
|
149
|
+
SIG: fact(family, 'pointAngle', family.pointAngle),
|
|
150
|
+
},
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* A tap, in **its own** native unit system rather than the family's.
|
|
155
|
+
*
|
|
156
|
+
* `Thread System` is a constant column the scraper tags on, because the table
|
|
157
|
+
* does not state it; metric taps read the mm columns and inch taps the inch
|
|
158
|
+
* ones. `Thread Pitch` is already native-unit — derived from `D1-TDZ` by
|
|
159
|
+
* `thread-column.addThreadPitch` — which is why `TP` is dimensional but
|
|
160
|
+
* unsuffixed.
|
|
161
|
+
*
|
|
162
|
+
* `DC` is **derived, not read**: a tap table publishes a thread designation
|
|
163
|
+
* (`#2-56`, `M6 X 1`) and no major-diameter column, so the major diameter is
|
|
164
|
+
* parsed out of the designation. That is arithmetic over a standard, which is
|
|
165
|
+
* why `threadMajorDiameter` sits in the core.
|
|
166
|
+
*/
|
|
167
|
+
export function tapRecord(row, family, columns) {
|
|
168
|
+
const system = threadSystem(row, row[MATERIAL_NUMBER] ?? '');
|
|
169
|
+
const unit = system === 'inch' ? 'inches' : 'millimeters';
|
|
170
|
+
const tdz = row['D1-TDZ'] ?? '';
|
|
171
|
+
const what = row[MATERIAL_NUMBER] ?? '';
|
|
172
|
+
return toolRecord({
|
|
173
|
+
vendor: BRANDS[familyBrand(family)].vendor,
|
|
174
|
+
materialNumber: what,
|
|
175
|
+
catalogNumber: row[CATALOG_NUMBER] ?? '',
|
|
176
|
+
// The designation is part of what a tap *is*, and the catalog number alone
|
|
177
|
+
// does not carry the size.
|
|
178
|
+
description: `${row[CATALOG_NUMBER] ?? ''} ${tdz}`,
|
|
179
|
+
kind: 'tap',
|
|
180
|
+
unit,
|
|
181
|
+
substrate: fact(family, 'bmc', family.bmc),
|
|
182
|
+
// A tap has no carbide grade; the record's grade carries the coating.
|
|
183
|
+
grade: row['Coating'] ?? '',
|
|
184
|
+
materialGroups: materialClasses(row[MATERIALS_COLUMN]),
|
|
185
|
+
coolantThrough: false,
|
|
186
|
+
geometry: {
|
|
187
|
+
DC: threadMajorDiameter(tdz, system),
|
|
188
|
+
TP: require_(row, columns, 'TP', unit, what),
|
|
189
|
+
SFDM: require_(row, columns, 'SFDM', unit, what),
|
|
190
|
+
OAL: require_(row, columns, 'OAL', unit, what),
|
|
191
|
+
LCF: require_(row, columns, 'LCF', unit, what),
|
|
192
|
+
NOF: count(row, 'Z', what),
|
|
193
|
+
},
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* A solid end mill, native unit per the family's `unit` fact.
|
|
198
|
+
*
|
|
199
|
+
* Three optional columns, and each absence carries a meaning this table
|
|
200
|
+
* assigns rather than one the domain does:
|
|
201
|
+
*
|
|
202
|
+
* - **no `Re`** → a square-end family, corner radius 0;
|
|
203
|
+
* - **no `L3`** → nothing below the flutes to reach past, so the maximum flute
|
|
204
|
+
* length is the shoulder length too (the WIDIA VariMill tables);
|
|
205
|
+
* - **no `D3`** → a plain shank, so the shoulder is the cutting diameter.
|
|
206
|
+
*
|
|
207
|
+
* Whether a radius makes it a bull nose is a consumer's call, not this one.
|
|
208
|
+
*/
|
|
209
|
+
export function endmillRecord(row, family, columns) {
|
|
210
|
+
const unit = fact(family, 'unit', family.unit);
|
|
211
|
+
const what = row[MATERIAL_NUMBER] ?? '';
|
|
212
|
+
const dc = require_(row, columns, 'DC', unit, what);
|
|
213
|
+
const fluteLength = require_(row, columns, 'LCF', unit, what);
|
|
214
|
+
return toolRecord({
|
|
215
|
+
vendor: BRANDS[familyBrand(family)].vendor,
|
|
216
|
+
materialNumber: what,
|
|
217
|
+
catalogNumber: row[CATALOG_NUMBER] ?? '',
|
|
218
|
+
description: row[CATALOG_NUMBER] ?? '',
|
|
219
|
+
kind: 'endmill',
|
|
220
|
+
unit,
|
|
221
|
+
substrate: fact(family, 'bmc', family.bmc),
|
|
222
|
+
grade: row['Grade'] ?? '',
|
|
223
|
+
materialGroups: materialClasses(row[MATERIALS_COLUMN]),
|
|
224
|
+
coolantThrough: fact(family, 'coolantThrough', family.coolantThrough),
|
|
225
|
+
geometry: {
|
|
226
|
+
DC: dc,
|
|
227
|
+
RE: dim(row, columns, 'RE', unit) ?? 0,
|
|
228
|
+
SFDM: require_(row, columns, 'SFDM', unit, what),
|
|
229
|
+
OAL: require_(row, columns, 'OAL', unit, what),
|
|
230
|
+
LCF: fluteLength,
|
|
231
|
+
'shoulder-length': dim(row, columns, 'shoulder-length', unit) ?? fluteLength,
|
|
232
|
+
'shoulder-diameter': dim(row, columns, 'shoulder-diameter', unit) ?? dc,
|
|
233
|
+
NOF: count(row, 'Z', what),
|
|
234
|
+
},
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
export const RECORD_MAPPERS = {
|
|
238
|
+
drill: drillRecord,
|
|
239
|
+
tap: tapRecord,
|
|
240
|
+
endmill: endmillRecord,
|
|
241
|
+
};
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Family page -> rows, for every brand on Kennametal's AEM platform.
|
|
3
|
+
*
|
|
4
|
+
* Family pages render their variant table client-side, but the table comes
|
|
5
|
+
* from a plain AEM GET that returns ALL variants as one HTML table — no
|
|
6
|
+
* pagination, no JS, no bot-blocking. The package runbook records how the
|
|
7
|
+
* endpoint was found and how to read a new brand's component node off a family
|
|
8
|
+
* page.
|
|
9
|
+
*
|
|
10
|
+
* Column identity comes from the `<th>` class attribute (e.g.
|
|
11
|
+
* `"DRL_CUT_D1_MIN metric"`), NOT the visible label — labels like `D1` repeat
|
|
12
|
+
* across unit pairs and non-dimensional columns like wire size.
|
|
13
|
+
*
|
|
14
|
+
* ## The one dependency in this package, and why it is here
|
|
15
|
+
*
|
|
16
|
+
* Node ships no HTML parser, so this module declares `htmlparser2` — and
|
|
17
|
+
* declares it rather than putting it in the core, on the rule that *a
|
|
18
|
+
* transport that needs a parser declares it; nothing shared may pretend to
|
|
19
|
+
* know how a vendor serves a table.* This module is the only importer:
|
|
20
|
+
* REGO-FIX's XML is read by regex and Destiny Tool's Firestore is JSON.
|
|
21
|
+
*
|
|
22
|
+
* `decodeEntities` is on — these tables carry `°` and `Ø` in cell
|
|
23
|
+
* text, and a raw `ø` in a description would reach the CSV as five
|
|
24
|
+
* characters.
|
|
25
|
+
*/
|
|
26
|
+
import type { Fetcher } from '../../fetch.js';
|
|
27
|
+
import { type AemBrandName } from '../../identity.js';
|
|
28
|
+
import type { ScrapeResult } from '../../scrape.js';
|
|
29
|
+
export declare const BASE: string;
|
|
30
|
+
/**
|
|
31
|
+
* The Hybris/Solr facet string that scopes a request to a family's active
|
|
32
|
+
* variants. The family code in the URL path already scopes to the family, so
|
|
33
|
+
* this only drops the discontinued ones. `materials` appends a second facet to
|
|
34
|
+
* it; nothing else should need to.
|
|
35
|
+
*/
|
|
36
|
+
export declare const ACTIVE_ONLY = ":relevance:obsoleteFacet:false";
|
|
37
|
+
/**
|
|
38
|
+
* The class the vendor renders instead of a table when a query matches
|
|
39
|
+
* nothing. Distinguishing that from a response we failed to parse is the whole
|
|
40
|
+
* reason {@link parseVariantTable} looks for it.
|
|
41
|
+
*/
|
|
42
|
+
export declare const NO_RESULTS = "class=\"no-results\"";
|
|
43
|
+
/**
|
|
44
|
+
* One parsed cell: its collapsed text, and the tag's attributes. Column
|
|
45
|
+
* identity lives in those attributes, never in the text — see the note above.
|
|
46
|
+
*/
|
|
47
|
+
export type Cell = readonly [text: string, attrs: Record<string, string>];
|
|
48
|
+
export type Row = Cell[];
|
|
49
|
+
/** A constant column appended to every row — `['Thread System', 'metric']`. */
|
|
50
|
+
export type Tag = readonly [name: string, value: string];
|
|
51
|
+
/** Collects rows of `[cellText, attributes]` pairs. */
|
|
52
|
+
export declare class TableParser {
|
|
53
|
+
readonly rows: Row[];
|
|
54
|
+
private row;
|
|
55
|
+
private cell;
|
|
56
|
+
private attrs;
|
|
57
|
+
/** Feed a whole document. */
|
|
58
|
+
feed(html: string): void;
|
|
59
|
+
}
|
|
60
|
+
/** One family's variants URL. */
|
|
61
|
+
export declare function variantsUrl(code: string, brand?: AemBrandName, query?: string): string;
|
|
62
|
+
/**
|
|
63
|
+
* One family's variants response, as HTML.
|
|
64
|
+
*
|
|
65
|
+
* The one network call in this module, and the seam every test replaces —
|
|
66
|
+
* everything below it parses a string.
|
|
67
|
+
*/
|
|
68
|
+
export declare function fetchVariants(fetcher: Fetcher, code: string, brand?: AemBrandName, query?: string): Promise<string>;
|
|
69
|
+
/** What {@link parseVariantTable} answers with. */
|
|
70
|
+
export interface VariantTable {
|
|
71
|
+
/** Null when the vendor's own no-results notice came back instead. */
|
|
72
|
+
header: Row | null;
|
|
73
|
+
rows: Row[];
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* The header row and the data rows of a variants response.
|
|
77
|
+
*
|
|
78
|
+
* Both callers need the same two subtleties and neither is obvious, which is
|
|
79
|
+
* why this is one function rather than two copies: the header is found by its
|
|
80
|
+
* "Material Number" cell rather than by position (the response opens with
|
|
81
|
+
* filter rows that are also `<tr>`s), and a data row is one exactly as long as
|
|
82
|
+
* the header whose second cell is all digits. Header, filter and footer rows
|
|
83
|
+
* all fail one of those two tests.
|
|
84
|
+
*
|
|
85
|
+
* A **matched-nothing** response has no table in it at all, only the vendor's
|
|
86
|
+
* own no-results notice, and comes back with a null header. That is the
|
|
87
|
+
* ordinary answer to a facet query for a group a family isn't rated for
|
|
88
|
+
* (`materials`), and a hard error for a family scrape — so it is returned as a
|
|
89
|
+
* state rather than thrown, and {@link scrapeFamily} is what decides it's fatal.
|
|
90
|
+
*
|
|
91
|
+
* A response with neither the notice nor a header is a *third* thing: the
|
|
92
|
+
* endpoint changed shape. That throws, because silently reporting zero rows
|
|
93
|
+
* would look exactly like the vendor discontinuing a family.
|
|
94
|
+
*/
|
|
95
|
+
export declare function parseVariantTable(html: string): VariantTable;
|
|
96
|
+
/**
|
|
97
|
+
* Build unique column names from header label + `th` class unit hints.
|
|
98
|
+
*
|
|
99
|
+
* A `null` entry is a column the CSV drops — the checkbox, the sticky CTA, and
|
|
100
|
+
* the marketing flag. The list stays positional so it can be zipped against a
|
|
101
|
+
* data row.
|
|
102
|
+
*/
|
|
103
|
+
export declare function columnNames(header: readonly Cell[]): (string | null)[];
|
|
104
|
+
/**
|
|
105
|
+
* Scrape one family into rows.
|
|
106
|
+
*
|
|
107
|
+
* `tags` is a sequence of `[name, value]` pairs appended to every row as
|
|
108
|
+
* constant columns — used to tag facts the table doesn't state, e.g. the
|
|
109
|
+
* thread system on a tap family.
|
|
110
|
+
*/
|
|
111
|
+
export declare function scrapeFamily(fetcher: Fetcher, code: string, brand?: AemBrandName, tags?: readonly Tag[]): Promise<ScrapeResult>;
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Family page -> rows, for every brand on Kennametal's AEM platform.
|
|
3
|
+
*
|
|
4
|
+
* Family pages render their variant table client-side, but the table comes
|
|
5
|
+
* from a plain AEM GET that returns ALL variants as one HTML table — no
|
|
6
|
+
* pagination, no JS, no bot-blocking. The package runbook records how the
|
|
7
|
+
* endpoint was found and how to read a new brand's component node off a family
|
|
8
|
+
* page.
|
|
9
|
+
*
|
|
10
|
+
* Column identity comes from the `<th>` class attribute (e.g.
|
|
11
|
+
* `"DRL_CUT_D1_MIN metric"`), NOT the visible label — labels like `D1` repeat
|
|
12
|
+
* across unit pairs and non-dimensional columns like wire size.
|
|
13
|
+
*
|
|
14
|
+
* ## The one dependency in this package, and why it is here
|
|
15
|
+
*
|
|
16
|
+
* Node ships no HTML parser, so this module declares `htmlparser2` — and
|
|
17
|
+
* declares it rather than putting it in the core, on the rule that *a
|
|
18
|
+
* transport that needs a parser declares it; nothing shared may pretend to
|
|
19
|
+
* know how a vendor serves a table.* This module is the only importer:
|
|
20
|
+
* REGO-FIX's XML is read by regex and Destiny Tool's Firestore is JSON.
|
|
21
|
+
*
|
|
22
|
+
* `decodeEntities` is on — these tables carry `°` and `Ø` in cell
|
|
23
|
+
* text, and a raw `ø` in a description would reach the CSV as five
|
|
24
|
+
* characters.
|
|
25
|
+
*/
|
|
26
|
+
import { Parser } from 'htmlparser2';
|
|
27
|
+
import { VendorResponseError } from '../../errors.js';
|
|
28
|
+
import { BRANDS } from '../../identity.js';
|
|
29
|
+
export const BASE = 'https://www.{host}/us/en/products/fam/_jcr_content/root/' +
|
|
30
|
+
'responsivegrid/{node}.variants.{code}.html' +
|
|
31
|
+
'?query={query}&uom=metric';
|
|
32
|
+
/**
|
|
33
|
+
* The Hybris/Solr facet string that scopes a request to a family's active
|
|
34
|
+
* variants. The family code in the URL path already scopes to the family, so
|
|
35
|
+
* this only drops the discontinued ones. `materials` appends a second facet to
|
|
36
|
+
* it; nothing else should need to.
|
|
37
|
+
*/
|
|
38
|
+
export const ACTIVE_ONLY = ':relevance:obsoleteFacet:false';
|
|
39
|
+
/**
|
|
40
|
+
* The class the vendor renders instead of a table when a query matches
|
|
41
|
+
* nothing. Distinguishing that from a response we failed to parse is the whole
|
|
42
|
+
* reason {@link parseVariantTable} looks for it.
|
|
43
|
+
*/
|
|
44
|
+
export const NO_RESULTS = 'class="no-results"';
|
|
45
|
+
const SKIP_CLASSES = ['collab-checkbox-header', 'sticky-column', 'marketingFirstChoice'];
|
|
46
|
+
/** Collects rows of `[cellText, attributes]` pairs. */
|
|
47
|
+
export class TableParser {
|
|
48
|
+
rows = [];
|
|
49
|
+
row = null;
|
|
50
|
+
cell = null;
|
|
51
|
+
attrs = {};
|
|
52
|
+
/** Feed a whole document. */
|
|
53
|
+
feed(html) {
|
|
54
|
+
const parser = new Parser({
|
|
55
|
+
onopentag: (tag, attribs) => {
|
|
56
|
+
if (tag === 'tr') {
|
|
57
|
+
this.row = [];
|
|
58
|
+
}
|
|
59
|
+
else if ((tag === 'td' || tag === 'th') && this.row !== null) {
|
|
60
|
+
this.cell = '';
|
|
61
|
+
this.attrs = { ...attribs };
|
|
62
|
+
}
|
|
63
|
+
},
|
|
64
|
+
ontext: (text) => {
|
|
65
|
+
if (this.cell !== null)
|
|
66
|
+
this.cell += text;
|
|
67
|
+
},
|
|
68
|
+
onclosetag: (tag) => {
|
|
69
|
+
if (tag === 'tr' && this.row !== null) {
|
|
70
|
+
this.rows.push(this.row);
|
|
71
|
+
this.row = null;
|
|
72
|
+
}
|
|
73
|
+
else if ((tag === 'td' || tag === 'th') && this.cell !== null) {
|
|
74
|
+
this.row?.push([collapse(this.cell), this.attrs]);
|
|
75
|
+
this.cell = null;
|
|
76
|
+
}
|
|
77
|
+
},
|
|
78
|
+
}, { decodeEntities: true });
|
|
79
|
+
parser.write(html);
|
|
80
|
+
parser.end();
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
/** `" a b \n"` -> `"a b"`, as Python's `' '.join(s.split())` does. */
|
|
84
|
+
function collapse(text) {
|
|
85
|
+
return text.split(/\s+/).filter(Boolean).join(' ');
|
|
86
|
+
}
|
|
87
|
+
/** One family's variants URL. */
|
|
88
|
+
export function variantsUrl(code, brand = 'kennametal', query = ACTIVE_ONLY) {
|
|
89
|
+
const { host, node } = BRANDS[brand];
|
|
90
|
+
return BASE.replace('{host}', host)
|
|
91
|
+
.replace('{node}', node)
|
|
92
|
+
.replace('{code}', code)
|
|
93
|
+
.replace('{query}', encodeURIComponent(query));
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* One family's variants response, as HTML.
|
|
97
|
+
*
|
|
98
|
+
* The one network call in this module, and the seam every test replaces —
|
|
99
|
+
* everything below it parses a string.
|
|
100
|
+
*/
|
|
101
|
+
export async function fetchVariants(fetcher, code, brand = 'kennametal', query = ACTIVE_ONLY) {
|
|
102
|
+
return fetcher.text(variantsUrl(code, brand, query));
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* The header row and the data rows of a variants response.
|
|
106
|
+
*
|
|
107
|
+
* Both callers need the same two subtleties and neither is obvious, which is
|
|
108
|
+
* why this is one function rather than two copies: the header is found by its
|
|
109
|
+
* "Material Number" cell rather than by position (the response opens with
|
|
110
|
+
* filter rows that are also `<tr>`s), and a data row is one exactly as long as
|
|
111
|
+
* the header whose second cell is all digits. Header, filter and footer rows
|
|
112
|
+
* all fail one of those two tests.
|
|
113
|
+
*
|
|
114
|
+
* A **matched-nothing** response has no table in it at all, only the vendor's
|
|
115
|
+
* own no-results notice, and comes back with a null header. That is the
|
|
116
|
+
* ordinary answer to a facet query for a group a family isn't rated for
|
|
117
|
+
* (`materials`), and a hard error for a family scrape — so it is returned as a
|
|
118
|
+
* state rather than thrown, and {@link scrapeFamily} is what decides it's fatal.
|
|
119
|
+
*
|
|
120
|
+
* A response with neither the notice nor a header is a *third* thing: the
|
|
121
|
+
* endpoint changed shape. That throws, because silently reporting zero rows
|
|
122
|
+
* would look exactly like the vendor discontinuing a family.
|
|
123
|
+
*/
|
|
124
|
+
export function parseVariantTable(html) {
|
|
125
|
+
const parser = new TableParser();
|
|
126
|
+
parser.feed(html);
|
|
127
|
+
const header = parser.rows.find((row) => row.some(([text]) => text === 'Material Number')) ?? null;
|
|
128
|
+
if (header === null) {
|
|
129
|
+
if (html.includes(NO_RESULTS))
|
|
130
|
+
return { header: null, rows: [] };
|
|
131
|
+
throw new VendorResponseError('variants response', `has neither a Material Number header nor the vendor's ` +
|
|
132
|
+
`${JSON.stringify(NO_RESULTS)} marker — the endpoint changed shape`);
|
|
133
|
+
}
|
|
134
|
+
const rows = parser.rows.filter((row) => row.length === header.length && /^\d+$/.test(row[1]?.[0] ?? ''));
|
|
135
|
+
return { header, rows };
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Build unique column names from header label + `th` class unit hints.
|
|
139
|
+
*
|
|
140
|
+
* A `null` entry is a column the CSV drops — the checkbox, the sticky CTA, and
|
|
141
|
+
* the marketing flag. The list stays positional so it can be zipped against a
|
|
142
|
+
* data row.
|
|
143
|
+
*/
|
|
144
|
+
export function columnNames(header) {
|
|
145
|
+
const labelCounts = new Map();
|
|
146
|
+
for (const [text] of header) {
|
|
147
|
+
labelCounts.set(text, (labelCounts.get(text) ?? 0) + 1);
|
|
148
|
+
}
|
|
149
|
+
const names = header.map(([text, attrs]) => {
|
|
150
|
+
const cls = attrs['class'] ?? '';
|
|
151
|
+
if (!text || SKIP_CLASSES.some((skip) => cls.includes(skip)))
|
|
152
|
+
return null;
|
|
153
|
+
if (cls.includes('CatNo')) {
|
|
154
|
+
// Catalog number columns carry unit classes but are not dimensions.
|
|
155
|
+
return text;
|
|
156
|
+
}
|
|
157
|
+
if (cls.includes('metric'))
|
|
158
|
+
return `${text}_mm`;
|
|
159
|
+
if (cls.includes('inch'))
|
|
160
|
+
return `${text}_in`;
|
|
161
|
+
if ((labelCounts.get(text) ?? 0) > 1) {
|
|
162
|
+
// A unitless column sharing a label with a unit pair, e.g. a third `D1`
|
|
163
|
+
// with data-value `[D1] Wire Size` -> `D1_wire_size`.
|
|
164
|
+
const title = (attrs['data-value'] ?? '').replace(/^\[[^\]]*\]\s*/, '').trim();
|
|
165
|
+
if (title && title.toLowerCase() !== text.toLowerCase()) {
|
|
166
|
+
const slug = title
|
|
167
|
+
.replace(/[^A-Za-z0-9]+/g, '_')
|
|
168
|
+
.replace(/^_+|_+$/g, '')
|
|
169
|
+
.toLowerCase();
|
|
170
|
+
return `${text}_${slug}`;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return text;
|
|
174
|
+
});
|
|
175
|
+
// Refused rather than allowed to collide: `scrapeFamily` writes `out[name]`,
|
|
176
|
+
// so a repeated name kept the last column's data under a header the CSV
|
|
177
|
+
// still printed twice — the loss the class-based identity above exists to
|
|
178
|
+
// prevent.
|
|
179
|
+
const seen = new Set();
|
|
180
|
+
for (const name of names) {
|
|
181
|
+
if (name === null)
|
|
182
|
+
continue;
|
|
183
|
+
if (seen.has(name)) {
|
|
184
|
+
throw new VendorResponseError('variants response', `two columns are both named ${JSON.stringify(name)} — neither a unit ` +
|
|
185
|
+
`class nor a data-value title tells them apart`);
|
|
186
|
+
}
|
|
187
|
+
seen.add(name);
|
|
188
|
+
}
|
|
189
|
+
return names;
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* Scrape one family into rows.
|
|
193
|
+
*
|
|
194
|
+
* `tags` is a sequence of `[name, value]` pairs appended to every row as
|
|
195
|
+
* constant columns — used to tag facts the table doesn't state, e.g. the
|
|
196
|
+
* thread system on a tap family.
|
|
197
|
+
*/
|
|
198
|
+
export async function scrapeFamily(fetcher, code, brand = 'kennametal', tags = []) {
|
|
199
|
+
const url = variantsUrl(code, brand);
|
|
200
|
+
const { header, rows: dataRows } = parseVariantTable(await fetcher.text(url));
|
|
201
|
+
if (header === null) {
|
|
202
|
+
throw new VendorResponseError(`family ${code}`, 'the vendor returned no variants');
|
|
203
|
+
}
|
|
204
|
+
const names = columnNames(header);
|
|
205
|
+
const kept = names.filter((name) => name !== null);
|
|
206
|
+
const csvHeader = [...kept, ...tags.map(([name]) => name)];
|
|
207
|
+
const rows = dataRows.map((row) => {
|
|
208
|
+
// `dataRows` is filtered to rows exactly as long as the header, and
|
|
209
|
+
// `names` has one entry per header cell, so a length mismatch here means
|
|
210
|
+
// the table changed shape mid-parse. Truncating silently would shift every
|
|
211
|
+
// column after the gap by one.
|
|
212
|
+
if (row.length !== names.length) {
|
|
213
|
+
throw new VendorResponseError(`family ${code}`, `a data row has ${row.length} cells where the header has ` +
|
|
214
|
+
`${names.length} — the table changed shape mid-parse`);
|
|
215
|
+
}
|
|
216
|
+
const out = {};
|
|
217
|
+
names.forEach((name, index) => {
|
|
218
|
+
if (name !== null)
|
|
219
|
+
out[name] = row[index]?.[0] ?? '';
|
|
220
|
+
});
|
|
221
|
+
for (const [name, value] of tags)
|
|
222
|
+
out[name] = value;
|
|
223
|
+
return out;
|
|
224
|
+
});
|
|
225
|
+
return { header: csvHeader, rows, source: url, familyCode: code };
|
|
226
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `Thread Pitch` column Kennametal's tap tables do not publish.
|
|
3
|
+
*
|
|
4
|
+
* The arithmetic is in `thread.ts` — parsing a thread designation is a
|
|
5
|
+
* standard, not a vendor's. What is Kennametal's, and therefore here, is that
|
|
6
|
+
* the designation arrives in a column named `D1-TDZ`, that the thread system
|
|
7
|
+
* arrives in a `Thread System` tag column this package appends at scrape time,
|
|
8
|
+
* and that a pitch column has to be derived at all because the vendor's table
|
|
9
|
+
* carries none.
|
|
10
|
+
*/
|
|
11
|
+
import type { ScrapeResult } from '../../scrape.js';
|
|
12
|
+
/** The vendor's column holding the thread designation — `#2-56`, `M6 X 1`. */
|
|
13
|
+
export declare const DESIGNATION_COLUMN = "D1-TDZ";
|
|
14
|
+
/** The constant column the scraper tags on, because the table states none. */
|
|
15
|
+
export declare const SYSTEM_COLUMN = "Thread System";
|
|
16
|
+
/** The column this module derives. */
|
|
17
|
+
export declare const PITCH_COLUMN = "Thread Pitch";
|
|
18
|
+
/**
|
|
19
|
+
* Add (or replace) the `Thread Pitch` column on a tap scrape.
|
|
20
|
+
*
|
|
21
|
+
* Safe to re-run: an existing pitch column is dropped and rebuilt, and the
|
|
22
|
+
* column is always reinserted directly after `D1-TDZ`, so re-running produces
|
|
23
|
+
* a byte-identical result rather than appending a second copy.
|
|
24
|
+
*
|
|
25
|
+
* Returns a new result rather than rewriting a file in place, so this step
|
|
26
|
+
* composes with the others instead of needing a CSV between them.
|
|
27
|
+
*/
|
|
28
|
+
export declare function addThreadPitch(scrape: ScrapeResult): ScrapeResult;
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `Thread Pitch` column Kennametal's tap tables do not publish.
|
|
3
|
+
*
|
|
4
|
+
* The arithmetic is in `thread.ts` — parsing a thread designation is a
|
|
5
|
+
* standard, not a vendor's. What is Kennametal's, and therefore here, is that
|
|
6
|
+
* the designation arrives in a column named `D1-TDZ`, that the thread system
|
|
7
|
+
* arrives in a `Thread System` tag column this package appends at scrape time,
|
|
8
|
+
* and that a pitch column has to be derived at all because the vendor's table
|
|
9
|
+
* carries none.
|
|
10
|
+
*/
|
|
11
|
+
import { VendorResponseError } from '../../errors.js';
|
|
12
|
+
import { threadPitch } from '../../thread.js';
|
|
13
|
+
/** The vendor's column holding the thread designation — `#2-56`, `M6 X 1`. */
|
|
14
|
+
export const DESIGNATION_COLUMN = 'D1-TDZ';
|
|
15
|
+
/** The constant column the scraper tags on, because the table states none. */
|
|
16
|
+
export const SYSTEM_COLUMN = 'Thread System';
|
|
17
|
+
/** The column this module derives. */
|
|
18
|
+
export const PITCH_COLUMN = 'Thread Pitch';
|
|
19
|
+
/**
|
|
20
|
+
* Add (or replace) the `Thread Pitch` column on a tap scrape.
|
|
21
|
+
*
|
|
22
|
+
* Safe to re-run: an existing pitch column is dropped and rebuilt, and the
|
|
23
|
+
* column is always reinserted directly after `D1-TDZ`, so re-running produces
|
|
24
|
+
* a byte-identical result rather than appending a second copy.
|
|
25
|
+
*
|
|
26
|
+
* Returns a new result rather than rewriting a file in place, so this step
|
|
27
|
+
* composes with the others instead of needing a CSV between them.
|
|
28
|
+
*/
|
|
29
|
+
export function addThreadPitch(scrape) {
|
|
30
|
+
const header = scrape.header.filter((name) => name !== PITCH_COLUMN);
|
|
31
|
+
const at = header.indexOf(DESIGNATION_COLUMN);
|
|
32
|
+
if (at === -1) {
|
|
33
|
+
throw new VendorResponseError(PITCH_COLUMN, `the scrape has no ${DESIGNATION_COLUMN} column to derive a pitch from`);
|
|
34
|
+
}
|
|
35
|
+
header.splice(at + 1, 0, PITCH_COLUMN);
|
|
36
|
+
const rows = scrape.rows.map((row) => ({
|
|
37
|
+
...row,
|
|
38
|
+
[PITCH_COLUMN]: threadPitch(row[DESIGNATION_COLUMN] ?? '', (row[SYSTEM_COLUMN] ?? '')),
|
|
39
|
+
}));
|
|
40
|
+
return { ...scrape, header, rows };
|
|
41
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* REGO-FIX — toolholding only, and nothing here is shared with Kennametal.
|
|
3
|
+
*
|
|
4
|
+
* Different CMS, different transport, and — the part that surprises — the
|
|
5
|
+
* roster and the geometry are two different fetches. See
|
|
6
|
+
* `docs/REGOFIX_PRODUCTFINDER_API.md`.
|
|
7
|
+
*/
|
|
8
|
+
export * from './scrape.js';
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* REGO-FIX — toolholding only, and nothing here is shared with Kennametal.
|
|
3
|
+
*
|
|
4
|
+
* Different CMS, different transport, and — the part that surprises — the
|
|
5
|
+
* roster and the geometry are two different fetches. See
|
|
6
|
+
* `docs/REGOFIX_PRODUCTFINDER_API.md`.
|
|
7
|
+
*/
|
|
8
|
+
export * from './scrape.js';
|