@toolpath/tool-scraper 0.1.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -11
- package/dist/columns.d.ts +62 -0
- package/dist/columns.js +62 -0
- package/dist/conventions.d.ts +89 -15
- package/dist/conventions.js +108 -15
- package/dist/families/emuge.d.ts +183 -0
- package/dist/families/emuge.js +159 -0
- package/dist/families/harvey.d.ts +1782 -0
- package/dist/families/harvey.js +1328 -0
- package/dist/families/index.js +5 -2
- package/dist/families/kennametal.d.ts +21 -0
- package/dist/families/kennametal.js +10 -0
- package/dist/families/maritool.d.ts +120 -0
- package/dist/families/maritool.js +175 -0
- package/dist/family.d.ts +35 -1
- package/dist/family.js +28 -0
- package/dist/identity.d.ts +18 -0
- package/dist/identity.js +46 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +16 -0
- package/dist/measure.d.ts +112 -0
- package/dist/measure.js +130 -0
- package/dist/node/cli.d.ts +3 -0
- package/dist/node/cli.js +152 -2
- package/dist/node/main.js +0 -0
- package/dist/records.d.ts +169 -12
- package/dist/records.js +116 -5
- package/dist/registry.d.ts +31 -1
- package/dist/registry.js +40 -1
- package/dist/scrape.d.ts +15 -0
- package/dist/scrape.js +24 -0
- package/dist/vendors/destinytool/records.d.ts +13 -3
- package/dist/vendors/destinytool/records.js +41 -37
- package/dist/vendors/emuge/index.d.ts +17 -0
- package/dist/vendors/emuge/index.js +17 -0
- package/dist/vendors/emuge/records.d.ts +150 -0
- package/dist/vendors/emuge/records.js +375 -0
- package/dist/vendors/emuge/scrape.d.ts +227 -0
- package/dist/vendors/emuge/scrape.js +358 -0
- package/dist/vendors/emuge/value.d.ts +75 -0
- package/dist/vendors/emuge/value.js +116 -0
- package/dist/vendors/harvey/catalog.d.ts +53 -0
- package/dist/vendors/harvey/catalog.js +120 -0
- package/dist/vendors/harvey/header.d.ts +89 -0
- package/dist/vendors/harvey/header.js +185 -0
- package/dist/vendors/harvey/index.d.ts +21 -0
- package/dist/vendors/harvey/index.js +21 -0
- package/dist/vendors/harvey/lexicon.d.ts +73 -0
- package/dist/vendors/harvey/lexicon.js +126 -0
- package/dist/vendors/harvey/literal.d.ts +68 -0
- package/dist/vendors/harvey/literal.js +214 -0
- package/dist/vendors/harvey/records.d.ts +79 -0
- package/dist/vendors/harvey/records.js +163 -0
- package/dist/vendors/harvey/scrape.d.ts +187 -0
- package/dist/vendors/harvey/scrape.js +483 -0
- package/dist/vendors/harvey/value.d.ts +74 -0
- package/dist/vendors/harvey/value.js +119 -0
- package/dist/vendors/kennametal/records.d.ts +13 -4
- package/dist/vendors/kennametal/records.js +61 -26
- package/dist/vendors/maritool/catalog.d.ts +81 -0
- package/dist/vendors/maritool/catalog.js +132 -0
- package/dist/vendors/maritool/index.d.ts +10 -0
- package/dist/vendors/maritool/index.js +10 -0
- package/dist/vendors/maritool/scrape.d.ts +297 -0
- package/dist/vendors/maritool/scrape.js +593 -0
- package/dist/vendors/regofix/scrape.d.ts +8 -11
- package/dist/vendors/regofix/scrape.js +21 -36
- package/package.json +31 -7
|
@@ -0,0 +1,358 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EMUGE-FRANKEN's SAP Commerce API -> cutting-tool rows.
|
|
3
|
+
*
|
|
4
|
+
* Nothing here parses HTML, because there is none to parse: the storefront is
|
|
5
|
+
* SAP Commerce (Hybris) behind a Vue front end, and a category page
|
|
6
|
+
* server-renders a `<title>`, a base64 CMS blob and an empty
|
|
7
|
+
* `<category-detail-page>` element. The data comes from the JSON API that
|
|
8
|
+
* element's component reads. `docs/EMUGE_FRANKEN_COMMERCE_API.md` records how
|
|
9
|
+
* the endpoint was found, what it answers with, and what was tried first.
|
|
10
|
+
*
|
|
11
|
+
* ```
|
|
12
|
+
* GET {BASE}/search/products?query=<facet query>¤tPage=&pageSize=
|
|
13
|
+
* &searchQueryContext=KLAMMER_GROUPING -- grouped products
|
|
14
|
+
* &searchQueryContext=VARIANT_SEARCH -- one group's orderable parts
|
|
15
|
+
* GET {BASE}/products?productCodes=<up to 30>&fields=FULL
|
|
16
|
+
* ```
|
|
17
|
+
*
|
|
18
|
+
* ## Three calls, because the fields are in three places
|
|
19
|
+
*
|
|
20
|
+
* A *klammer* product is EMUGE's grouping — `H301025`, "Solid Carbide End Mill
|
|
21
|
+
* TOP-Cut VAR" — and its variants are the parts somebody orders. Neither one
|
|
22
|
+
* alone carries a record:
|
|
23
|
+
*
|
|
24
|
+
* - the **grouped listing** states the product line, the category and the
|
|
25
|
+
* version, and no dimensions;
|
|
26
|
+
* - the **variant listing** states the material number, the catalog number and
|
|
27
|
+
* every dimension, under `mainDrawing.technicalDetails`;
|
|
28
|
+
* - the **per-part detail** states what appears in neither — the end mill's
|
|
29
|
+
* flute count, the drill's point angle, the tap's thread symbol and pitch,
|
|
30
|
+
* and `applicationMaterials`, which is the vendor's own ISO 513 index.
|
|
31
|
+
*
|
|
32
|
+
* The third is the reason a scrape is not two calls. It is batched 30 codes at
|
|
33
|
+
* a time, so it costs one request per 30 parts rather than one per part.
|
|
34
|
+
*
|
|
35
|
+
* ## Two buckets of property, and only one of them gets a unit suffix
|
|
36
|
+
*
|
|
37
|
+
* `mainDrawing.technicalDetails` is the dimension table — that is the bucket
|
|
38
|
+
* whose columns carry `_mm`/`_in` from the family's declared unit. Everything
|
|
39
|
+
* in the flat `technicalDetails` keeps its bare label, which is what puts a
|
|
40
|
+
* tap's `pitch [mm]` in a column called `pitch`: `records.DIMENSIONAL_COLUMNS`
|
|
41
|
+
* excludes `TP` from unit pairing, so the core reads that column by its bare
|
|
42
|
+
* label, and a suffix here would name a column nothing scraped.
|
|
43
|
+
*
|
|
44
|
+
* Values are written **verbatim** — `1 1/2 "`, `3 mm`, `140 deg` — the call
|
|
45
|
+
* `vendors/harvey/scrape.ts` makes and for the same reason: the CSV is the
|
|
46
|
+
* receipt, and EMUGE's own fractional inches and stated units are part of what
|
|
47
|
+
* it published. `records.ts` is where a cell becomes a number, through
|
|
48
|
+
* `value.ts`.
|
|
49
|
+
*
|
|
50
|
+
* ## What the vendor gets wrong
|
|
51
|
+
*
|
|
52
|
+
* Warned about, never corrected — two disagreeing vendor cells cannot say which
|
|
53
|
+
* one is wrong, and a scraper that picks becomes a place tool data is authored
|
|
54
|
+
* by hand.
|
|
55
|
+
*
|
|
56
|
+
* - **`number of flutes Z` is `999` on 64 end mill variants** (2026-09-01), a
|
|
57
|
+
* sentinel rather than a count. `records.ts` refuses it; a CSV cell keeps it,
|
|
58
|
+
* because that is what the vendor published.
|
|
59
|
+
* - **`name` comes back German on the US English storefront** — "TOP-Cut VAR
|
|
60
|
+
* HM-Schaftfräser / lang Typ N ALCR" with `lang=en&country=US`. Recorded as
|
|
61
|
+
* sent.
|
|
62
|
+
* - **The same property is spelled two ways across categories**: `coating` on
|
|
63
|
+
* an end mill and `Coating` on a tap, `Cutting material` on both. Two
|
|
64
|
+
* columns, because relabelling one onto the other would be this adapter
|
|
65
|
+
* deciding what the vendor meant.
|
|
66
|
+
* - **The same measurement is published under two unit tags** — the drill
|
|
67
|
+
* detail record carries `nominal diameter d₁ [in]` beside the millimetre one.
|
|
68
|
+
* `value.bareLabel` strips the tag, so both want one column; {@link put} lets
|
|
69
|
+
* the later one win and says so, rather than dropping a number out of the
|
|
70
|
+
* receipt in silence.
|
|
71
|
+
*/
|
|
72
|
+
import { DESCRIPTION_COLUMN, dimensionalColumn } from '../../conventions.js';
|
|
73
|
+
import { VendorResponseError } from '../../errors.js';
|
|
74
|
+
import { consoleWarn, pause, REQUEST_DELAY_MS, unionHeader, } from '../../scrape.js';
|
|
75
|
+
import { bareLabel } from './value.js';
|
|
76
|
+
/**
|
|
77
|
+
* The `emugefrankenUSA` base site, which is the US storefront.
|
|
78
|
+
*
|
|
79
|
+
* Both halves are stated in `window.appConfig` on every page of the site
|
|
80
|
+
* (`apiNodeUrl` and `basesiteId`), and the `/api/v2/{basesiteId}` shape is what
|
|
81
|
+
* the front end's own client builds.
|
|
82
|
+
*/
|
|
83
|
+
export const BASE = 'https://api.emuge-franken-group.com/api/v2/emugefrankenUSA';
|
|
84
|
+
/** Answered in full at 500; the largest group seen holds 214 variants. */
|
|
85
|
+
export const SEARCH_PAGE_SIZE = 500;
|
|
86
|
+
/**
|
|
87
|
+
* Codes per `productCodes=` request.
|
|
88
|
+
*
|
|
89
|
+
* Thirty is what the storefront's own product-comparison call uses, and it is
|
|
90
|
+
* answered whole — 30 codes in, 30 records out. Raising it is a request-volume
|
|
91
|
+
* decision about somebody else's server, so it stays where the vendor put it.
|
|
92
|
+
*/
|
|
93
|
+
export const DETAIL_BATCH = 30;
|
|
94
|
+
/** Grouped products: one row per product line, no dimensions. */
|
|
95
|
+
const KLAMMER_GROUPING = 'KLAMMER_GROUPING';
|
|
96
|
+
/** One group's orderable parts, with their dimension table. */
|
|
97
|
+
const VARIANT_SEARCH = 'VARIANT_SEARCH';
|
|
98
|
+
/** The CSV columns this adapter names, as against the vendor's own labels. */
|
|
99
|
+
export const MATERIAL_NUMBER_COLUMN = 'Material Number';
|
|
100
|
+
export const CATALOG_NUMBER_COLUMN = 'ISO Catalog Number';
|
|
101
|
+
export const GROUP_COLUMN = 'klammerProductCode';
|
|
102
|
+
export const DIMENSION_FEATURE_COLUMN = 'dimensionFeatureValue';
|
|
103
|
+
export const NAME_COLUMN = 'name';
|
|
104
|
+
export const APPLICATION_MATERIALS_COLUMN = 'applicationMaterials';
|
|
105
|
+
/** The facet query for a target's grouped products. */
|
|
106
|
+
export function groupQuery(target) {
|
|
107
|
+
const base = `:relevance:allCategories:${target.category}:klammerProduct:false`;
|
|
108
|
+
return target.facet === undefined ? base : `${base}:${target.facet.code}:${target.facet.value}`;
|
|
109
|
+
}
|
|
110
|
+
/** The facet query for one group's variants. */
|
|
111
|
+
export function variantQuery(klammerCode) {
|
|
112
|
+
return `:relevance:klammerProductCode:${klammerCode}`;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* One `/search/products` URL.
|
|
116
|
+
*
|
|
117
|
+
* `lang` and `country` are the front end's own defaults for this base site and
|
|
118
|
+
* are what make the property labels English; without them the API answers in
|
|
119
|
+
* German.
|
|
120
|
+
*/
|
|
121
|
+
export function searchUrl(query, context, page, sort) {
|
|
122
|
+
const params = new URLSearchParams({
|
|
123
|
+
query,
|
|
124
|
+
currentPage: String(page),
|
|
125
|
+
pageSize: String(SEARCH_PAGE_SIZE),
|
|
126
|
+
searchQueryContext: context,
|
|
127
|
+
lang: 'en',
|
|
128
|
+
country: 'US',
|
|
129
|
+
});
|
|
130
|
+
if (sort !== undefined)
|
|
131
|
+
params.set('sort', sort);
|
|
132
|
+
return `${BASE}/search/products?${params.toString()}`;
|
|
133
|
+
}
|
|
134
|
+
/** The batched per-part detail URL. */
|
|
135
|
+
export function detailUrl(codes) {
|
|
136
|
+
const params = new URLSearchParams({
|
|
137
|
+
productCodes: codes.join(','),
|
|
138
|
+
fields: 'FULL',
|
|
139
|
+
lang: 'en',
|
|
140
|
+
country: 'US',
|
|
141
|
+
});
|
|
142
|
+
return `${BASE}/products?${params.toString()}`;
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Every page of one search, paced.
|
|
146
|
+
*
|
|
147
|
+
* `totalPages` rather than an empty page, because the API states it and a walk
|
|
148
|
+
* that stopped on emptiness would page forever against a response shape that
|
|
149
|
+
* changed.
|
|
150
|
+
*/
|
|
151
|
+
async function pages(fetcher, url, delayMs) {
|
|
152
|
+
const found = [];
|
|
153
|
+
let page = 0;
|
|
154
|
+
for (;;) {
|
|
155
|
+
if (page > 0)
|
|
156
|
+
await pause(delayMs);
|
|
157
|
+
const answer = await fetcher.json(url(page));
|
|
158
|
+
found.push(...(answer.products ?? []));
|
|
159
|
+
const total = answer.pagination?.totalPages ?? 0;
|
|
160
|
+
page += 1;
|
|
161
|
+
if (page >= total)
|
|
162
|
+
return found;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
/** Every grouped product under one target. */
|
|
166
|
+
export async function fetchGroups(fetcher, target, delayMs = REQUEST_DELAY_MS) {
|
|
167
|
+
const query = groupQuery(target);
|
|
168
|
+
return pages(fetcher, (page) => searchUrl(query, KLAMMER_GROUPING, page), delayMs);
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Every orderable part in one group.
|
|
172
|
+
*
|
|
173
|
+
* `sort=prod-detail-variant` is the vendor's own variant order — the sequence
|
|
174
|
+
* its product page lists sizes in — so a re-scrape diffs against the last one
|
|
175
|
+
* rather than against a relevance ranking that moved.
|
|
176
|
+
*
|
|
177
|
+
* **The grouped product comes back inside its own variant listing**, last, and
|
|
178
|
+
* it is not a part: `klammerProduct` is true on it, it carries the base article
|
|
179
|
+
* code with no size suffix (`TA219744` against the variants' `TA219744.0300`),
|
|
180
|
+
* and it has no `mainDrawing.technicalDetails` at all. `KLAMMER_GROUPING`
|
|
181
|
+
* queries filter it out with `:klammerProduct:false` and `VARIANT_SEARCH` ones
|
|
182
|
+
* do not, so the flag is read here instead — the vendor's own discriminator
|
|
183
|
+
* rather than a guess from the missing dimensions.
|
|
184
|
+
*
|
|
185
|
+
* It cost exactly one bogus row per group, which is the kind of thing only a
|
|
186
|
+
* real scrape finds: the first drill run wrote 2,687 rows where the family
|
|
187
|
+
* declares 2,670, and `node/receipts.checkRows` is what said so (JG 2026-09-01).
|
|
188
|
+
*/
|
|
189
|
+
export async function fetchGroupVariants(fetcher, klammerCode, delayMs = REQUEST_DELAY_MS) {
|
|
190
|
+
const query = variantQuery(klammerCode);
|
|
191
|
+
const found = await pages(fetcher, (page) => searchUrl(query, VARIANT_SEARCH, page, 'prod-detail-variant'), delayMs);
|
|
192
|
+
return found.filter((product) => product.klammerProduct !== true);
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* The full record for each of `codes`, by code, in batches of
|
|
196
|
+
* {@link DETAIL_BATCH}.
|
|
197
|
+
*
|
|
198
|
+
* A code the vendor answers nothing for is warned about and left out of the
|
|
199
|
+
* map; the row is still written, without the fields only this call carries. A
|
|
200
|
+
* dropped row would lose a part somebody can order over a field that is
|
|
201
|
+
* missing rather than wrong.
|
|
202
|
+
*/
|
|
203
|
+
export async function fetchDetails(fetcher, codes, delayMs = REQUEST_DELAY_MS, warn = consoleWarn) {
|
|
204
|
+
const found = new Map();
|
|
205
|
+
for (let at = 0; at < codes.length; at += DETAIL_BATCH) {
|
|
206
|
+
if (at > 0)
|
|
207
|
+
await pause(delayMs);
|
|
208
|
+
const batch = codes.slice(at, at + DETAIL_BATCH);
|
|
209
|
+
const answer = await fetcher.json(detailUrl(batch));
|
|
210
|
+
if (!Array.isArray(answer)) {
|
|
211
|
+
throw new VendorResponseError(detailUrl(batch), 'the detail endpoint answered something that is not a list of products');
|
|
212
|
+
}
|
|
213
|
+
for (const product of answer) {
|
|
214
|
+
if (product.code !== undefined)
|
|
215
|
+
found.set(product.code, product);
|
|
216
|
+
}
|
|
217
|
+
const missing = batch.filter((code) => !found.has(code));
|
|
218
|
+
if (missing.length > 0) {
|
|
219
|
+
warn(` WARNING: the detail endpoint published nothing for ${missing.join(', ')} — ` +
|
|
220
|
+
`those rows carry no flute count, point angle, thread pitch or material index`);
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
return found;
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* Write one `{ property, value }` pair into a row under `column`.
|
|
227
|
+
*
|
|
228
|
+
* Two properties can land on one column, and both ways of it are the vendor's
|
|
229
|
+
* doing rather than this adapter's:
|
|
230
|
+
*
|
|
231
|
+
* - **A later empty value never erases a filled one.** The three sources
|
|
232
|
+
* overlap, and an absent value is not a more specific statement than a
|
|
233
|
+
* present one — a part that states no coating does not unstate the group's.
|
|
234
|
+
* Silent, because it is the ordinary case rather than a surprise.
|
|
235
|
+
* - **Two different non-empty values collide, and that is warned about.**
|
|
236
|
+
* {@link bareLabel} strips the unit tag, so `nominal diameter d₁ [mm]` and
|
|
237
|
+
* `nominal diameter d₁ [in]` — which the drill detail record really does
|
|
238
|
+
* publish side by side — are one column name. The later one still wins, so
|
|
239
|
+
* the CSV keeps a reading rather than a blank, but nothing about it is quiet:
|
|
240
|
+
* the receipt is short one number the vendor published, and `unionHeader`
|
|
241
|
+
* cannot show a column that was overwritten rather than missing.
|
|
242
|
+
*/
|
|
243
|
+
function put(row, column, detail, what, warn) {
|
|
244
|
+
if (detail.property === undefined)
|
|
245
|
+
return;
|
|
246
|
+
const value = detail.value ?? '';
|
|
247
|
+
const held = row[column];
|
|
248
|
+
if (held !== undefined && held !== '') {
|
|
249
|
+
if (value === '')
|
|
250
|
+
return;
|
|
251
|
+
if (held !== value) {
|
|
252
|
+
warn(` WARNING: ${what}: ${JSON.stringify(detail.property)} and an earlier ` +
|
|
253
|
+
`property both write ${JSON.stringify(column)} — it held ` +
|
|
254
|
+
`${JSON.stringify(held)} and now states ${JSON.stringify(value)}`);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
row[column] = value;
|
|
258
|
+
}
|
|
259
|
+
/**
|
|
260
|
+
* One orderable part as a row.
|
|
261
|
+
*
|
|
262
|
+
* The three property sources are written in the order they may override each
|
|
263
|
+
* other: the group's, then the part's own, then its dimension table. Group and
|
|
264
|
+
* part overlap — both state the coating and the cutting material — and the
|
|
265
|
+
* part's is the more specific of the two.
|
|
266
|
+
*/
|
|
267
|
+
export function variantRow(group, variant, detail, unit, warn = consoleWarn) {
|
|
268
|
+
const what = variant.code ?? '';
|
|
269
|
+
const row = {
|
|
270
|
+
[MATERIAL_NUMBER_COLUMN]: variant.code ?? '',
|
|
271
|
+
[CATALOG_NUMBER_COLUMN]: variant.articleCode ?? '',
|
|
272
|
+
[DESCRIPTION_COLUMN]: group.productListInfo ?? '',
|
|
273
|
+
[GROUP_COLUMN]: group.code ?? '',
|
|
274
|
+
[DIMENSION_FEATURE_COLUMN]: variant.dimensionFeatureValue ?? '',
|
|
275
|
+
[NAME_COLUMN]: variant.name ?? '',
|
|
276
|
+
};
|
|
277
|
+
for (const property of group.technicalDetails ?? []) {
|
|
278
|
+
put(row, bareLabel(property.property ?? ''), property, what, warn);
|
|
279
|
+
}
|
|
280
|
+
for (const property of detail?.technicalDetails ?? []) {
|
|
281
|
+
put(row, bareLabel(property.property ?? ''), property, what, warn);
|
|
282
|
+
}
|
|
283
|
+
for (const property of variant.mainDrawing?.technicalDetails ?? []) {
|
|
284
|
+
put(row, dimensionalColumn(bareLabel(property.property ?? ''), unit), property, what, warn);
|
|
285
|
+
}
|
|
286
|
+
// The vendor's own order, space-separated like every other multi-value cell
|
|
287
|
+
// here. `records.ts` reorders onto `ISO_MATERIAL_GROUPS` — a consumer that
|
|
288
|
+
// renders a facet from one order and a tool's list from another has no way to
|
|
289
|
+
// notice the two disagree.
|
|
290
|
+
//
|
|
291
|
+
// **Written only where a detail record answered**, so that an empty cell
|
|
292
|
+
// means the vendor's index rates this part for nothing and an absent key
|
|
293
|
+
// means this package has no evidence either way. The second is the part whose
|
|
294
|
+
// detail request answered nothing, which is warned about above.
|
|
295
|
+
if (detail !== undefined) {
|
|
296
|
+
row[APPLICATION_MATERIALS_COLUMN] = (detail.applicationMaterials ?? [])
|
|
297
|
+
.map((m) => m.code ?? '')
|
|
298
|
+
.filter((code) => code !== '')
|
|
299
|
+
.join(' ');
|
|
300
|
+
}
|
|
301
|
+
return row;
|
|
302
|
+
}
|
|
303
|
+
/**
|
|
304
|
+
* One category — optionally one facet of it — as rows, one per orderable part.
|
|
305
|
+
*
|
|
306
|
+
* Paced between every request it makes, so unlike the Harvey command there is
|
|
307
|
+
* nothing for a caller to pace between families.
|
|
308
|
+
*
|
|
309
|
+
* A target that yields no rows is refused rather than returned empty: a facet
|
|
310
|
+
* value the vendor retired answers exactly like a category that was
|
|
311
|
+
* discontinued, and only one of those is a scrape this package should write a
|
|
312
|
+
* receipt for.
|
|
313
|
+
*/
|
|
314
|
+
export async function scrapeCategory(fetcher, target, options) {
|
|
315
|
+
const { unit, warn = consoleWarn, delayMs = REQUEST_DELAY_MS } = options;
|
|
316
|
+
const source = searchUrl(groupQuery(target), KLAMMER_GROUPING, 0);
|
|
317
|
+
const groups = await fetchGroups(fetcher, target, delayMs);
|
|
318
|
+
const rows = [];
|
|
319
|
+
for (const group of groups) {
|
|
320
|
+
if (group.code === undefined || group.code === '') {
|
|
321
|
+
warn(' WARNING: a grouped product carries no code — skipped');
|
|
322
|
+
continue;
|
|
323
|
+
}
|
|
324
|
+
await pause(delayMs);
|
|
325
|
+
const variants = await fetchGroupVariants(fetcher, group.code, delayMs);
|
|
326
|
+
if (variants.length === 0) {
|
|
327
|
+
warn(` WARNING: ${group.code} publishes no variants — skipped`);
|
|
328
|
+
continue;
|
|
329
|
+
}
|
|
330
|
+
// The vendor's own count of the group, against what the walk kept. A second
|
|
331
|
+
// number nothing here computed, which is the whole of its value — it is the
|
|
332
|
+
// per-group form of the argument `family.rows` makes for a whole CSV, and it
|
|
333
|
+
// fires at the group rather than at the end of a scrape that already ran.
|
|
334
|
+
//
|
|
335
|
+
// It is what the grouped product appearing inside its own variant listing
|
|
336
|
+
// looked like before the flag was read: seventeen groups, each one row over.
|
|
337
|
+
const declared = group.numberOfMaterials;
|
|
338
|
+
if (declared !== undefined && declared !== variants.length) {
|
|
339
|
+
warn(` WARNING: ${group.code} kept ${variants.length} parts where the vendor ` +
|
|
340
|
+
`states ${declared} — the group's shape changed, or the walk did`);
|
|
341
|
+
}
|
|
342
|
+
const codes = variants.map((v) => v.code ?? '').filter((code) => code !== '');
|
|
343
|
+
await pause(delayMs);
|
|
344
|
+
const details = await fetchDetails(fetcher, codes, delayMs, warn);
|
|
345
|
+
for (const variant of variants) {
|
|
346
|
+
if (variant.code === undefined || variant.code === '') {
|
|
347
|
+
warn(` WARNING: ${group.code} has a variant with no material number — skipped`);
|
|
348
|
+
continue;
|
|
349
|
+
}
|
|
350
|
+
rows.push(variantRow(group, variant, details.get(variant.code), unit, warn));
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
if (rows.length === 0) {
|
|
354
|
+
throw new VendorResponseError(source, `no rows from ${groups.length} grouped products — the category code, the ` +
|
|
355
|
+
`facet or the response shape changed`);
|
|
356
|
+
}
|
|
357
|
+
return { header: unionHeader(rows), rows, source, familyCode: target.category };
|
|
358
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One EMUGE-FRANKEN property — its label and its value — read.
|
|
3
|
+
*
|
|
4
|
+
* The API answers every measurement as a `{ property, value }` pair of display
|
|
5
|
+
* strings, and both halves need work before they can be a CSV column and a
|
|
6
|
+
* number:
|
|
7
|
+
*
|
|
8
|
+
* ```json
|
|
9
|
+
* { "property": "cutting diameter Ød₁ [inch]", "value": "1 1/2 \"" }
|
|
10
|
+
* { "property": "overall length l₁", "value": "38 mm" }
|
|
11
|
+
* { "property": "point angle", "value": "140 deg" }
|
|
12
|
+
* ```
|
|
13
|
+
*
|
|
14
|
+
* ## The unit is in the value; the label's tag is unreliable
|
|
15
|
+
*
|
|
16
|
+
* `cutting length l₂ [mm]` carries a unit tag and `overall length l₁` and
|
|
17
|
+
* `neck diameter Ød₃` do not, on the same part, in the same response. So a
|
|
18
|
+
* label's tag is stripped for the column name — the CSV states the unit once,
|
|
19
|
+
* in `conventions.UNIT_SUFFIX`'s suffix, and a label that carried `[mm]` as
|
|
20
|
+
* well would state it twice — and the reading of the unit comes from the value,
|
|
21
|
+
* which states one on every measurement seen.
|
|
22
|
+
*
|
|
23
|
+
* A stated unit that disagrees with the family's is converted and warned about
|
|
24
|
+
* rather than refused: the value is right and the column's suffix is right, so
|
|
25
|
+
* dropping the row would lose a part somebody can order. That call is not made
|
|
26
|
+
* here — it is `measure.asLength`'s, which is the one place every vendor makes
|
|
27
|
+
* it. What this module owns is the grammar below and nothing else.
|
|
28
|
+
*
|
|
29
|
+
* ## What is deliberately not read
|
|
30
|
+
*
|
|
31
|
+
* **A range.** `helix angle` is `35-38 deg` and `clamping diameter ØD₁` is
|
|
32
|
+
* `2 - 10 mm`; `measure.fractionValue` would read the first as `73`, because
|
|
33
|
+
* its mixed-number form is `1-1/2` and `35-38` fits it with the denominator
|
|
34
|
+
* absent. {@link NUMBER} admits no hyphenated form at all — EMUGE writes a
|
|
35
|
+
* mixed number space-separated, `1 1/2` — so a range has no reading here, and
|
|
36
|
+
* the columns those two land in are unmapped receipt columns anyway.
|
|
37
|
+
*
|
|
38
|
+
* **A tolerance.** `<=0,003 mm`, `± 0,0008 "`. These carry a German decimal
|
|
39
|
+
* comma where every dimensional value uses a dot, and they lead with a
|
|
40
|
+
* comparator. Guessing at what a comma means in a number this package is about
|
|
41
|
+
* to publish is exactly the kind of quiet wrong answer `measure.ts` refuses to
|
|
42
|
+
* make; they get no reading and stay in the CSV as the vendor's own text.
|
|
43
|
+
*/
|
|
44
|
+
import { type UnitSystem } from '../../conventions.js';
|
|
45
|
+
import { type Measured } from '../../measure.js';
|
|
46
|
+
import { type Warn } from '../../scrape.js';
|
|
47
|
+
/**
|
|
48
|
+
* A property label with its unit tag removed — the CSV's bare label.
|
|
49
|
+
*
|
|
50
|
+
* `'cutting diameter Ød₁ [inch]'` -> `'cutting diameter Ød₁'`. Everything else
|
|
51
|
+
* about the label is left exactly as the vendor wrote it, subscripts and `Ø`
|
|
52
|
+
* included: the CSV is the receipt, and the tag is the one part of it that the
|
|
53
|
+
* column's own `_mm`/`_in` suffix already says.
|
|
54
|
+
*/
|
|
55
|
+
export declare function bareLabel(property: string): string;
|
|
56
|
+
/**
|
|
57
|
+
* One value string -> its number and the unit it states.
|
|
58
|
+
*
|
|
59
|
+
* `''`, a range, a tolerance and any other text come back stating nothing —
|
|
60
|
+
* there is no number there and 0 is not a substitute for one.
|
|
61
|
+
*/
|
|
62
|
+
export declare function parseMeasure(display: string): Measured;
|
|
63
|
+
/**
|
|
64
|
+
* One value as a length in `unit`, or null where it publishes none.
|
|
65
|
+
*
|
|
66
|
+
* `measure.asLength` makes both calls — convert-and-warn a value whose stated
|
|
67
|
+
* unit disagrees with the family's, refuse an angle in a length column —
|
|
68
|
+
* because they are the same two calls for every vendor. They were a verbatim
|
|
69
|
+
* copy of `vendors/harvey/value.ts`'s, warnings included, until 2026-09-01.
|
|
70
|
+
* What is EMUGE's is above: the grammar, and that the unit is read out of the
|
|
71
|
+
* value rather than off the label.
|
|
72
|
+
*/
|
|
73
|
+
export declare function measureIn(display: string, unit: UnitSystem, what: string, warn?: Warn): number | null;
|
|
74
|
+
/** One value as a whole count — a flute number. Null where there is none. */
|
|
75
|
+
export declare function wholeCount(display: string): number | null;
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One EMUGE-FRANKEN property — its label and its value — read.
|
|
3
|
+
*
|
|
4
|
+
* The API answers every measurement as a `{ property, value }` pair of display
|
|
5
|
+
* strings, and both halves need work before they can be a CSV column and a
|
|
6
|
+
* number:
|
|
7
|
+
*
|
|
8
|
+
* ```json
|
|
9
|
+
* { "property": "cutting diameter Ød₁ [inch]", "value": "1 1/2 \"" }
|
|
10
|
+
* { "property": "overall length l₁", "value": "38 mm" }
|
|
11
|
+
* { "property": "point angle", "value": "140 deg" }
|
|
12
|
+
* ```
|
|
13
|
+
*
|
|
14
|
+
* ## The unit is in the value; the label's tag is unreliable
|
|
15
|
+
*
|
|
16
|
+
* `cutting length l₂ [mm]` carries a unit tag and `overall length l₁` and
|
|
17
|
+
* `neck diameter Ød₃` do not, on the same part, in the same response. So a
|
|
18
|
+
* label's tag is stripped for the column name — the CSV states the unit once,
|
|
19
|
+
* in `conventions.UNIT_SUFFIX`'s suffix, and a label that carried `[mm]` as
|
|
20
|
+
* well would state it twice — and the reading of the unit comes from the value,
|
|
21
|
+
* which states one on every measurement seen.
|
|
22
|
+
*
|
|
23
|
+
* A stated unit that disagrees with the family's is converted and warned about
|
|
24
|
+
* rather than refused: the value is right and the column's suffix is right, so
|
|
25
|
+
* dropping the row would lose a part somebody can order. That call is not made
|
|
26
|
+
* here — it is `measure.asLength`'s, which is the one place every vendor makes
|
|
27
|
+
* it. What this module owns is the grammar below and nothing else.
|
|
28
|
+
*
|
|
29
|
+
* ## What is deliberately not read
|
|
30
|
+
*
|
|
31
|
+
* **A range.** `helix angle` is `35-38 deg` and `clamping diameter ØD₁` is
|
|
32
|
+
* `2 - 10 mm`; `measure.fractionValue` would read the first as `73`, because
|
|
33
|
+
* its mixed-number form is `1-1/2` and `35-38` fits it with the denominator
|
|
34
|
+
* absent. {@link NUMBER} admits no hyphenated form at all — EMUGE writes a
|
|
35
|
+
* mixed number space-separated, `1 1/2` — so a range has no reading here, and
|
|
36
|
+
* the columns those two land in are unmapped receipt columns anyway.
|
|
37
|
+
*
|
|
38
|
+
* **A tolerance.** `<=0,003 mm`, `± 0,0008 "`. These carry a German decimal
|
|
39
|
+
* comma where every dimensional value uses a dot, and they lead with a
|
|
40
|
+
* comparator. Guessing at what a comma means in a number this package is about
|
|
41
|
+
* to publish is exactly the kind of quiet wrong answer `measure.ts` refuses to
|
|
42
|
+
* make; they get no reading and stay in the CSV as the vendor's own text.
|
|
43
|
+
*/
|
|
44
|
+
import { asCount, asLength, fractionValue } from '../../measure.js';
|
|
45
|
+
import { consoleWarn } from '../../scrape.js';
|
|
46
|
+
/** A label's trailing unit tag: `[mm]`, `[inch]`, `[in]`. */
|
|
47
|
+
const LABEL_UNIT = /\s*\[(?:mm|in|inch)\]\s*$/i;
|
|
48
|
+
/**
|
|
49
|
+
* A decimal, a simple fraction, or a mixed number written with a **space**,
|
|
50
|
+
* followed by the unit the value states.
|
|
51
|
+
*
|
|
52
|
+
* Ordered longest-first, because `\d*\.?\d+` alone matches the `1` of `1 1/2`
|
|
53
|
+
* and would leave the rest unread — a 1.5 inch tool published as a 1 inch one.
|
|
54
|
+
*
|
|
55
|
+
* No hyphenated mixed number: see the module docstring. `"` is EMUGE's inch
|
|
56
|
+
* mark and `deg` its degree word; both are written with a leading space in
|
|
57
|
+
* every value seen, and the `\s*` tolerates one that is not.
|
|
58
|
+
*/
|
|
59
|
+
const NUMBER = /^(\d+\s+\d+\/\d+|\d*\.?\d+\/\d+|\d*\.?\d+)\s*(mm|"|deg)?$/;
|
|
60
|
+
const NOTHING = { value: null, stated: null };
|
|
61
|
+
const UNITS = {
|
|
62
|
+
mm: 'millimeters',
|
|
63
|
+
'"': 'inches',
|
|
64
|
+
deg: 'degrees',
|
|
65
|
+
};
|
|
66
|
+
/**
|
|
67
|
+
* A property label with its unit tag removed — the CSV's bare label.
|
|
68
|
+
*
|
|
69
|
+
* `'cutting diameter Ød₁ [inch]'` -> `'cutting diameter Ød₁'`. Everything else
|
|
70
|
+
* about the label is left exactly as the vendor wrote it, subscripts and `Ø`
|
|
71
|
+
* included: the CSV is the receipt, and the tag is the one part of it that the
|
|
72
|
+
* column's own `_mm`/`_in` suffix already says.
|
|
73
|
+
*/
|
|
74
|
+
export function bareLabel(property) {
|
|
75
|
+
return property.replace(LABEL_UNIT, '').trim();
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* One value string -> its number and the unit it states.
|
|
79
|
+
*
|
|
80
|
+
* `''`, a range, a tolerance and any other text come back stating nothing —
|
|
81
|
+
* there is no number there and 0 is not a substitute for one.
|
|
82
|
+
*/
|
|
83
|
+
export function parseMeasure(display) {
|
|
84
|
+
const text = display.trim();
|
|
85
|
+
if (text === '')
|
|
86
|
+
return NOTHING;
|
|
87
|
+
const matched = NUMBER.exec(text);
|
|
88
|
+
if (matched === null)
|
|
89
|
+
return NOTHING;
|
|
90
|
+
// A space-separated mixed number is the one shape `fractionValue` does not
|
|
91
|
+
// take: its own form is `1-1/2`. Normalising here rather than widening the
|
|
92
|
+
// core reader keeps the hyphen shape — and the `35-38` range that shape would
|
|
93
|
+
// swallow — out of this vendor entirely.
|
|
94
|
+
const value = fractionValue(matched[1].replace(/\s+/, '-'));
|
|
95
|
+
if (value === null)
|
|
96
|
+
return NOTHING;
|
|
97
|
+
const unit = matched[2];
|
|
98
|
+
return { value, stated: unit === undefined ? null : UNITS[unit] };
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* One value as a length in `unit`, or null where it publishes none.
|
|
102
|
+
*
|
|
103
|
+
* `measure.asLength` makes both calls — convert-and-warn a value whose stated
|
|
104
|
+
* unit disagrees with the family's, refuse an angle in a length column —
|
|
105
|
+
* because they are the same two calls for every vendor. They were a verbatim
|
|
106
|
+
* copy of `vendors/harvey/value.ts`'s, warnings included, until 2026-09-01.
|
|
107
|
+
* What is EMUGE's is above: the grammar, and that the unit is read out of the
|
|
108
|
+
* value rather than off the label.
|
|
109
|
+
*/
|
|
110
|
+
export function measureIn(display, unit, what, warn = consoleWarn) {
|
|
111
|
+
return asLength(parseMeasure(display), display, unit, what, warn);
|
|
112
|
+
}
|
|
113
|
+
/** One value as a whole count — a flute number. Null where there is none. */
|
|
114
|
+
export function wholeCount(display) {
|
|
115
|
+
return asCount(parseMeasure(display));
|
|
116
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The category tree -> the product pages under it.
|
|
3
|
+
*
|
|
4
|
+
* Harvey publishes no sitemap and no flat index: `robots.txt`, `sitemap.xml` and
|
|
5
|
+
* `sitemap_index.xml` all 404, and `/products/all-products` lists the seven top
|
|
6
|
+
* categories and no products. Walking the catalog grid is the only way to find
|
|
7
|
+
* out what exists — see `docs/HARVEY_PRODUCT_TABLE.md` §1.1 and §1.6.
|
|
8
|
+
*
|
|
9
|
+
* Nothing in a scrape needs this: `families/harvey.ts` names all 52 product
|
|
10
|
+
* pages, and a family scrape fetches exactly one. It is here for the maintenance
|
|
11
|
+
* question that table cannot answer — *has Harvey added a page?* — and the CLI
|
|
12
|
+
* exposes it as `harvey --catalog` for that purpose.
|
|
13
|
+
*
|
|
14
|
+
* A page carries subcategory links or product links, never both, so the
|
|
15
|
+
* recursion terminates where products appear. Both selectors were verified
|
|
16
|
+
* against all 33 category pages the four roots reach.
|
|
17
|
+
*/
|
|
18
|
+
import type { Fetcher } from '../../fetch.js';
|
|
19
|
+
import { type Warn } from '../../scrape.js';
|
|
20
|
+
/**
|
|
21
|
+
* The four category trees this adapter covers.
|
|
22
|
+
*
|
|
23
|
+
* Harvey's catalog is far larger; these are the trees asked for, and adding one
|
|
24
|
+
* is a decision rather than a widening of a pattern — see the package's rule on
|
|
25
|
+
* not adding vendor scope without being asked.
|
|
26
|
+
*/
|
|
27
|
+
export declare const CATEGORY_ROOTS: readonly string[];
|
|
28
|
+
/** What one catalog page links to. */
|
|
29
|
+
export interface CategoryLinks {
|
|
30
|
+
subcategories: string[];
|
|
31
|
+
products: string[];
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* The subcategory and product links on one catalog page.
|
|
35
|
+
*
|
|
36
|
+
* Product links are identified by their **wrapper** rather than by the anchor,
|
|
37
|
+
* because the anchor carries no class of its own — the grid renders
|
|
38
|
+
* `<div class="col-md-4 col-12 item-wrapper"><a href="/products/...">`. A rule
|
|
39
|
+
* that took every `/products/` anchor on the page would take the breadcrumb and
|
|
40
|
+
* the footer with it.
|
|
41
|
+
*/
|
|
42
|
+
export declare function parseCategoryPage(html: string): CategoryLinks;
|
|
43
|
+
/**
|
|
44
|
+
* Every product page reachable from `roots`, sorted.
|
|
45
|
+
*
|
|
46
|
+
* Sequential and paced by the package's shared politeness delay. The whole walk
|
|
47
|
+
* is about 33 requests; nothing here needs concurrency, and Cloudflare fronts
|
|
48
|
+
* this site, so raising request volume is the only real risk.
|
|
49
|
+
*/
|
|
50
|
+
export declare function discoverProducts(fetcher: Fetcher, roots?: readonly string[], options?: {
|
|
51
|
+
warn?: Warn;
|
|
52
|
+
delayMs?: number;
|
|
53
|
+
}): Promise<string[]>;
|