@toolpath/tool-scraper 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +98 -0
- package/dist/conventions.d.ts +124 -0
- package/dist/conventions.js +143 -0
- package/dist/errors.d.ts +46 -0
- package/dist/errors.js +53 -0
- package/dist/families/destinytool.d.ts +49 -0
- package/dist/families/destinytool.js +55 -0
- package/dist/families/index.d.ts +59 -0
- package/dist/families/index.js +91 -0
- package/dist/families/kennametal.d.ts +757 -0
- package/dist/families/kennametal.js +660 -0
- package/dist/families/regofix.d.ts +185 -0
- package/dist/families/regofix.js +250 -0
- package/dist/family.d.ts +130 -0
- package/dist/family.js +38 -0
- package/dist/fetch.d.ts +98 -0
- package/dist/fetch.js +116 -0
- package/dist/identity.d.ts +133 -0
- package/dist/identity.js +118 -0
- package/dist/index.d.ts +31 -0
- package/dist/index.js +31 -0
- package/dist/node/cad-mirror.d.ts +55 -0
- package/dist/node/cad-mirror.js +89 -0
- package/dist/node/cli.d.ts +35 -0
- package/dist/node/cli.js +340 -0
- package/dist/node/csv.d.ts +47 -0
- package/dist/node/csv.js +123 -0
- package/dist/node/index.d.ts +16 -0
- package/dist/node/index.js +16 -0
- package/dist/node/main.d.ts +13 -0
- package/dist/node/main.js +14 -0
- package/dist/node/paths.d.ts +60 -0
- package/dist/node/paths.js +80 -0
- package/dist/node/receipts.d.ts +100 -0
- package/dist/node/receipts.js +107 -0
- package/dist/order.d.ts +10 -0
- package/dist/order.js +12 -0
- package/dist/provenance.d.ts +125 -0
- package/dist/provenance.js +133 -0
- package/dist/records.d.ts +305 -0
- package/dist/records.js +297 -0
- package/dist/registry.d.ts +63 -0
- package/dist/registry.js +145 -0
- package/dist/scrape.d.ts +70 -0
- package/dist/scrape.js +37 -0
- package/dist/thread.d.ts +48 -0
- package/dist/thread.js +98 -0
- package/dist/uuid5.d.ts +31 -0
- package/dist/uuid5.js +64 -0
- package/dist/vendors/destinytool/index.d.ts +11 -0
- package/dist/vendors/destinytool/index.js +11 -0
- package/dist/vendors/destinytool/records.d.ts +118 -0
- package/dist/vendors/destinytool/records.js +266 -0
- package/dist/vendors/destinytool/scrape.d.ts +108 -0
- package/dist/vendors/destinytool/scrape.js +192 -0
- package/dist/vendors/kennametal/cad.d.ts +87 -0
- package/dist/vendors/kennametal/cad.js +119 -0
- package/dist/vendors/kennametal/index.d.ts +21 -0
- package/dist/vendors/kennametal/index.js +21 -0
- package/dist/vendors/kennametal/materials.d.ts +143 -0
- package/dist/vendors/kennametal/materials.js +200 -0
- package/dist/vendors/kennametal/records.d.ts +88 -0
- package/dist/vendors/kennametal/records.js +241 -0
- package/dist/vendors/kennametal/scrape.d.ts +111 -0
- package/dist/vendors/kennametal/scrape.js +226 -0
- package/dist/vendors/kennametal/thread-column.d.ts +28 -0
- package/dist/vendors/kennametal/thread-column.js +41 -0
- package/dist/vendors/regofix/index.d.ts +8 -0
- package/dist/vendors/regofix/index.js +8 -0
- package/dist/vendors/regofix/scrape.d.ts +237 -0
- package/dist/vendors/regofix/scrape.js +521 -0
- package/package.json +76 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Toolpath
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# @toolpath/tool-scraper
|
|
2
|
+
|
|
3
|
+
Scrapes cutting-tool and toolholding geometry from vendor catalogs into records.
|
|
4
|
+
|
|
5
|
+
Node ≥20, ESM, one runtime dependency. One vendor-neutral core plus one adapter per manufacturer
|
|
6
|
+
under `src/vendors/`; two adapters share the core and never each other, which
|
|
7
|
+
`tests/vendor-boundary.test.ts` asserts from the package tree rather than from a list.
|
|
8
|
+
|
|
9
|
+
**Scraped output is not committed.** A CSV is the vendor's data and a working file, not source —
|
|
10
|
+
and this repository is public, which is a second reason independent of size.
|
|
11
|
+
|
|
12
|
+
## Install
|
|
13
|
+
|
|
14
|
+
```sh
|
|
15
|
+
pnpm add @toolpath/tool-scraper
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Vendors
|
|
19
|
+
|
|
20
|
+
| Vendor | Transport | What it publishes |
|
|
21
|
+
| ------------------ | ------------------------------------------------ | --------------------- |
|
|
22
|
+
| Kennametal / WIDIA | AEM variant-table GET, parsed with `htmlparser2` | tools and toolholding |
|
|
23
|
+
| REGO-FIX | Elasticsearch proxy POST + per-part DIN 4000 XML | toolholding |
|
|
24
|
+
| Destiny Tool | Firestore REST, paginated | solid end mills |
|
|
25
|
+
|
|
26
|
+
## Two entry points
|
|
27
|
+
|
|
28
|
+
**`@toolpath/tool-scraper` returns records.** Every scrape hands back rows and enough provenance to
|
|
29
|
+
say where they came from; nothing in it touches the filesystem, so a backend can embed it and do
|
|
30
|
+
what it likes with the result.
|
|
31
|
+
|
|
32
|
+
```ts
|
|
33
|
+
import { createFetcher } from '@toolpath/tool-scraper'
|
|
34
|
+
import { scrapeFamily } from '@toolpath/tool-scraper/vendors/kennametal'
|
|
35
|
+
|
|
36
|
+
const fetcher = createFetcher() // or your own: retries, proxy, rate limits
|
|
37
|
+
const { header, rows, source } = await scrapeFamily(fetcher, '100003658')
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
The transport is a parameter, not a module global. Supply your own `Fetcher` and the vendor
|
|
41
|
+
adapters read through it — which is also how every test in this package runs without a network.
|
|
42
|
+
|
|
43
|
+
**`@toolpath/tool-scraper/node` writes files.** CSV serialization, the provenance sidecar, the
|
|
44
|
+
scrape-root resolution and the bulk CAD mirror all need `fs`, so they are a separate entry point and
|
|
45
|
+
a consumer that only wants records never imports them.
|
|
46
|
+
|
|
47
|
+
## Command line
|
|
48
|
+
|
|
49
|
+
Every command prints the resolved scrape root before it does anything, and writes a receipt beside
|
|
50
|
+
what it produces — the source URL, the family code, a timestamp, the row count and the scraper
|
|
51
|
+
version.
|
|
52
|
+
|
|
53
|
+
```sh
|
|
54
|
+
export TOOLPATH_SCRAPE_ROOT=~/toolpath-scrapes # default: ./scrape-out, gitignored
|
|
55
|
+
|
|
56
|
+
toolpath-scrape kennametal 100003658 "$TOOLPATH_SCRAPE_ROOT/kennametal/csv/godrill_3xd_metric.csv"
|
|
57
|
+
toolpath-scrape materials godrill_3xd_metric.csv
|
|
58
|
+
toolpath-scrape regofix holders "$TOOLPATH_SCRAPE_ROOT/regofix/csv/regofix_bt30_pg_holders.csv"
|
|
59
|
+
toolpath-scrape destinytool "$TOOLPATH_SCRAPE_ROOT/destinytool/csv/destinytool_end_mills_inch.csv"
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
`toolpath-scrape --help` lists the rest.
|
|
63
|
+
|
|
64
|
+
## The record
|
|
65
|
+
|
|
66
|
+
Geometry lands in **ISO 13399** codes — `DC`, `OAL`, `LCF`, `RE`, `NOF`, `SIG`, `TP` — the
|
|
67
|
+
machine-tool industry's own interchange dictionary. CAM vendors implement subsets of it, which is
|
|
68
|
+
why these names also appear in Fusion's tool JSON. `records.GEOMETRY_FIELDS` carries each code's
|
|
69
|
+
definition and names the three that are Autodesk's rather than the standard's.
|
|
70
|
+
|
|
71
|
+
Vendor CSVs keep the **vendor's** own column labels. Nothing reads a vendor's CSV but that vendor's
|
|
72
|
+
adapter, and `conventions.ts` holds the short list of rules that do hold across all of them.
|
|
73
|
+
|
|
74
|
+
Every per-family constant no vendor table states carries its provenance — whether it was
|
|
75
|
+
vendor-stated, derived or assumed, and by whom on what date. The types enforce it: an assumed fact
|
|
76
|
+
without a note, a date and initials does not compile.
|
|
77
|
+
|
|
78
|
+
## Documentation
|
|
79
|
+
|
|
80
|
+
- [`docs/ADDING-A-VENDOR.md`](docs/ADDING-A-VENDOR.md) — the runbook.
|
|
81
|
+
- [`docs/KENNAMETAL_CAD_API.md`](docs/KENNAMETAL_CAD_API.md),
|
|
82
|
+
[`docs/KENNAMETAL_SPEEDFEED_API.md`](docs/KENNAMETAL_SPEEDFEED_API.md),
|
|
83
|
+
[`docs/REGOFIX_PRODUCTFINDER_API.md`](docs/REGOFIX_PRODUCTFINDER_API.md) — how each endpoint was
|
|
84
|
+
found, and the dead ends tried first.
|
|
85
|
+
- [`../../docs/TOOL-SCRAPER-PLAN.md`](../../docs/TOOL-SCRAPER-PLAN.md) — the structure, the evidence
|
|
86
|
+
behind it, and what has landed.
|
|
87
|
+
|
|
88
|
+
## Tests
|
|
89
|
+
|
|
90
|
+
```sh
|
|
91
|
+
pnpm --filter @toolpath/tool-scraper test
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Nothing in the suite reaches a vendor: `tests/setup.ts` replaces the global `fetch` with one that
|
|
95
|
+
throws, so a test that forgets its stub fails loudly instead of quietly paging a vendor's catalog.
|
|
96
|
+
|
|
97
|
+
Tests that read a scraped CSV skip with a named reason where no scrape exists; set
|
|
98
|
+
`TOOLPATH_REQUIRE_CORPUS=1` on a machine that keeps one to turn those skips into failures.
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What every vendor's CSV agrees on, written down so it can be checked.
|
|
3
|
+
*
|
|
4
|
+
* A scraped CSV keeps **that vendor's own column labels**. Relabelling them
|
|
5
|
+
* into a shared vocabulary on the way into the file would put a lie in the
|
|
6
|
+
* file whose whole job is to record what the vendor published, and the labels
|
|
7
|
+
* really do not line up: Kennametal's `D1_mm` and Destiny Tool's `cutDia_in`
|
|
8
|
+
* are the same measurement under names neither vendor would recognise in the
|
|
9
|
+
* other's table. Nothing reads a vendor's CSV but that vendor's own adapter.
|
|
10
|
+
*
|
|
11
|
+
* So this is not a schema. It is the short list of conventions that hold
|
|
12
|
+
* *across* the CSVs anyway, and the reason to make them explicit is that
|
|
13
|
+
* vendor #3 already drifted from one:
|
|
14
|
+
*
|
|
15
|
+
* | Convention | Held by |
|
|
16
|
+
* | ------------------------------------------------- | ------------------------- |
|
|
17
|
+
* | `_mm`/`_in` carries the unit on a dimension | all three |
|
|
18
|
+
* | Multi-value cells are space-separated | all three |
|
|
19
|
+
* | One row per orderable part | all three |
|
|
20
|
+
* | `CAD_STEP_URL` names a CAD model where one exists | Kennametal, REGO-FIX |
|
|
21
|
+
* | Unmapped vendor codes keep a `DIN_` prefix | REGO-FIX; rule is general |
|
|
22
|
+
* | The identity columns | **broken** — see below |
|
|
23
|
+
*
|
|
24
|
+
* Identity and units are the two worth enforcing; the rest are advisory, and
|
|
25
|
+
* are here so that "advisory" is a decision on the page rather than an
|
|
26
|
+
* omission.
|
|
27
|
+
*
|
|
28
|
+
* **A vendor's column label can collide with a real ISO 13399 code and mean
|
|
29
|
+
* something else.** The standard defines `D1` as fixing hole diameter and `L`
|
|
30
|
+
* as cutting edge length; Kennametal's tables use `D1` for the cutting
|
|
31
|
+
* diameter and `L` for the overall length. The two vocabularies overlap on
|
|
32
|
+
* `D1`, `L`, `B`, `H`, `RE`, `SIG` and `TP` among others, so anything that
|
|
33
|
+
* reads a vendor CSV without going through that vendor's adapter can be
|
|
34
|
+
* confidently wrong rather than obviously broken. See `records.GEOMETRY_FIELDS`
|
|
35
|
+
* for the canonical side of that line.
|
|
36
|
+
*/
|
|
37
|
+
import type { BrandName } from './identity.js';
|
|
38
|
+
/** Which unit system a family's dimensional columns are published in. */
|
|
39
|
+
export type UnitSystem = 'millimeters' | 'inches';
|
|
40
|
+
/**
|
|
41
|
+
* The suffix a dimensional column carries, per unit system.
|
|
42
|
+
*
|
|
43
|
+
* **This is the whole of the unit rule, in one place, on purpose.** A vendor
|
|
44
|
+
* adapter declares a bare label — `'D1'`, never `'D1_mm'` — and the core
|
|
45
|
+
* appends the suffix from the family's declared unit. Choosing the suffix
|
|
46
|
+
* inside an adapter is exactly the mistake a declared `unit` exists to
|
|
47
|
+
* prevent, and a silent unit assumption is the likeliest way this data shows
|
|
48
|
+
* someone a wrong number: a family tagged metric that publishes both columns
|
|
49
|
+
* converts cleanly and reports 9.525 mm where a machinist ordered 3/8.
|
|
50
|
+
*/
|
|
51
|
+
export declare const UNIT_SUFFIX: Record<UnitSystem, string>;
|
|
52
|
+
/**
|
|
53
|
+
* The CSV column holding a part's downloadable STEP model, for every vendor.
|
|
54
|
+
*
|
|
55
|
+
* Named for what it holds rather than for how one vendor names the format. It
|
|
56
|
+
* was `CAD_STP_LWM` until 2026-08-08: `LWM` is CDS Visual's key for
|
|
57
|
+
* Kennametal's lightweight model, and REGO-FIX publishes no such thing, so the
|
|
58
|
+
* moment a second vendor wrote into this column the name became a claim about
|
|
59
|
+
* the data that was false. That fact is still recorded in the Kennametal CAD
|
|
60
|
+
* module, where it is true.
|
|
61
|
+
*
|
|
62
|
+
* It sits here rather than with either vendor because two of them write it and
|
|
63
|
+
* neither owns it — the leak the vendor-boundary test exists to catch.
|
|
64
|
+
*/
|
|
65
|
+
export declare const CAD_COLUMN = "CAD_STEP_URL";
|
|
66
|
+
/**
|
|
67
|
+
* The prefix an unmapped vendor code keeps, so it cannot read as a dimension.
|
|
68
|
+
*
|
|
69
|
+
* REGO-FIX's per-part DIN 4000 XML publishes codes — `A2`, `B1`, `B2` — whose
|
|
70
|
+
* meaning is not stated anywhere this package has been able to check. The
|
|
71
|
+
* standing rule is to leave such a code unlabelled rather than guess at what
|
|
72
|
+
* it measures, and a bare `A2` beside `L1_mm` reads as a labelled dimension.
|
|
73
|
+
* `DIN_A2` reads as what it is: a vendor code, pending a source.
|
|
74
|
+
*/
|
|
75
|
+
export declare const DIN_PREFIX = "DIN_";
|
|
76
|
+
/**
|
|
77
|
+
* The columns that name one orderable part: its vendor-local number, and the
|
|
78
|
+
* catalog designation a human orders by.
|
|
79
|
+
*
|
|
80
|
+
* `Material Number` is the guid seed — see `identity.recordGuid` — so this is
|
|
81
|
+
* the one convention whose erosion is not cosmetic.
|
|
82
|
+
*/
|
|
83
|
+
export declare const IDENTITY_COLUMNS: readonly ["Material Number", "ISO Catalog Number"];
|
|
84
|
+
/**
|
|
85
|
+
* Where a vendor's CSV does not use {@link IDENTITY_COLUMNS}, and what it uses
|
|
86
|
+
* instead.
|
|
87
|
+
*
|
|
88
|
+
* **One entry, and it is a record of drift rather than a licence.** REGO-FIX
|
|
89
|
+
* adopted Kennametal's identity labels; Destiny Tool passes Firestore's own
|
|
90
|
+
* `itemNumber` straight through and publishes no catalog designation at all —
|
|
91
|
+
* the convention was real but informal, and it eroded the first time a vendor
|
|
92
|
+
* did not resemble the first two. Writing the deviation down is what makes the
|
|
93
|
+
* fourth vendor's drift a decision somebody made rather than a thing that
|
|
94
|
+
* happened.
|
|
95
|
+
*/
|
|
96
|
+
export declare const IDENTITY_DEVIATIONS: Partial<Record<BrandName, readonly string[]>>;
|
|
97
|
+
/**
|
|
98
|
+
* A vendor's bare column label, suffixed for `unit`.
|
|
99
|
+
*
|
|
100
|
+
* Throws rather than defaulting on an unknown unit system: the two callers
|
|
101
|
+
* that could pass one are a family config and a CLI flag, and a typo that
|
|
102
|
+
* silently picked millimetres would produce a clean conversion with the wrong
|
|
103
|
+
* numbers in it. `unit` is typed as a plain string for the CLI's sake — the
|
|
104
|
+
* flag arrives as one, and the check is the only thing between it and a
|
|
105
|
+
* mis-suffixed column.
|
|
106
|
+
*/
|
|
107
|
+
export declare function dimensionalColumn(label: string, unit: string): string;
|
|
108
|
+
/**
|
|
109
|
+
* The columns that identify a part in `brand`'s CSV.
|
|
110
|
+
*
|
|
111
|
+
* {@link IDENTITY_COLUMNS} unless the brand is listed as a deviation, which is
|
|
112
|
+
* a lookup rather than a guess: a caller cannot resolve this by inspecting a
|
|
113
|
+
* header, because a header that is missing `Material Number` is
|
|
114
|
+
* indistinguishable from a scrape that lost it.
|
|
115
|
+
*/
|
|
116
|
+
export declare function identityColumns(brand: BrandName): readonly string[];
|
|
117
|
+
/**
|
|
118
|
+
* Every identity column `brand` claims is really in the CSV header.
|
|
119
|
+
*
|
|
120
|
+
* The failure this prevents: a re-scrape whose part-number column moved or was
|
|
121
|
+
* renamed produces a CSV that still parses, still has the right number of
|
|
122
|
+
* rows, and mints every guid off an empty string.
|
|
123
|
+
*/
|
|
124
|
+
export declare function checkIdentityColumns(brand: BrandName, header: Iterable<string>): void;
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What every vendor's CSV agrees on, written down so it can be checked.
|
|
3
|
+
*
|
|
4
|
+
* A scraped CSV keeps **that vendor's own column labels**. Relabelling them
|
|
5
|
+
* into a shared vocabulary on the way into the file would put a lie in the
|
|
6
|
+
* file whose whole job is to record what the vendor published, and the labels
|
|
7
|
+
* really do not line up: Kennametal's `D1_mm` and Destiny Tool's `cutDia_in`
|
|
8
|
+
* are the same measurement under names neither vendor would recognise in the
|
|
9
|
+
* other's table. Nothing reads a vendor's CSV but that vendor's own adapter.
|
|
10
|
+
*
|
|
11
|
+
* So this is not a schema. It is the short list of conventions that hold
|
|
12
|
+
* *across* the CSVs anyway, and the reason to make them explicit is that
|
|
13
|
+
* vendor #3 already drifted from one:
|
|
14
|
+
*
|
|
15
|
+
* | Convention | Held by |
|
|
16
|
+
* | ------------------------------------------------- | ------------------------- |
|
|
17
|
+
* | `_mm`/`_in` carries the unit on a dimension | all three |
|
|
18
|
+
* | Multi-value cells are space-separated | all three |
|
|
19
|
+
* | One row per orderable part | all three |
|
|
20
|
+
* | `CAD_STEP_URL` names a CAD model where one exists | Kennametal, REGO-FIX |
|
|
21
|
+
* | Unmapped vendor codes keep a `DIN_` prefix | REGO-FIX; rule is general |
|
|
22
|
+
* | The identity columns | **broken** — see below |
|
|
23
|
+
*
|
|
24
|
+
* Identity and units are the two worth enforcing; the rest are advisory, and
|
|
25
|
+
* are here so that "advisory" is a decision on the page rather than an
|
|
26
|
+
* omission.
|
|
27
|
+
*
|
|
28
|
+
* **A vendor's column label can collide with a real ISO 13399 code and mean
|
|
29
|
+
* something else.** The standard defines `D1` as fixing hole diameter and `L`
|
|
30
|
+
* as cutting edge length; Kennametal's tables use `D1` for the cutting
|
|
31
|
+
* diameter and `L` for the overall length. The two vocabularies overlap on
|
|
32
|
+
* `D1`, `L`, `B`, `H`, `RE`, `SIG` and `TP` among others, so anything that
|
|
33
|
+
* reads a vendor CSV without going through that vendor's adapter can be
|
|
34
|
+
* confidently wrong rather than obviously broken. See `records.GEOMETRY_FIELDS`
|
|
35
|
+
* for the canonical side of that line.
|
|
36
|
+
*/
|
|
37
|
+
import { ScraperConfigError } from './errors.js';
|
|
38
|
+
/**
|
|
39
|
+
* The suffix a dimensional column carries, per unit system.
|
|
40
|
+
*
|
|
41
|
+
* **This is the whole of the unit rule, in one place, on purpose.** A vendor
|
|
42
|
+
* adapter declares a bare label — `'D1'`, never `'D1_mm'` — and the core
|
|
43
|
+
* appends the suffix from the family's declared unit. Choosing the suffix
|
|
44
|
+
* inside an adapter is exactly the mistake a declared `unit` exists to
|
|
45
|
+
* prevent, and a silent unit assumption is the likeliest way this data shows
|
|
46
|
+
* someone a wrong number: a family tagged metric that publishes both columns
|
|
47
|
+
* converts cleanly and reports 9.525 mm where a machinist ordered 3/8.
|
|
48
|
+
*/
|
|
49
|
+
export const UNIT_SUFFIX = {
|
|
50
|
+
millimeters: '_mm',
|
|
51
|
+
inches: '_in',
|
|
52
|
+
};
|
|
53
|
+
/**
|
|
54
|
+
* The CSV column holding a part's downloadable STEP model, for every vendor.
|
|
55
|
+
*
|
|
56
|
+
* Named for what it holds rather than for how one vendor names the format. It
|
|
57
|
+
* was `CAD_STP_LWM` until 2026-08-08: `LWM` is CDS Visual's key for
|
|
58
|
+
* Kennametal's lightweight model, and REGO-FIX publishes no such thing, so the
|
|
59
|
+
* moment a second vendor wrote into this column the name became a claim about
|
|
60
|
+
* the data that was false. That fact is still recorded in the Kennametal CAD
|
|
61
|
+
* module, where it is true.
|
|
62
|
+
*
|
|
63
|
+
* It sits here rather than with either vendor because two of them write it and
|
|
64
|
+
* neither owns it — the leak the vendor-boundary test exists to catch.
|
|
65
|
+
*/
|
|
66
|
+
export const CAD_COLUMN = 'CAD_STEP_URL';
|
|
67
|
+
/**
|
|
68
|
+
* The prefix an unmapped vendor code keeps, so it cannot read as a dimension.
|
|
69
|
+
*
|
|
70
|
+
* REGO-FIX's per-part DIN 4000 XML publishes codes — `A2`, `B1`, `B2` — whose
|
|
71
|
+
* meaning is not stated anywhere this package has been able to check. The
|
|
72
|
+
* standing rule is to leave such a code unlabelled rather than guess at what
|
|
73
|
+
* it measures, and a bare `A2` beside `L1_mm` reads as a labelled dimension.
|
|
74
|
+
* `DIN_A2` reads as what it is: a vendor code, pending a source.
|
|
75
|
+
*/
|
|
76
|
+
export const DIN_PREFIX = 'DIN_';
|
|
77
|
+
/**
|
|
78
|
+
* The columns that name one orderable part: its vendor-local number, and the
|
|
79
|
+
* catalog designation a human orders by.
|
|
80
|
+
*
|
|
81
|
+
* `Material Number` is the guid seed — see `identity.recordGuid` — so this is
|
|
82
|
+
* the one convention whose erosion is not cosmetic.
|
|
83
|
+
*/
|
|
84
|
+
export const IDENTITY_COLUMNS = ['Material Number', 'ISO Catalog Number'];
|
|
85
|
+
/**
|
|
86
|
+
* Where a vendor's CSV does not use {@link IDENTITY_COLUMNS}, and what it uses
|
|
87
|
+
* instead.
|
|
88
|
+
*
|
|
89
|
+
* **One entry, and it is a record of drift rather than a licence.** REGO-FIX
|
|
90
|
+
* adopted Kennametal's identity labels; Destiny Tool passes Firestore's own
|
|
91
|
+
* `itemNumber` straight through and publishes no catalog designation at all —
|
|
92
|
+
* the convention was real but informal, and it eroded the first time a vendor
|
|
93
|
+
* did not resemble the first two. Writing the deviation down is what makes the
|
|
94
|
+
* fourth vendor's drift a decision somebody made rather than a thing that
|
|
95
|
+
* happened.
|
|
96
|
+
*/
|
|
97
|
+
export const IDENTITY_DEVIATIONS = {
|
|
98
|
+
destinytool: ['itemNumber'],
|
|
99
|
+
};
|
|
100
|
+
/**
|
|
101
|
+
* A vendor's bare column label, suffixed for `unit`.
|
|
102
|
+
*
|
|
103
|
+
* Throws rather than defaulting on an unknown unit system: the two callers
|
|
104
|
+
* that could pass one are a family config and a CLI flag, and a typo that
|
|
105
|
+
* silently picked millimetres would produce a clean conversion with the wrong
|
|
106
|
+
* numbers in it. `unit` is typed as a plain string for the CLI's sake — the
|
|
107
|
+
* flag arrives as one, and the check is the only thing between it and a
|
|
108
|
+
* mis-suffixed column.
|
|
109
|
+
*/
|
|
110
|
+
export function dimensionalColumn(label, unit) {
|
|
111
|
+
if (!Object.hasOwn(UNIT_SUFFIX, unit)) {
|
|
112
|
+
throw new ScraperConfigError(label, `unknown unit system ${JSON.stringify(unit)} ` +
|
|
113
|
+
`(known: ${Object.keys(UNIT_SUFFIX).sort().join(', ')})`);
|
|
114
|
+
}
|
|
115
|
+
return label + UNIT_SUFFIX[unit];
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* The columns that identify a part in `brand`'s CSV.
|
|
119
|
+
*
|
|
120
|
+
* {@link IDENTITY_COLUMNS} unless the brand is listed as a deviation, which is
|
|
121
|
+
* a lookup rather than a guess: a caller cannot resolve this by inspecting a
|
|
122
|
+
* header, because a header that is missing `Material Number` is
|
|
123
|
+
* indistinguishable from a scrape that lost it.
|
|
124
|
+
*/
|
|
125
|
+
export function identityColumns(brand) {
|
|
126
|
+
return IDENTITY_DEVIATIONS[brand] ?? IDENTITY_COLUMNS;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Every identity column `brand` claims is really in the CSV header.
|
|
130
|
+
*
|
|
131
|
+
* The failure this prevents: a re-scrape whose part-number column moved or was
|
|
132
|
+
* renamed produces a CSV that still parses, still has the right number of
|
|
133
|
+
* rows, and mints every guid off an empty string.
|
|
134
|
+
*/
|
|
135
|
+
export function checkIdentityColumns(brand, header) {
|
|
136
|
+
const present = new Set(header);
|
|
137
|
+
const missing = identityColumns(brand)
|
|
138
|
+
.filter((column) => !present.has(column))
|
|
139
|
+
.sort();
|
|
140
|
+
if (missing.length > 0) {
|
|
141
|
+
throw new ScraperConfigError(brand, `identity column(s) absent from the CSV: ${missing.join(', ')}`);
|
|
142
|
+
}
|
|
143
|
+
}
|
package/dist/errors.d.ts
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The two ways this package refuses, split by who has to fix it.
|
|
3
|
+
*
|
|
4
|
+
* - {@link ScraperConfigError} — **this package's own catalog is wrong.** A
|
|
5
|
+
* family maps a canonical field that does not exist, a fact is missing the
|
|
6
|
+
* note its source kind requires, two vendors claim one CSV name. Nothing a
|
|
7
|
+
* retry or a different vendor response can fix; it is a bug here, and it is
|
|
8
|
+
* raised before any network call.
|
|
9
|
+
* - {@link VendorResponseError} — **the vendor sent something this package
|
|
10
|
+
* cannot read.** The variants endpoint changed shape, a scrape's row count
|
|
11
|
+
* disagrees with the declared one, a DIN 4000 code that should be pinned is
|
|
12
|
+
* absent. The catalog is fine; the world moved.
|
|
13
|
+
*
|
|
14
|
+
* They throw rather than exiting the process: a Node backend imports this, and
|
|
15
|
+
* a mapped column that moved must not take down somebody's request handler.
|
|
16
|
+
* Both carry a `subject` — the family, brand or part the failure is about — so
|
|
17
|
+
* a caller that catches one does not parse it back out of a sentence.
|
|
18
|
+
*/
|
|
19
|
+
/** Common shape: a message that already names its subject, plus the subject. */
|
|
20
|
+
declare abstract class ScraperError extends Error {
|
|
21
|
+
/** The family, brand or part number this is about, and `message` leads with it. */
|
|
22
|
+
readonly subject: string;
|
|
23
|
+
constructor(subject: string, message: string);
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* This package's own catalog or column map is wrong.
|
|
27
|
+
*
|
|
28
|
+
* Thrown at config-validation time, which is before a scrape reaches the
|
|
29
|
+
* network — the whole point of validating a family's map up front is that a
|
|
30
|
+
* typo fails naming the family instead of as a missing-key fault on row 1 of a
|
|
31
|
+
* scrape that already ran.
|
|
32
|
+
*/
|
|
33
|
+
export declare class ScraperConfigError extends ScraperError {
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* A vendor's response was not the shape this package requires.
|
|
37
|
+
*
|
|
38
|
+
* Distinct from "the vendor returned nothing", which is often an ordinary
|
|
39
|
+
* answer — a facet query for a group a family is not rated for matches no
|
|
40
|
+
* rows, and that is data, not a fault. This is for the case where the reply
|
|
41
|
+
* cannot be read at all, because reporting zero rows there would look exactly
|
|
42
|
+
* like a family being discontinued.
|
|
43
|
+
*/
|
|
44
|
+
export declare class VendorResponseError extends ScraperError {
|
|
45
|
+
}
|
|
46
|
+
export {};
|
package/dist/errors.js
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The two ways this package refuses, split by who has to fix it.
|
|
3
|
+
*
|
|
4
|
+
* - {@link ScraperConfigError} — **this package's own catalog is wrong.** A
|
|
5
|
+
* family maps a canonical field that does not exist, a fact is missing the
|
|
6
|
+
* note its source kind requires, two vendors claim one CSV name. Nothing a
|
|
7
|
+
* retry or a different vendor response can fix; it is a bug here, and it is
|
|
8
|
+
* raised before any network call.
|
|
9
|
+
* - {@link VendorResponseError} — **the vendor sent something this package
|
|
10
|
+
* cannot read.** The variants endpoint changed shape, a scrape's row count
|
|
11
|
+
* disagrees with the declared one, a DIN 4000 code that should be pinned is
|
|
12
|
+
* absent. The catalog is fine; the world moved.
|
|
13
|
+
*
|
|
14
|
+
* They throw rather than exiting the process: a Node backend imports this, and
|
|
15
|
+
* a mapped column that moved must not take down somebody's request handler.
|
|
16
|
+
* Both carry a `subject` — the family, brand or part the failure is about — so
|
|
17
|
+
* a caller that catches one does not parse it back out of a sentence.
|
|
18
|
+
*/
|
|
19
|
+
/** Common shape: a message that already names its subject, plus the subject. */
|
|
20
|
+
class ScraperError extends Error {
|
|
21
|
+
/** The family, brand or part number this is about, and `message` leads with it. */
|
|
22
|
+
subject;
|
|
23
|
+
constructor(subject, message) {
|
|
24
|
+
super(`${subject}: ${message}`);
|
|
25
|
+
this.subject = subject;
|
|
26
|
+
// `Error` is a built-in, so the prototype has to be restored by hand for
|
|
27
|
+
// `instanceof` to survive the ES2022 downlevel. Cheap, and the alternative
|
|
28
|
+
// is a catch block that silently never matches.
|
|
29
|
+
Object.setPrototypeOf(this, new.target.prototype);
|
|
30
|
+
this.name = new.target.name;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* This package's own catalog or column map is wrong.
|
|
35
|
+
*
|
|
36
|
+
* Thrown at config-validation time, which is before a scrape reaches the
|
|
37
|
+
* network — the whole point of validating a family's map up front is that a
|
|
38
|
+
* typo fails naming the family instead of as a missing-key fault on row 1 of a
|
|
39
|
+
* scrape that already ran.
|
|
40
|
+
*/
|
|
41
|
+
export class ScraperConfigError extends ScraperError {
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* A vendor's response was not the shape this package requires.
|
|
45
|
+
*
|
|
46
|
+
* Distinct from "the vendor returned nothing", which is often an ordinary
|
|
47
|
+
* answer — a facet query for a group a family is not rated for matches no
|
|
48
|
+
* rows, and that is data, not a fault. This is for the case where the reply
|
|
49
|
+
* cannot be read at all, because reporting zero rows there would look exactly
|
|
50
|
+
* like a family being discontinued.
|
|
51
|
+
*/
|
|
52
|
+
export class VendorResponseError extends ScraperError {
|
|
53
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Destiny Tool's families.
|
|
3
|
+
*
|
|
4
|
+
* One today, and the vendor's whole catalog is one Firestore collection rather
|
|
5
|
+
* than a set of family pages — so there is no `familyCode` here to name a scrape
|
|
6
|
+
* target with. `vendors/destinytool/scrape.ts` fetches the collection and narrows
|
|
7
|
+
* to `type == 'End Mill'` after decoding, because `documents.list` supports no
|
|
8
|
+
* server-side filter.
|
|
9
|
+
*
|
|
10
|
+
* The 3,898 End Mills are 4,309 products minus 411 the package has no `ToolKind`
|
|
11
|
+
* for — Chamfer Mill, Thread Mill, Dovetail, Corner Rounder, Spot Drill, Drill
|
|
12
|
+
* Mill, Countersink, Chamfer End Mill. Those are deferred rather than dropped.
|
|
13
|
+
*/
|
|
14
|
+
export declare const FAMILIES: {
|
|
15
|
+
readonly 'destinytool_end_mills_inch.csv': {
|
|
16
|
+
readonly id: "end-mills-inch";
|
|
17
|
+
readonly brand: "destinytool";
|
|
18
|
+
readonly kind: "endmill";
|
|
19
|
+
readonly rows: 3898;
|
|
20
|
+
readonly columns: {
|
|
21
|
+
readonly DC: "cutDia";
|
|
22
|
+
readonly SFDM: "cutDia";
|
|
23
|
+
readonly OAL: "oal";
|
|
24
|
+
readonly LCF: "loc";
|
|
25
|
+
readonly RE: "rad";
|
|
26
|
+
};
|
|
27
|
+
readonly facts: {
|
|
28
|
+
readonly unit: {
|
|
29
|
+
readonly value: "inches";
|
|
30
|
+
readonly source: "vendor-stated";
|
|
31
|
+
readonly cite: "Destiny Tool product API (Firestore project studio-6030841929-4a1a2, collection 'products'), full scrape 2026-08-19 — every dimension on every one of the 3,898 End Mill documents is a US-customary fractional-inch string; no metric row exists";
|
|
32
|
+
};
|
|
33
|
+
readonly bmc: {
|
|
34
|
+
readonly value: "carbide";
|
|
35
|
+
readonly source: "assumed";
|
|
36
|
+
readonly note: "'material' is blank on 1,223 of 3,898 End Mills (31.4%); 'Carbide' (one row spells it 'CARBIDE') is the only non-blank value across the whole collection, and solid-carbide is standard for this class of vendor";
|
|
37
|
+
readonly checked: "2026-08-19";
|
|
38
|
+
readonly by: "JG";
|
|
39
|
+
};
|
|
40
|
+
readonly coolantThrough: {
|
|
41
|
+
readonly value: false;
|
|
42
|
+
readonly source: "assumed";
|
|
43
|
+
readonly note: "no coolant field published anywhere in the Firestore document schema";
|
|
44
|
+
readonly checked: "2026-08-19";
|
|
45
|
+
readonly by: "JG";
|
|
46
|
+
};
|
|
47
|
+
};
|
|
48
|
+
};
|
|
49
|
+
};
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Destiny Tool's families.
|
|
3
|
+
*
|
|
4
|
+
* One today, and the vendor's whole catalog is one Firestore collection rather
|
|
5
|
+
* than a set of family pages — so there is no `familyCode` here to name a scrape
|
|
6
|
+
* target with. `vendors/destinytool/scrape.ts` fetches the collection and narrows
|
|
7
|
+
* to `type == 'End Mill'` after decoding, because `documents.list` supports no
|
|
8
|
+
* server-side filter.
|
|
9
|
+
*
|
|
10
|
+
* The 3,898 End Mills are 4,309 products minus 411 the package has no `ToolKind`
|
|
11
|
+
* for — Chamfer Mill, Thread Mill, Dovetail, Corner Rounder, Spot Drill, Drill
|
|
12
|
+
* Mill, Countersink, Chamfer End Mill. Those are deferred rather than dropped.
|
|
13
|
+
*/
|
|
14
|
+
export const FAMILIES = {
|
|
15
|
+
'destinytool_end_mills_inch.csv': {
|
|
16
|
+
id: 'end-mills-inch',
|
|
17
|
+
brand: 'destinytool',
|
|
18
|
+
kind: 'endmill',
|
|
19
|
+
rows: 3898,
|
|
20
|
+
// `SFDM` maps to the **same** label as `DC` on purpose: Destiny Tool
|
|
21
|
+
// publishes no structured shank column at all, and `endmillRecord`
|
|
22
|
+
// reads a shank diameter out of the description's own "SHK" text when
|
|
23
|
+
// the tool is necked, falling back to that shared column — i.e. to DC —
|
|
24
|
+
// otherwise. Two more per-record derivations the map cannot express are
|
|
25
|
+
// in `vendors/destinytool/records.ts`, with the evidence beside them.
|
|
26
|
+
columns: {
|
|
27
|
+
DC: 'cutDia',
|
|
28
|
+
SFDM: 'cutDia',
|
|
29
|
+
OAL: 'oal',
|
|
30
|
+
LCF: 'loc',
|
|
31
|
+
RE: 'rad',
|
|
32
|
+
},
|
|
33
|
+
facts: {
|
|
34
|
+
unit: {
|
|
35
|
+
value: 'inches',
|
|
36
|
+
source: 'vendor-stated',
|
|
37
|
+
cite: "Destiny Tool product API (Firestore project studio-6030841929-4a1a2, collection 'products'), full scrape 2026-08-19 — every dimension on every one of the 3,898 End Mill documents is a US-customary fractional-inch string; no metric row exists",
|
|
38
|
+
},
|
|
39
|
+
bmc: {
|
|
40
|
+
value: 'carbide',
|
|
41
|
+
source: 'assumed',
|
|
42
|
+
note: "'material' is blank on 1,223 of 3,898 End Mills (31.4%); 'Carbide' (one row spells it 'CARBIDE') is the only non-blank value across the whole collection, and solid-carbide is standard for this class of vendor",
|
|
43
|
+
checked: '2026-08-19',
|
|
44
|
+
by: 'JG',
|
|
45
|
+
},
|
|
46
|
+
coolantThrough: {
|
|
47
|
+
value: false,
|
|
48
|
+
source: 'assumed',
|
|
49
|
+
note: 'no coolant field published anywhere in the Firestore document schema',
|
|
50
|
+
checked: '2026-08-19',
|
|
51
|
+
by: 'JG',
|
|
52
|
+
},
|
|
53
|
+
},
|
|
54
|
+
},
|
|
55
|
+
};
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-family scrape config: what to fetch, and how its columns are labelled.
|
|
3
|
+
*
|
|
4
|
+
* One module per vendor, merged here. The split is the point: a REGO-FIX
|
|
5
|
+
* family is named whatever REGO-FIX calls it without anybody checking
|
|
6
|
+
* Kennametal's list first, and adding a vendor is a new file rather than an
|
|
7
|
+
* edit to a table three other vendors depend on.
|
|
8
|
+
*
|
|
9
|
+
* **Config, not code.** Nothing here imports an adapter — `registry` is the
|
|
10
|
+
* one module that knows both, and `tests/vendor-boundary.test.ts` refuses the
|
|
11
|
+
* alternative. In the source package this table briefly did the binding
|
|
12
|
+
* itself, which made the config import a manufacturer; the table is read by
|
|
13
|
+
* every test, and none of them should drag a vendor's scraper in behind it.
|
|
14
|
+
*
|
|
15
|
+
* Three kinds of key, and they are worth telling apart:
|
|
16
|
+
*
|
|
17
|
+
* - **Scrape targets** — `familyCode`, and whatever else names the thing to
|
|
18
|
+
* fetch. This is what makes a scrape re-runnable without going back to the
|
|
19
|
+
* browser to find the family page again.
|
|
20
|
+
* - **`columns`** — the vendor's own column labels, keyed by canonical ISO
|
|
21
|
+
* 13399 name, *without* a unit suffix. `registry.bindFamilies` runs each
|
|
22
|
+
* through `records.checkColumnMap`, so a typo fails when the registry binds,
|
|
23
|
+
* naming the family.
|
|
24
|
+
* - **`facts`** — the per-family constants no vendor table states, each
|
|
25
|
+
* carrying its provenance. The registry checks and projects them, so readers
|
|
26
|
+
* say `family.unit` and never learn about provenance.
|
|
27
|
+
*
|
|
28
|
+
* **Where a scrape lands is not here.** Resolving a data root needs `fs` and
|
|
29
|
+
* `process.env`, which would make this table unimportable in anything that
|
|
30
|
+
* never writes a file. It is `node/paths.ts`.
|
|
31
|
+
*/
|
|
32
|
+
import type { FamilyDefinition, ToolholdingDefinition } from '../family.js';
|
|
33
|
+
/** Every cutting-tool family, keyed by the CSV it is scraped into. */
|
|
34
|
+
export declare const FAMILIES: Record<string, FamilyDefinition>;
|
|
35
|
+
/**
|
|
36
|
+
* Every toolholding family — holders, and the collets that go in them.
|
|
37
|
+
*
|
|
38
|
+
* Separate tables rather than a `kind` on one, because a holder and a collet
|
|
39
|
+
* are not variants of a thing: they carry different discriminants (a holder
|
|
40
|
+
* states a taper and a clamping mode; a collet states a series and a capacity
|
|
41
|
+
* band) and a scrape of one is not a scrape of the other.
|
|
42
|
+
*/
|
|
43
|
+
export declare const HOLDER_FAMILIES: Record<string, ToolholdingDefinition>;
|
|
44
|
+
export declare const COLLET_FAMILIES: Record<string, ToolholdingDefinition>;
|
|
45
|
+
/**
|
|
46
|
+
* Every family this package knows, by CSV name — tools and toolholding alike.
|
|
47
|
+
*
|
|
48
|
+
* Built once rather than searched per call, so a lookup can refuse an unknown
|
|
49
|
+
* name by listing what it does know.
|
|
50
|
+
*
|
|
51
|
+
* Through {@link merge} and not a spread, for the reason {@link merge} exists:
|
|
52
|
+
* a spread here would let a name claimed by two of the three tables resolve
|
|
53
|
+
* silently to whichever was spread last, and this is the table `familyConfig`
|
|
54
|
+
* — and therefore `familyBrand`, and therefore where a CSV is written — reads.
|
|
55
|
+
*/
|
|
56
|
+
export declare const ALL_FAMILIES: Record<string, FamilyDefinition | ToolholdingDefinition>;
|
|
57
|
+
/** One family's config by CSV name, refusing a name nothing declares. */
|
|
58
|
+
export declare function familyConfig(name: string): FamilyDefinition | ToolholdingDefinition;
|
|
59
|
+
export { familyBrand, familyId } from '../family.js';
|