@toolpath/tool-scraper 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +98 -0
- package/dist/conventions.d.ts +124 -0
- package/dist/conventions.js +143 -0
- package/dist/errors.d.ts +46 -0
- package/dist/errors.js +53 -0
- package/dist/families/destinytool.d.ts +49 -0
- package/dist/families/destinytool.js +55 -0
- package/dist/families/index.d.ts +59 -0
- package/dist/families/index.js +91 -0
- package/dist/families/kennametal.d.ts +757 -0
- package/dist/families/kennametal.js +660 -0
- package/dist/families/regofix.d.ts +185 -0
- package/dist/families/regofix.js +250 -0
- package/dist/family.d.ts +130 -0
- package/dist/family.js +38 -0
- package/dist/fetch.d.ts +98 -0
- package/dist/fetch.js +116 -0
- package/dist/identity.d.ts +133 -0
- package/dist/identity.js +118 -0
- package/dist/index.d.ts +31 -0
- package/dist/index.js +31 -0
- package/dist/node/cad-mirror.d.ts +55 -0
- package/dist/node/cad-mirror.js +89 -0
- package/dist/node/cli.d.ts +35 -0
- package/dist/node/cli.js +340 -0
- package/dist/node/csv.d.ts +47 -0
- package/dist/node/csv.js +123 -0
- package/dist/node/index.d.ts +16 -0
- package/dist/node/index.js +16 -0
- package/dist/node/main.d.ts +13 -0
- package/dist/node/main.js +14 -0
- package/dist/node/paths.d.ts +60 -0
- package/dist/node/paths.js +80 -0
- package/dist/node/receipts.d.ts +100 -0
- package/dist/node/receipts.js +107 -0
- package/dist/order.d.ts +10 -0
- package/dist/order.js +12 -0
- package/dist/provenance.d.ts +125 -0
- package/dist/provenance.js +133 -0
- package/dist/records.d.ts +305 -0
- package/dist/records.js +297 -0
- package/dist/registry.d.ts +63 -0
- package/dist/registry.js +145 -0
- package/dist/scrape.d.ts +70 -0
- package/dist/scrape.js +37 -0
- package/dist/thread.d.ts +48 -0
- package/dist/thread.js +98 -0
- package/dist/uuid5.d.ts +31 -0
- package/dist/uuid5.js +64 -0
- package/dist/vendors/destinytool/index.d.ts +11 -0
- package/dist/vendors/destinytool/index.js +11 -0
- package/dist/vendors/destinytool/records.d.ts +118 -0
- package/dist/vendors/destinytool/records.js +266 -0
- package/dist/vendors/destinytool/scrape.d.ts +108 -0
- package/dist/vendors/destinytool/scrape.js +192 -0
- package/dist/vendors/kennametal/cad.d.ts +87 -0
- package/dist/vendors/kennametal/cad.js +119 -0
- package/dist/vendors/kennametal/index.d.ts +21 -0
- package/dist/vendors/kennametal/index.js +21 -0
- package/dist/vendors/kennametal/materials.d.ts +143 -0
- package/dist/vendors/kennametal/materials.js +200 -0
- package/dist/vendors/kennametal/records.d.ts +88 -0
- package/dist/vendors/kennametal/records.js +241 -0
- package/dist/vendors/kennametal/scrape.d.ts +111 -0
- package/dist/vendors/kennametal/scrape.js +226 -0
- package/dist/vendors/kennametal/thread-column.d.ts +28 -0
- package/dist/vendors/kennametal/thread-column.js +41 -0
- package/dist/vendors/regofix/index.d.ts +8 -0
- package/dist/vendors/regofix/index.js +8 -0
- package/dist/vendors/regofix/scrape.d.ts +237 -0
- package/dist/vendors/regofix/scrape.js +521 -0
- package/package.json +76 -0
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The console entry point. Argv handling only — no scraping logic lives here.
|
|
3
|
+
*
|
|
4
|
+
* ```
|
|
5
|
+
* toolpath-scrape kennametal family page -> CSV
|
|
6
|
+
* toolpath-scrape regofix ProductFinder index -> toolholding CSV
|
|
7
|
+
* toolpath-scrape destinytool Firestore products -> End Mill CSV
|
|
8
|
+
* toolpath-scrape thread-pitch add the derived Thread Pitch column
|
|
9
|
+
* toolpath-scrape cad add the vendor CAD model column
|
|
10
|
+
* toolpath-scrape materials add the ISO workpiece-group column
|
|
11
|
+
* toolpath-scrape mirror-cad download the vendor STEP models
|
|
12
|
+
* ```
|
|
13
|
+
*
|
|
14
|
+
* **One binary with subcommands.** Seven names in `node_modules/.bin` for one
|
|
15
|
+
* package is not the idiom.
|
|
16
|
+
*
|
|
17
|
+
* **Every command prints the resolved scrape root before it does anything.**
|
|
18
|
+
* The default is derived from this package's own location, which is right in a
|
|
19
|
+
* working tree and meaningless in `node_modules`, so where a scrape lands is a
|
|
20
|
+
* thing to state rather than to assume.
|
|
21
|
+
*
|
|
22
|
+
* The convert commands are not here. This package acquires; a Fusion library
|
|
23
|
+
* or an assembly catalog is a different product, and none of it is in this
|
|
24
|
+
* tree.
|
|
25
|
+
*/
|
|
26
|
+
import { type Fetcher } from '../fetch.js';
|
|
27
|
+
/** Everything the CLI prints, injectable so a test is not stdout. */
|
|
28
|
+
export interface Console_ {
|
|
29
|
+
log: (message: string) => void;
|
|
30
|
+
error: (message: string) => void;
|
|
31
|
+
}
|
|
32
|
+
/** Run one command. Exported so the tests drive it without a subprocess. */
|
|
33
|
+
export declare function run(argv: string[], io?: Console_, fetcher?: Fetcher): Promise<number>;
|
|
34
|
+
/** The process entry point: run, and turn a refusal into an exit code. */
|
|
35
|
+
export declare function main(argv?: string[]): Promise<number>;
|
package/dist/node/cli.js
ADDED
|
@@ -0,0 +1,340 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The console entry point. Argv handling only — no scraping logic lives here.
|
|
3
|
+
*
|
|
4
|
+
* ```
|
|
5
|
+
* toolpath-scrape kennametal family page -> CSV
|
|
6
|
+
* toolpath-scrape regofix ProductFinder index -> toolholding CSV
|
|
7
|
+
* toolpath-scrape destinytool Firestore products -> End Mill CSV
|
|
8
|
+
* toolpath-scrape thread-pitch add the derived Thread Pitch column
|
|
9
|
+
* toolpath-scrape cad add the vendor CAD model column
|
|
10
|
+
* toolpath-scrape materials add the ISO workpiece-group column
|
|
11
|
+
* toolpath-scrape mirror-cad download the vendor STEP models
|
|
12
|
+
* ```
|
|
13
|
+
*
|
|
14
|
+
* **One binary with subcommands.** Seven names in `node_modules/.bin` for one
|
|
15
|
+
* package is not the idiom.
|
|
16
|
+
*
|
|
17
|
+
* **Every command prints the resolved scrape root before it does anything.**
|
|
18
|
+
* The default is derived from this package's own location, which is right in a
|
|
19
|
+
* working tree and meaningless in `node_modules`, so where a scrape lands is a
|
|
20
|
+
* thing to state rather than to assume.
|
|
21
|
+
*
|
|
22
|
+
* The convert commands are not here. This package acquires; a Fusion library
|
|
23
|
+
* or an assembly catalog is a different product, and none of it is in this
|
|
24
|
+
* tree.
|
|
25
|
+
*/
|
|
26
|
+
import { readFileSync, writeFileSync } from 'node:fs';
|
|
27
|
+
import { basename } from 'node:path';
|
|
28
|
+
import { ScraperConfigError, VendorResponseError } from '../errors.js';
|
|
29
|
+
import { familyBrand } from '../family.js';
|
|
30
|
+
import { ALL_FAMILIES, FAMILIES, HOLDER_FAMILIES, familyConfig } from '../families/index.js';
|
|
31
|
+
import { createFetcher } from '../fetch.js';
|
|
32
|
+
import { AEM_BRANDS } from '../identity.js';
|
|
33
|
+
import { boundFamily } from '../registry.js';
|
|
34
|
+
import { mirrorFamilySteps } from './cad-mirror.js';
|
|
35
|
+
import { parseCsv, toCsv } from './csv.js';
|
|
36
|
+
import { describeRoot, familyCsv, stepDir } from './paths.js';
|
|
37
|
+
import * as receipts from './receipts.js';
|
|
38
|
+
import { scrapeFamily } from '../vendors/kennametal/scrape.js';
|
|
39
|
+
import { annotateCadUrls } from '../vendors/kennametal/cad.js';
|
|
40
|
+
import { addMaterialGroups, groupsByMaterial } from '../vendors/kennametal/materials.js';
|
|
41
|
+
import { addThreadPitch } from '../vendors/kennametal/thread-column.js';
|
|
42
|
+
import { scrapeEndMills, DOCUMENTS_URL } from '../vendors/destinytool/scrape.js';
|
|
43
|
+
import { SEARCH_URL, scrapeCollets, scrapeHolders } from '../vendors/regofix/scrape.js';
|
|
44
|
+
/**
|
|
45
|
+
* The PG series a BT 30 holder can take. PG 32 and PG 48 collets exist and no
|
|
46
|
+
* BT 30 holder in this catalog accepts one, so scraping them would add parts
|
|
47
|
+
* that fit nothing.
|
|
48
|
+
*/
|
|
49
|
+
const BT30_COLLET_SIZES = ['6', '10', '15', '25'];
|
|
50
|
+
const USAGE = `usage: toolpath-scrape <command> [args]
|
|
51
|
+
|
|
52
|
+
kennametal [--brand kennametal|widia] FAMILY_CODE OUTPUT_CSV [Name=Value ...]
|
|
53
|
+
One AEM family page -> a CSV. Trailing Name=Value args are appended to
|
|
54
|
+
every row as constant columns, for facts the vendor table does not state
|
|
55
|
+
(e.g. "Thread System=metric").
|
|
56
|
+
|
|
57
|
+
regofix holders OUT.csv
|
|
58
|
+
regofix collets "<PRODUCT GROUP>" OUT.csv
|
|
59
|
+
The REGO-FIX ProductFinder index. \`holders\` takes every powRgrip BT/PG
|
|
60
|
+
holder whose taper is BT 30 or BT+ 30, with geometry from each part's
|
|
61
|
+
DIN 4000 document. \`collets\` takes one product group, restricted to PG
|
|
62
|
+
sizes ${BT30_COLLET_SIZES.join(', ')} — the group is the vendor's own
|
|
63
|
+
\`product_group_name\`, e.g. "Standard", "Coolant flush",
|
|
64
|
+
"Tapping collet TAP".
|
|
65
|
+
|
|
66
|
+
destinytool OUTPUT_CSV
|
|
67
|
+
Pages the whole Destiny Tool \`products\` Firestore collection and writes
|
|
68
|
+
every End Mill row.
|
|
69
|
+
|
|
70
|
+
thread-pitch TAP.csv [more.csv ...]
|
|
71
|
+
Adds a Thread Pitch column derived from D1-TDZ, in place. Safe to re-run.
|
|
72
|
+
|
|
73
|
+
cad HOLDERS.csv [more.csv ...]
|
|
74
|
+
Adds the vendor CAD model URL column, in place. One request per row
|
|
75
|
+
against product-config.net; safe to re-run.
|
|
76
|
+
|
|
77
|
+
materials FAMILY.csv [more.csv ...]
|
|
78
|
+
Adds the ISO workpiece-group column, in place. One request per material
|
|
79
|
+
group (32) per family; safe to re-run.
|
|
80
|
+
|
|
81
|
+
mirror-cad HOLDERS.csv [more.csv ...]
|
|
82
|
+
Downloads each row's STEP model into <root>/<brand>/step. Run \`cad\`
|
|
83
|
+
first — a CSV with no CAD column yields nothing and says so.
|
|
84
|
+
|
|
85
|
+
An output path is used verbatim; scraped CSVs belong under the scrape root,
|
|
86
|
+
in <brand>/csv/. The in-place commands take a bare CSV name and resolve it
|
|
87
|
+
through the family's own brand.`;
|
|
88
|
+
const STDOUT = {
|
|
89
|
+
log: (message) => process.stdout.write(`${message}\n`),
|
|
90
|
+
error: (message) => process.stderr.write(`${message}\n`),
|
|
91
|
+
};
|
|
92
|
+
/** Read one family's CSV off disk as a scrape result. */
|
|
93
|
+
function readCsv(name, source) {
|
|
94
|
+
const path = familyCsv(name);
|
|
95
|
+
const { header, rows } = parseCsv(readFileSync(path, 'utf8'));
|
|
96
|
+
return { header, rows, source, familyCode: null };
|
|
97
|
+
}
|
|
98
|
+
/** Write a scrape to `path`. */
|
|
99
|
+
function writeCsv(path, scrape) {
|
|
100
|
+
writeFileSync(path, toCsv(scrape.header, scrape.rows));
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Report a scrape and record its receipt.
|
|
104
|
+
*
|
|
105
|
+
* One function because the two belong together: a scrape that reported a count
|
|
106
|
+
* and wrote no receipt would be exactly the state this package is trying to
|
|
107
|
+
* stop existing — data with nothing saying where it came from.
|
|
108
|
+
*/
|
|
109
|
+
function wrote(out, brand, scrape, io) {
|
|
110
|
+
writeCsv(out, scrape);
|
|
111
|
+
const receipt = receipts.write(out, {
|
|
112
|
+
brand,
|
|
113
|
+
source: scrape.source,
|
|
114
|
+
rows: scrape.rows.length,
|
|
115
|
+
familyCode: scrape.familyCode,
|
|
116
|
+
});
|
|
117
|
+
io.log(`wrote ${scrape.rows.length} rows to ${out}`);
|
|
118
|
+
io.log(` receipt: ${basename(receipt)}`);
|
|
119
|
+
const name = basename(out);
|
|
120
|
+
// Through the merged table rather than three chained lookups, so a name two
|
|
121
|
+
// tables both claim is refused where it is built instead of resolving here
|
|
122
|
+
// to whichever happened to be checked first.
|
|
123
|
+
const declared = ALL_FAMILIES[name]?.rows;
|
|
124
|
+
const written = receipts.read(out);
|
|
125
|
+
if (declared !== undefined && written !== null) {
|
|
126
|
+
receipts.checkRows(name, declared, written);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Argv as family CSV names, refusing anything unknown by name.
|
|
131
|
+
*
|
|
132
|
+
* A path's directory is ignored: the family's own brand decides where its CSV
|
|
133
|
+
* lives, and honouring a typed directory would let one vendor's receipt be
|
|
134
|
+
* written into another's.
|
|
135
|
+
*/
|
|
136
|
+
function namesIn(argv, known, what) {
|
|
137
|
+
const names = argv.map((a) => basename(a));
|
|
138
|
+
const unknown = names.filter((n) => !Object.hasOwn(known, n));
|
|
139
|
+
if (unknown.length > 0) {
|
|
140
|
+
throw new ScraperConfigError(unknown.join(', '), `unknown ${what} CSV (known: ${Object.keys(known).sort().join(', ')})`);
|
|
141
|
+
}
|
|
142
|
+
return names;
|
|
143
|
+
}
|
|
144
|
+
/** Run one command. Exported so the tests drive it without a subprocess. */
|
|
145
|
+
export async function run(argv, io = STDOUT, fetcher = createFetcher()) {
|
|
146
|
+
// Somebody reading the usage text is the person most likely to be about to
|
|
147
|
+
// point a scrape at the wrong place, so help gets the root too.
|
|
148
|
+
io.log(describeRoot());
|
|
149
|
+
const [command, ...rest] = argv;
|
|
150
|
+
if (command === undefined || command === '-h' || command === '--help') {
|
|
151
|
+
io.log(USAGE);
|
|
152
|
+
return 0;
|
|
153
|
+
}
|
|
154
|
+
switch (command) {
|
|
155
|
+
case 'kennametal':
|
|
156
|
+
return kennametal(rest, io, fetcher);
|
|
157
|
+
case 'regofix':
|
|
158
|
+
return regofix(rest, io, fetcher);
|
|
159
|
+
case 'destinytool':
|
|
160
|
+
return destinytool(rest, io, fetcher);
|
|
161
|
+
case 'thread-pitch':
|
|
162
|
+
return threadPitch(rest, io);
|
|
163
|
+
case 'cad':
|
|
164
|
+
return cad(rest, io, fetcher);
|
|
165
|
+
case 'materials':
|
|
166
|
+
return materials(rest, io, fetcher);
|
|
167
|
+
case 'mirror-cad':
|
|
168
|
+
return mirrorCad(rest, io, fetcher);
|
|
169
|
+
default:
|
|
170
|
+
io.error(`unknown command ${JSON.stringify(command)}\n\n${USAGE}`);
|
|
171
|
+
return 2;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
async function kennametal(argv, io, fetcher) {
|
|
175
|
+
const args = [...argv];
|
|
176
|
+
let brand = 'kennametal';
|
|
177
|
+
const flag = args.indexOf('--brand');
|
|
178
|
+
if (flag !== -1) {
|
|
179
|
+
const value = args[flag + 1];
|
|
180
|
+
if (value === undefined) {
|
|
181
|
+
io.error(`--brand needs a value\n\n${USAGE}`);
|
|
182
|
+
return 2;
|
|
183
|
+
}
|
|
184
|
+
brand = value;
|
|
185
|
+
args.splice(flag, 2);
|
|
186
|
+
}
|
|
187
|
+
// Against the AEM brands rather than every brand: `scrapeFamily` reads
|
|
188
|
+
// `Brand.node`, and a brand without one built a URL with `undefined` in the
|
|
189
|
+
// path and died on the 404 instead of on this line.
|
|
190
|
+
if (!AEM_BRANDS.includes(brand)) {
|
|
191
|
+
io.error(`unknown brand: ${brand} (known: ${[...AEM_BRANDS].sort().join(', ')})`);
|
|
192
|
+
return 2;
|
|
193
|
+
}
|
|
194
|
+
if (args.length < 2) {
|
|
195
|
+
io.error(USAGE);
|
|
196
|
+
return 2;
|
|
197
|
+
}
|
|
198
|
+
const [code, out] = args;
|
|
199
|
+
// Refused rather than dropped: a quoting slip like `"Thread System" metric`
|
|
200
|
+
// used to scrape, write the CSV and exit 0 with the column missing, and the
|
|
201
|
+
// failure surfaced later in `addThreadPitch` against a file no longer being
|
|
202
|
+
// written.
|
|
203
|
+
const constants = args.slice(2);
|
|
204
|
+
const malformed = constants.find((a) => a.indexOf('=') < 1);
|
|
205
|
+
if (malformed !== undefined) {
|
|
206
|
+
io.error(`constant column ${JSON.stringify(malformed)} is not Name=Value\n\n${USAGE}`);
|
|
207
|
+
return 2;
|
|
208
|
+
}
|
|
209
|
+
const tags = constants.map((a) => {
|
|
210
|
+
const at = a.indexOf('=');
|
|
211
|
+
return [a.slice(0, at), a.slice(at + 1)];
|
|
212
|
+
});
|
|
213
|
+
const scrape = await scrapeFamily(fetcher, code, brand, tags);
|
|
214
|
+
wrote(out, brand, scrape, io);
|
|
215
|
+
return 0;
|
|
216
|
+
}
|
|
217
|
+
async function regofix(argv, io, fetcher) {
|
|
218
|
+
const [what, ...rest] = argv;
|
|
219
|
+
if (what === 'holders') {
|
|
220
|
+
if (rest.length !== 1) {
|
|
221
|
+
io.error(USAGE);
|
|
222
|
+
return 2;
|
|
223
|
+
}
|
|
224
|
+
const scrape = await scrapeHolders(fetcher, 'BT/PG', 'BT', {
|
|
225
|
+
warn: io.error,
|
|
226
|
+
});
|
|
227
|
+
wrote(rest[0], 'regofix', { ...scrape, source: SEARCH_URL }, io);
|
|
228
|
+
return 0;
|
|
229
|
+
}
|
|
230
|
+
if (what === 'collets') {
|
|
231
|
+
if (rest.length !== 2) {
|
|
232
|
+
io.error(USAGE);
|
|
233
|
+
return 2;
|
|
234
|
+
}
|
|
235
|
+
const [group, out] = rest;
|
|
236
|
+
const scrape = await scrapeCollets(fetcher, group, BT30_COLLET_SIZES, {
|
|
237
|
+
warn: io.error,
|
|
238
|
+
});
|
|
239
|
+
wrote(out, 'regofix', { ...scrape, source: SEARCH_URL }, io);
|
|
240
|
+
return 0;
|
|
241
|
+
}
|
|
242
|
+
io.error(`unknown subcommand ${JSON.stringify(what)}\n\n${USAGE}`);
|
|
243
|
+
return 2;
|
|
244
|
+
}
|
|
245
|
+
async function destinytool(argv, io, fetcher) {
|
|
246
|
+
if (argv.length !== 1) {
|
|
247
|
+
io.error(USAGE);
|
|
248
|
+
return 2;
|
|
249
|
+
}
|
|
250
|
+
const scrape = await scrapeEndMills(fetcher);
|
|
251
|
+
wrote(argv[0], 'destinytool', { ...scrape, source: DOCUMENTS_URL }, io);
|
|
252
|
+
return 0;
|
|
253
|
+
}
|
|
254
|
+
function threadPitch(argv, io) {
|
|
255
|
+
if (argv.length === 0) {
|
|
256
|
+
io.error(USAGE);
|
|
257
|
+
return 2;
|
|
258
|
+
}
|
|
259
|
+
for (const name of namesIn(argv, FAMILIES, 'family')) {
|
|
260
|
+
const path = familyCsv(name);
|
|
261
|
+
const updated = addThreadPitch(readCsv(name, path));
|
|
262
|
+
writeCsv(path, updated);
|
|
263
|
+
io.log(`${name}: ${updated.rows.length} rows updated`);
|
|
264
|
+
}
|
|
265
|
+
return 0;
|
|
266
|
+
}
|
|
267
|
+
async function cad(argv, io, fetcher) {
|
|
268
|
+
if (argv.length === 0) {
|
|
269
|
+
io.error(USAGE);
|
|
270
|
+
return 2;
|
|
271
|
+
}
|
|
272
|
+
for (const name of namesIn(argv, HOLDER_FAMILIES, 'holder')) {
|
|
273
|
+
// `annotateCadUrls` is Kennametal's CDS lookup, not a vendor-neutral one —
|
|
274
|
+
// it queries product-config.net and rewrites `CAD_COLUMN` on every row. Run
|
|
275
|
+
// against a REGO-FIX holder it would post that vendor's SKUs to Kennametal
|
|
276
|
+
// and blank the STEP URLs the REGO-FIX scrape had already filled in.
|
|
277
|
+
// `mirror-cad` reads the column and *is* neutral; this writes it and is not.
|
|
278
|
+
const brand = familyBrand(familyConfig(name));
|
|
279
|
+
if (!AEM_BRANDS.includes(brand)) {
|
|
280
|
+
io.error(`${name}: the cad step is ${[...AEM_BRANDS].sort().join('/')}-only — ` +
|
|
281
|
+
`${brand} publishes its own CAD URLs with the scrape`);
|
|
282
|
+
return 2;
|
|
283
|
+
}
|
|
284
|
+
const path = familyCsv(name);
|
|
285
|
+
const { scrape, found } = await annotateCadUrls(fetcher, readCsv(name, path));
|
|
286
|
+
writeCsv(path, scrape);
|
|
287
|
+
io.log(`${name}: ${found} CAD models`);
|
|
288
|
+
}
|
|
289
|
+
return 0;
|
|
290
|
+
}
|
|
291
|
+
async function materials(argv, io, fetcher) {
|
|
292
|
+
if (argv.length === 0) {
|
|
293
|
+
io.error(USAGE);
|
|
294
|
+
return 2;
|
|
295
|
+
}
|
|
296
|
+
// The family code and brand come from config, so a re-run needs neither
|
|
297
|
+
// typed again.
|
|
298
|
+
for (const name of namesIn(argv, FAMILIES, 'family')) {
|
|
299
|
+
const cfg = boundFamily(name);
|
|
300
|
+
if (cfg.familyCode === undefined) {
|
|
301
|
+
throw new ScraperConfigError(name, 'has no familyCode — the material sweep re-queries the family page ' +
|
|
302
|
+
'and cannot without one');
|
|
303
|
+
}
|
|
304
|
+
const path = familyCsv(name);
|
|
305
|
+
const found = await groupsByMaterial(fetcher, cfg.familyCode, {
|
|
306
|
+
brand: (cfg.brand ?? 'kennametal'),
|
|
307
|
+
});
|
|
308
|
+
const { scrape, matched } = addMaterialGroups(readCsv(name, path), found);
|
|
309
|
+
writeCsv(path, scrape);
|
|
310
|
+
io.log(`${name}: ${matched} rows with a material group`);
|
|
311
|
+
}
|
|
312
|
+
return 0;
|
|
313
|
+
}
|
|
314
|
+
async function mirrorCad(argv, io, fetcher) {
|
|
315
|
+
if (argv.length === 0) {
|
|
316
|
+
io.error(USAGE);
|
|
317
|
+
return 2;
|
|
318
|
+
}
|
|
319
|
+
for (const name of namesIn(argv, HOLDER_FAMILIES, 'holder')) {
|
|
320
|
+
const path = familyCsv(name);
|
|
321
|
+
const brand = familyBrand(familyConfig(name));
|
|
322
|
+
const written = await mirrorFamilySteps(fetcher, readCsv(name, path).rows, stepDir(brand), undefined, io.error);
|
|
323
|
+
const total = written.reduce((sum, f) => sum + f.bytes, 0);
|
|
324
|
+
io.log(`${name}: ${written.length} STEP files, ${Math.floor(total / 1024)} KB`);
|
|
325
|
+
}
|
|
326
|
+
return 0;
|
|
327
|
+
}
|
|
328
|
+
/** The process entry point: run, and turn a refusal into an exit code. */
|
|
329
|
+
export async function main(argv = process.argv.slice(2)) {
|
|
330
|
+
try {
|
|
331
|
+
return await run(argv);
|
|
332
|
+
}
|
|
333
|
+
catch (error) {
|
|
334
|
+
if (error instanceof ScraperConfigError || error instanceof VendorResponseError) {
|
|
335
|
+
STDOUT.error(error.message);
|
|
336
|
+
return 2;
|
|
337
|
+
}
|
|
338
|
+
throw error;
|
|
339
|
+
}
|
|
340
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CSV, to the letter of what Python's `csv` module writes.
|
|
3
|
+
*
|
|
4
|
+
* Node has no CSV in its standard library and this package needs about sixty
|
|
5
|
+
* lines of one, so it is here rather than a dependency — the same call
|
|
6
|
+
* `uuid5.ts` makes, and for the same reason: a dependency whose whole surface
|
|
7
|
+
* is two functions is a supply chain for no gain.
|
|
8
|
+
*
|
|
9
|
+
* **Byte-compatible with the Python this replaces, deliberately.** Scraped
|
|
10
|
+
* CSVs already exist on the machines that keep a corpus, and a re-scrape has
|
|
11
|
+
* to produce the same file or every diff is noise. That means three details
|
|
12
|
+
* that are not the obvious defaults:
|
|
13
|
+
*
|
|
14
|
+
* 1. **`\r\n` line endings.** Python's `csv.writer` uses them regardless of
|
|
15
|
+
* platform; `open(path, 'w', newline='')` is what stops the runtime
|
|
16
|
+
* translating them, not a choice to emit `\n`.
|
|
17
|
+
* 2. **Minimal quoting.** A field is quoted only when it contains a comma, a
|
|
18
|
+
* quote or a newline — `QUOTE_MINIMAL`, Python's default.
|
|
19
|
+
* 3. **Doubled quotes**, not backslash escapes.
|
|
20
|
+
*
|
|
21
|
+
* The reader accepts either line ending, because a CSV that has been through
|
|
22
|
+
* an editor is still a CSV.
|
|
23
|
+
*/
|
|
24
|
+
import type { ScrapedRow } from '../scrape.js';
|
|
25
|
+
/**
|
|
26
|
+
* Rows to CSV text, in `header` order.
|
|
27
|
+
*
|
|
28
|
+
* A row missing one of the header's columns writes an empty cell rather than
|
|
29
|
+
* failing: a mixed-unit collet family has `D1_mm` on its metric rows and
|
|
30
|
+
* `D1_in` on its inch ones, and both columns are in the union header.
|
|
31
|
+
*/
|
|
32
|
+
export declare function toCsv(header: readonly string[], rows: readonly ScrapedRow[]): string;
|
|
33
|
+
/** What {@link parseCsv} answers with. */
|
|
34
|
+
export interface ParsedCsv {
|
|
35
|
+
header: string[];
|
|
36
|
+
rows: ScrapedRow[];
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* CSV text back into a header and rows.
|
|
40
|
+
*
|
|
41
|
+
* A row longer than the header keeps its extra cells under no name, and a
|
|
42
|
+
* shorter one leaves the missing columns empty — the same shape Python's
|
|
43
|
+
* `DictReader` produces, and the reason neither is an error is that these
|
|
44
|
+
* files are re-read by the annotate steps, which have to be able to say what
|
|
45
|
+
* changed rather than refuse the file.
|
|
46
|
+
*/
|
|
47
|
+
export declare function parseCsv(text: string): ParsedCsv;
|
package/dist/node/csv.js
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CSV, to the letter of what Python's `csv` module writes.
|
|
3
|
+
*
|
|
4
|
+
* Node has no CSV in its standard library and this package needs about sixty
|
|
5
|
+
* lines of one, so it is here rather than a dependency — the same call
|
|
6
|
+
* `uuid5.ts` makes, and for the same reason: a dependency whose whole surface
|
|
7
|
+
* is two functions is a supply chain for no gain.
|
|
8
|
+
*
|
|
9
|
+
* **Byte-compatible with the Python this replaces, deliberately.** Scraped
|
|
10
|
+
* CSVs already exist on the machines that keep a corpus, and a re-scrape has
|
|
11
|
+
* to produce the same file or every diff is noise. That means three details
|
|
12
|
+
* that are not the obvious defaults:
|
|
13
|
+
*
|
|
14
|
+
* 1. **`\r\n` line endings.** Python's `csv.writer` uses them regardless of
|
|
15
|
+
* platform; `open(path, 'w', newline='')` is what stops the runtime
|
|
16
|
+
* translating them, not a choice to emit `\n`.
|
|
17
|
+
* 2. **Minimal quoting.** A field is quoted only when it contains a comma, a
|
|
18
|
+
* quote or a newline — `QUOTE_MINIMAL`, Python's default.
|
|
19
|
+
* 3. **Doubled quotes**, not backslash escapes.
|
|
20
|
+
*
|
|
21
|
+
* The reader accepts either line ending, because a CSV that has been through
|
|
22
|
+
* an editor is still a CSV.
|
|
23
|
+
*/
|
|
24
|
+
const NEEDS_QUOTING = /[",\r\n]/;
|
|
25
|
+
/** One field, quoted only if it has to be. */
|
|
26
|
+
function field(value) {
|
|
27
|
+
return NEEDS_QUOTING.test(value) ? `"${value.replaceAll('"', '""')}"` : value;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Rows to CSV text, in `header` order.
|
|
31
|
+
*
|
|
32
|
+
* A row missing one of the header's columns writes an empty cell rather than
|
|
33
|
+
* failing: a mixed-unit collet family has `D1_mm` on its metric rows and
|
|
34
|
+
* `D1_in` on its inch ones, and both columns are in the union header.
|
|
35
|
+
*/
|
|
36
|
+
export function toCsv(header, rows) {
|
|
37
|
+
const lines = [header.map(field).join(',')];
|
|
38
|
+
for (const row of rows) {
|
|
39
|
+
lines.push(header.map((column) => field(row[column] ?? '')).join(','));
|
|
40
|
+
}
|
|
41
|
+
return `${lines.join('\r\n')}\r\n`;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* CSV text back into a header and rows.
|
|
45
|
+
*
|
|
46
|
+
* A row longer than the header keeps its extra cells under no name, and a
|
|
47
|
+
* shorter one leaves the missing columns empty — the same shape Python's
|
|
48
|
+
* `DictReader` produces, and the reason neither is an error is that these
|
|
49
|
+
* files are re-read by the annotate steps, which have to be able to say what
|
|
50
|
+
* changed rather than refuse the file.
|
|
51
|
+
*/
|
|
52
|
+
export function parseCsv(text) {
|
|
53
|
+
const records = parseRecords(text);
|
|
54
|
+
const header = records.shift() ?? [];
|
|
55
|
+
const rows = records.map((cells) => {
|
|
56
|
+
const row = {};
|
|
57
|
+
header.forEach((column, index) => {
|
|
58
|
+
row[column] = cells[index] ?? '';
|
|
59
|
+
});
|
|
60
|
+
return row;
|
|
61
|
+
});
|
|
62
|
+
return { header, rows };
|
|
63
|
+
}
|
|
64
|
+
/** The raw grid, before a header is applied. */
|
|
65
|
+
function parseRecords(text) {
|
|
66
|
+
const records = [];
|
|
67
|
+
let record = [];
|
|
68
|
+
let cell = '';
|
|
69
|
+
let quoted = false;
|
|
70
|
+
let started = false;
|
|
71
|
+
const endCell = () => {
|
|
72
|
+
record.push(cell);
|
|
73
|
+
cell = '';
|
|
74
|
+
};
|
|
75
|
+
const endRecord = () => {
|
|
76
|
+
endCell();
|
|
77
|
+
records.push(record);
|
|
78
|
+
record = [];
|
|
79
|
+
started = false;
|
|
80
|
+
};
|
|
81
|
+
for (let i = 0; i < text.length; i += 1) {
|
|
82
|
+
const char = text[i];
|
|
83
|
+
if (quoted) {
|
|
84
|
+
if (char === '"') {
|
|
85
|
+
if (text[i + 1] === '"') {
|
|
86
|
+
cell += '"';
|
|
87
|
+
i += 1;
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
quoted = false;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
else {
|
|
94
|
+
cell += char;
|
|
95
|
+
}
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
if (char === '"' && cell === '') {
|
|
99
|
+
quoted = true;
|
|
100
|
+
started = true;
|
|
101
|
+
}
|
|
102
|
+
else if (char === ',') {
|
|
103
|
+
endCell();
|
|
104
|
+
started = true;
|
|
105
|
+
}
|
|
106
|
+
else if (char === '\r' || char === '\n') {
|
|
107
|
+
if (char === '\r' && text[i + 1] === '\n')
|
|
108
|
+
i += 1;
|
|
109
|
+
// A blank line between records is not a record. Python's reader skips
|
|
110
|
+
// it, and a trailing newline would otherwise add a row of one empty cell
|
|
111
|
+
// to every file this package writes.
|
|
112
|
+
if (started || cell !== '' || record.length > 0)
|
|
113
|
+
endRecord();
|
|
114
|
+
}
|
|
115
|
+
else {
|
|
116
|
+
cell += char;
|
|
117
|
+
started = true;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
if (started || cell !== '' || record.length > 0)
|
|
121
|
+
endRecord();
|
|
122
|
+
return records;
|
|
123
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The half of this package that touches the filesystem.
|
|
3
|
+
*
|
|
4
|
+
* A scrape returns rows; turning them into a CSV, writing the provenance
|
|
5
|
+
* sidecar that goes beside one, resolving where a vendor's files live and
|
|
6
|
+
* mirroring the vendor's CAD binaries all need `fs`, and a backend that only
|
|
7
|
+
* wants records should not have to import any of it.
|
|
8
|
+
*
|
|
9
|
+
* So they are a separate entry point — `@toolpath/tool-scraper/node` — and the
|
|
10
|
+
* main one stays what a library ought to be: functions in, values out.
|
|
11
|
+
*/
|
|
12
|
+
export * from './cad-mirror.js';
|
|
13
|
+
export * from './cli.js';
|
|
14
|
+
export * from './csv.js';
|
|
15
|
+
export * from './paths.js';
|
|
16
|
+
export * from './receipts.js';
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The half of this package that touches the filesystem.
|
|
3
|
+
*
|
|
4
|
+
* A scrape returns rows; turning them into a CSV, writing the provenance
|
|
5
|
+
* sidecar that goes beside one, resolving where a vendor's files live and
|
|
6
|
+
* mirroring the vendor's CAD binaries all need `fs`, and a backend that only
|
|
7
|
+
* wants records should not have to import any of it.
|
|
8
|
+
*
|
|
9
|
+
* So they are a separate entry point — `@toolpath/tool-scraper/node` — and the
|
|
10
|
+
* main one stays what a library ought to be: functions in, values out.
|
|
11
|
+
*/
|
|
12
|
+
export * from './cad-mirror.js';
|
|
13
|
+
export * from './cli.js';
|
|
14
|
+
export * from './csv.js';
|
|
15
|
+
export * from './paths.js';
|
|
16
|
+
export * from './receipts.js';
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* The executable. Nothing but the shebang and the call.
|
|
4
|
+
*
|
|
5
|
+
* Separate from `cli.ts` because that module is also imported —
|
|
6
|
+
* `@toolpath/tool-scraper/node` re-exports `run` so a consumer can drive a
|
|
7
|
+
* command without a subprocess, and a top-level "if this is the entry point"
|
|
8
|
+
* guard in an imported module is both fragile and a side effect. It was fragile
|
|
9
|
+
* here in exactly the way that matters: run through the `bin` symlink,
|
|
10
|
+
* `process.argv[1]` is the symlink's name and the guard never matched, so the
|
|
11
|
+
* installed command printed nothing and exited 0.
|
|
12
|
+
*/
|
|
13
|
+
export {};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* The executable. Nothing but the shebang and the call.
|
|
4
|
+
*
|
|
5
|
+
* Separate from `cli.ts` because that module is also imported —
|
|
6
|
+
* `@toolpath/tool-scraper/node` re-exports `run` so a consumer can drive a
|
|
7
|
+
* command without a subprocess, and a top-level "if this is the entry point"
|
|
8
|
+
* guard in an imported module is both fragile and a side effect. It was fragile
|
|
9
|
+
* here in exactly the way that matters: run through the `bin` symlink,
|
|
10
|
+
* `process.argv[1]` is the symlink's name and the guard never matched, so the
|
|
11
|
+
* installed command printed nothing and exited 0.
|
|
12
|
+
*/
|
|
13
|
+
import { main } from './cli.js';
|
|
14
|
+
process.exitCode = await main();
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where a scrape lands.
|
|
3
|
+
*
|
|
4
|
+
* **Scraped output is never committed.** A CSV is a vendor's data and a
|
|
5
|
+
* working file, not source, and this repository is public — which is a second
|
|
6
|
+
* reason, independent of size, to keep it out. Git was carrying the provenance
|
|
7
|
+
* of those CSVs for free; now that it is not, every scrape writes a
|
|
8
|
+
* {@link receipts} sidecar beside its file.
|
|
9
|
+
*
|
|
10
|
+
* This module is in `node/` rather than beside the config tables: resolving a
|
|
11
|
+
* root needs `process.env` and a path relative to this file's own location,
|
|
12
|
+
* and a family table that needed either would be unimportable in anything that
|
|
13
|
+
* only wants to read records.
|
|
14
|
+
*/
|
|
15
|
+
import type { BrandName } from '../identity.js';
|
|
16
|
+
/**
|
|
17
|
+
* Where scraped CSVs are read from and written to.
|
|
18
|
+
*
|
|
19
|
+
* Set it when the package is installed rather than run from a checkout: the
|
|
20
|
+
* default below is derived from this file's own location, which is right in a
|
|
21
|
+
* working tree and meaningless inside `node_modules`. Every command prints the
|
|
22
|
+
* resolved root for exactly that reason — a scrape that wrote somewhere
|
|
23
|
+
* surprising should say so on the way, not be discovered later.
|
|
24
|
+
*/
|
|
25
|
+
export declare const SCRAPE_ROOT_ENV = "TOOLPATH_SCRAPE_ROOT";
|
|
26
|
+
/** `packages/tool-scraper/scrape-out`, which `.gitignore` already covers. */
|
|
27
|
+
export declare const DEFAULT_SCRAPE_ROOT: string;
|
|
28
|
+
/** The directory holding every vendor's scraped CSVs. */
|
|
29
|
+
export declare function scrapeRoot(): string;
|
|
30
|
+
/**
|
|
31
|
+
* One line naming the resolved root and how it was resolved.
|
|
32
|
+
*
|
|
33
|
+
* Printed by every command. The distinction it carries is the one that matters
|
|
34
|
+
* when a scrape goes somewhere unexpected: whether the path came from the
|
|
35
|
+
* environment or from this package's own location.
|
|
36
|
+
*/
|
|
37
|
+
export declare function describeRoot(): string;
|
|
38
|
+
/**
|
|
39
|
+
* Where one vendor's scraped CSVs live — the receipts.
|
|
40
|
+
*
|
|
41
|
+
* Per brand rather than per adapter, and the distinction is worth holding on
|
|
42
|
+
* to: an adapter is a fact about *code*, a scraped table is a fact about who
|
|
43
|
+
* published it. WIDIA's tables are WIDIA's even though Kennametal's adapter is
|
|
44
|
+
* what fetched them.
|
|
45
|
+
*/
|
|
46
|
+
export declare function csvDir(brand: BrandName): string;
|
|
47
|
+
/**
|
|
48
|
+
* One vendor's mirrored STEP models.
|
|
49
|
+
*
|
|
50
|
+
* Nothing is redistributed from here: these are a local working copy for
|
|
51
|
+
* measuring a holder, and only a derived profile is ever meant to leave.
|
|
52
|
+
*/
|
|
53
|
+
export declare function stepDir(brand: BrandName): string;
|
|
54
|
+
/**
|
|
55
|
+
* Where one family's CSV lives, resolved through its own brand.
|
|
56
|
+
*
|
|
57
|
+
* Takes a bare CSV name rather than a path, so a caller cannot pass a file
|
|
58
|
+
* from somewhere else and have it silently treated as this family's receipt.
|
|
59
|
+
*/
|
|
60
|
+
export declare function familyCsv(name: string): string;
|