@toolpath/tool-scraper 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +98 -0
  3. package/dist/conventions.d.ts +124 -0
  4. package/dist/conventions.js +143 -0
  5. package/dist/errors.d.ts +46 -0
  6. package/dist/errors.js +53 -0
  7. package/dist/families/destinytool.d.ts +49 -0
  8. package/dist/families/destinytool.js +55 -0
  9. package/dist/families/index.d.ts +59 -0
  10. package/dist/families/index.js +91 -0
  11. package/dist/families/kennametal.d.ts +757 -0
  12. package/dist/families/kennametal.js +660 -0
  13. package/dist/families/regofix.d.ts +185 -0
  14. package/dist/families/regofix.js +250 -0
  15. package/dist/family.d.ts +130 -0
  16. package/dist/family.js +38 -0
  17. package/dist/fetch.d.ts +98 -0
  18. package/dist/fetch.js +116 -0
  19. package/dist/identity.d.ts +133 -0
  20. package/dist/identity.js +118 -0
  21. package/dist/index.d.ts +31 -0
  22. package/dist/index.js +31 -0
  23. package/dist/node/cad-mirror.d.ts +55 -0
  24. package/dist/node/cad-mirror.js +89 -0
  25. package/dist/node/cli.d.ts +35 -0
  26. package/dist/node/cli.js +340 -0
  27. package/dist/node/csv.d.ts +47 -0
  28. package/dist/node/csv.js +123 -0
  29. package/dist/node/index.d.ts +16 -0
  30. package/dist/node/index.js +16 -0
  31. package/dist/node/main.d.ts +13 -0
  32. package/dist/node/main.js +14 -0
  33. package/dist/node/paths.d.ts +60 -0
  34. package/dist/node/paths.js +80 -0
  35. package/dist/node/receipts.d.ts +100 -0
  36. package/dist/node/receipts.js +107 -0
  37. package/dist/order.d.ts +10 -0
  38. package/dist/order.js +12 -0
  39. package/dist/provenance.d.ts +125 -0
  40. package/dist/provenance.js +133 -0
  41. package/dist/records.d.ts +305 -0
  42. package/dist/records.js +297 -0
  43. package/dist/registry.d.ts +63 -0
  44. package/dist/registry.js +145 -0
  45. package/dist/scrape.d.ts +70 -0
  46. package/dist/scrape.js +37 -0
  47. package/dist/thread.d.ts +48 -0
  48. package/dist/thread.js +98 -0
  49. package/dist/uuid5.d.ts +31 -0
  50. package/dist/uuid5.js +64 -0
  51. package/dist/vendors/destinytool/index.d.ts +11 -0
  52. package/dist/vendors/destinytool/index.js +11 -0
  53. package/dist/vendors/destinytool/records.d.ts +118 -0
  54. package/dist/vendors/destinytool/records.js +266 -0
  55. package/dist/vendors/destinytool/scrape.d.ts +108 -0
  56. package/dist/vendors/destinytool/scrape.js +192 -0
  57. package/dist/vendors/kennametal/cad.d.ts +87 -0
  58. package/dist/vendors/kennametal/cad.js +119 -0
  59. package/dist/vendors/kennametal/index.d.ts +21 -0
  60. package/dist/vendors/kennametal/index.js +21 -0
  61. package/dist/vendors/kennametal/materials.d.ts +143 -0
  62. package/dist/vendors/kennametal/materials.js +200 -0
  63. package/dist/vendors/kennametal/records.d.ts +88 -0
  64. package/dist/vendors/kennametal/records.js +241 -0
  65. package/dist/vendors/kennametal/scrape.d.ts +111 -0
  66. package/dist/vendors/kennametal/scrape.js +226 -0
  67. package/dist/vendors/kennametal/thread-column.d.ts +28 -0
  68. package/dist/vendors/kennametal/thread-column.js +41 -0
  69. package/dist/vendors/regofix/index.d.ts +8 -0
  70. package/dist/vendors/regofix/index.js +8 -0
  71. package/dist/vendors/regofix/scrape.d.ts +237 -0
  72. package/dist/vendors/regofix/scrape.js +521 -0
  73. package/package.json +76 -0
@@ -0,0 +1,35 @@
1
+ /**
2
+ * The console entry point. Argv handling only — no scraping logic lives here.
3
+ *
4
+ * ```
5
+ * toolpath-scrape kennametal family page -> CSV
6
+ * toolpath-scrape regofix ProductFinder index -> toolholding CSV
7
+ * toolpath-scrape destinytool Firestore products -> End Mill CSV
8
+ * toolpath-scrape thread-pitch add the derived Thread Pitch column
9
+ * toolpath-scrape cad add the vendor CAD model column
10
+ * toolpath-scrape materials add the ISO workpiece-group column
11
+ * toolpath-scrape mirror-cad download the vendor STEP models
12
+ * ```
13
+ *
14
+ * **One binary with subcommands.** Seven names in `node_modules/.bin` for one
15
+ * package is not the idiom.
16
+ *
17
+ * **Every command prints the resolved scrape root before it does anything.**
18
+ * The default is derived from this package's own location, which is right in a
19
+ * working tree and meaningless in `node_modules`, so where a scrape lands is a
20
+ * thing to state rather than to assume.
21
+ *
22
+ * The convert commands are not here. This package acquires; a Fusion library
23
+ * or an assembly catalog is a different product, and none of it is in this
24
+ * tree.
25
+ */
26
+ import { type Fetcher } from '../fetch.js';
27
+ /** Everything the CLI prints, injectable so a test is not stdout. */
28
+ export interface Console_ {
29
+ log: (message: string) => void;
30
+ error: (message: string) => void;
31
+ }
32
+ /** Run one command. Exported so the tests drive it without a subprocess. */
33
+ export declare function run(argv: string[], io?: Console_, fetcher?: Fetcher): Promise<number>;
34
+ /** The process entry point: run, and turn a refusal into an exit code. */
35
+ export declare function main(argv?: string[]): Promise<number>;
@@ -0,0 +1,340 @@
1
+ /**
2
+ * The console entry point. Argv handling only — no scraping logic lives here.
3
+ *
4
+ * ```
5
+ * toolpath-scrape kennametal family page -> CSV
6
+ * toolpath-scrape regofix ProductFinder index -> toolholding CSV
7
+ * toolpath-scrape destinytool Firestore products -> End Mill CSV
8
+ * toolpath-scrape thread-pitch add the derived Thread Pitch column
9
+ * toolpath-scrape cad add the vendor CAD model column
10
+ * toolpath-scrape materials add the ISO workpiece-group column
11
+ * toolpath-scrape mirror-cad download the vendor STEP models
12
+ * ```
13
+ *
14
+ * **One binary with subcommands.** Seven names in `node_modules/.bin` for one
15
+ * package is not the idiom.
16
+ *
17
+ * **Every command prints the resolved scrape root before it does anything.**
18
+ * The default is derived from this package's own location, which is right in a
19
+ * working tree and meaningless in `node_modules`, so where a scrape lands is a
20
+ * thing to state rather than to assume.
21
+ *
22
+ * The convert commands are not here. This package acquires; a Fusion library
23
+ * or an assembly catalog is a different product, and none of it is in this
24
+ * tree.
25
+ */
26
+ import { readFileSync, writeFileSync } from 'node:fs';
27
+ import { basename } from 'node:path';
28
+ import { ScraperConfigError, VendorResponseError } from '../errors.js';
29
+ import { familyBrand } from '../family.js';
30
+ import { ALL_FAMILIES, FAMILIES, HOLDER_FAMILIES, familyConfig } from '../families/index.js';
31
+ import { createFetcher } from '../fetch.js';
32
+ import { AEM_BRANDS } from '../identity.js';
33
+ import { boundFamily } from '../registry.js';
34
+ import { mirrorFamilySteps } from './cad-mirror.js';
35
+ import { parseCsv, toCsv } from './csv.js';
36
+ import { describeRoot, familyCsv, stepDir } from './paths.js';
37
+ import * as receipts from './receipts.js';
38
+ import { scrapeFamily } from '../vendors/kennametal/scrape.js';
39
+ import { annotateCadUrls } from '../vendors/kennametal/cad.js';
40
+ import { addMaterialGroups, groupsByMaterial } from '../vendors/kennametal/materials.js';
41
+ import { addThreadPitch } from '../vendors/kennametal/thread-column.js';
42
+ import { scrapeEndMills, DOCUMENTS_URL } from '../vendors/destinytool/scrape.js';
43
+ import { SEARCH_URL, scrapeCollets, scrapeHolders } from '../vendors/regofix/scrape.js';
44
+ /**
45
+ * The PG series a BT 30 holder can take. PG 32 and PG 48 collets exist and no
46
+ * BT 30 holder in this catalog accepts one, so scraping them would add parts
47
+ * that fit nothing.
48
+ */
49
+ const BT30_COLLET_SIZES = ['6', '10', '15', '25'];
50
+ const USAGE = `usage: toolpath-scrape <command> [args]
51
+
52
+ kennametal [--brand kennametal|widia] FAMILY_CODE OUTPUT_CSV [Name=Value ...]
53
+ One AEM family page -> a CSV. Trailing Name=Value args are appended to
54
+ every row as constant columns, for facts the vendor table does not state
55
+ (e.g. "Thread System=metric").
56
+
57
+ regofix holders OUT.csv
58
+ regofix collets "<PRODUCT GROUP>" OUT.csv
59
+ The REGO-FIX ProductFinder index. \`holders\` takes every powRgrip BT/PG
60
+ holder whose taper is BT 30 or BT+ 30, with geometry from each part's
61
+ DIN 4000 document. \`collets\` takes one product group, restricted to PG
62
+ sizes ${BT30_COLLET_SIZES.join(', ')} — the group is the vendor's own
63
+ \`product_group_name\`, e.g. "Standard", "Coolant flush",
64
+ "Tapping collet TAP".
65
+
66
+ destinytool OUTPUT_CSV
67
+ Pages the whole Destiny Tool \`products\` Firestore collection and writes
68
+ every End Mill row.
69
+
70
+ thread-pitch TAP.csv [more.csv ...]
71
+ Adds a Thread Pitch column derived from D1-TDZ, in place. Safe to re-run.
72
+
73
+ cad HOLDERS.csv [more.csv ...]
74
+ Adds the vendor CAD model URL column, in place. One request per row
75
+ against product-config.net; safe to re-run.
76
+
77
+ materials FAMILY.csv [more.csv ...]
78
+ Adds the ISO workpiece-group column, in place. One request per material
79
+ group (32) per family; safe to re-run.
80
+
81
+ mirror-cad HOLDERS.csv [more.csv ...]
82
+ Downloads each row's STEP model into <root>/<brand>/step. Run \`cad\`
83
+ first — a CSV with no CAD column yields nothing and says so.
84
+
85
+ An output path is used verbatim; scraped CSVs belong under the scrape root,
86
+ in <brand>/csv/. The in-place commands take a bare CSV name and resolve it
87
+ through the family's own brand.`;
88
+ const STDOUT = {
89
+ log: (message) => process.stdout.write(`${message}\n`),
90
+ error: (message) => process.stderr.write(`${message}\n`),
91
+ };
92
+ /** Read one family's CSV off disk as a scrape result. */
93
+ function readCsv(name, source) {
94
+ const path = familyCsv(name);
95
+ const { header, rows } = parseCsv(readFileSync(path, 'utf8'));
96
+ return { header, rows, source, familyCode: null };
97
+ }
98
+ /** Write a scrape to `path`. */
99
+ function writeCsv(path, scrape) {
100
+ writeFileSync(path, toCsv(scrape.header, scrape.rows));
101
+ }
102
+ /**
103
+ * Report a scrape and record its receipt.
104
+ *
105
+ * One function because the two belong together: a scrape that reported a count
106
+ * and wrote no receipt would be exactly the state this package is trying to
107
+ * stop existing — data with nothing saying where it came from.
108
+ */
109
+ function wrote(out, brand, scrape, io) {
110
+ writeCsv(out, scrape);
111
+ const receipt = receipts.write(out, {
112
+ brand,
113
+ source: scrape.source,
114
+ rows: scrape.rows.length,
115
+ familyCode: scrape.familyCode,
116
+ });
117
+ io.log(`wrote ${scrape.rows.length} rows to ${out}`);
118
+ io.log(` receipt: ${basename(receipt)}`);
119
+ const name = basename(out);
120
+ // Through the merged table rather than three chained lookups, so a name two
121
+ // tables both claim is refused where it is built instead of resolving here
122
+ // to whichever happened to be checked first.
123
+ const declared = ALL_FAMILIES[name]?.rows;
124
+ const written = receipts.read(out);
125
+ if (declared !== undefined && written !== null) {
126
+ receipts.checkRows(name, declared, written);
127
+ }
128
+ }
129
+ /**
130
+ * Argv as family CSV names, refusing anything unknown by name.
131
+ *
132
+ * A path's directory is ignored: the family's own brand decides where its CSV
133
+ * lives, and honouring a typed directory would let one vendor's receipt be
134
+ * written into another's.
135
+ */
136
+ function namesIn(argv, known, what) {
137
+ const names = argv.map((a) => basename(a));
138
+ const unknown = names.filter((n) => !Object.hasOwn(known, n));
139
+ if (unknown.length > 0) {
140
+ throw new ScraperConfigError(unknown.join(', '), `unknown ${what} CSV (known: ${Object.keys(known).sort().join(', ')})`);
141
+ }
142
+ return names;
143
+ }
144
+ /** Run one command. Exported so the tests drive it without a subprocess. */
145
+ export async function run(argv, io = STDOUT, fetcher = createFetcher()) {
146
+ // Somebody reading the usage text is the person most likely to be about to
147
+ // point a scrape at the wrong place, so help gets the root too.
148
+ io.log(describeRoot());
149
+ const [command, ...rest] = argv;
150
+ if (command === undefined || command === '-h' || command === '--help') {
151
+ io.log(USAGE);
152
+ return 0;
153
+ }
154
+ switch (command) {
155
+ case 'kennametal':
156
+ return kennametal(rest, io, fetcher);
157
+ case 'regofix':
158
+ return regofix(rest, io, fetcher);
159
+ case 'destinytool':
160
+ return destinytool(rest, io, fetcher);
161
+ case 'thread-pitch':
162
+ return threadPitch(rest, io);
163
+ case 'cad':
164
+ return cad(rest, io, fetcher);
165
+ case 'materials':
166
+ return materials(rest, io, fetcher);
167
+ case 'mirror-cad':
168
+ return mirrorCad(rest, io, fetcher);
169
+ default:
170
+ io.error(`unknown command ${JSON.stringify(command)}\n\n${USAGE}`);
171
+ return 2;
172
+ }
173
+ }
174
+ async function kennametal(argv, io, fetcher) {
175
+ const args = [...argv];
176
+ let brand = 'kennametal';
177
+ const flag = args.indexOf('--brand');
178
+ if (flag !== -1) {
179
+ const value = args[flag + 1];
180
+ if (value === undefined) {
181
+ io.error(`--brand needs a value\n\n${USAGE}`);
182
+ return 2;
183
+ }
184
+ brand = value;
185
+ args.splice(flag, 2);
186
+ }
187
+ // Against the AEM brands rather than every brand: `scrapeFamily` reads
188
+ // `Brand.node`, and a brand without one built a URL with `undefined` in the
189
+ // path and died on the 404 instead of on this line.
190
+ if (!AEM_BRANDS.includes(brand)) {
191
+ io.error(`unknown brand: ${brand} (known: ${[...AEM_BRANDS].sort().join(', ')})`);
192
+ return 2;
193
+ }
194
+ if (args.length < 2) {
195
+ io.error(USAGE);
196
+ return 2;
197
+ }
198
+ const [code, out] = args;
199
+ // Refused rather than dropped: a quoting slip like `"Thread System" metric`
200
+ // used to scrape, write the CSV and exit 0 with the column missing, and the
201
+ // failure surfaced later in `addThreadPitch` against a file no longer being
202
+ // written.
203
+ const constants = args.slice(2);
204
+ const malformed = constants.find((a) => a.indexOf('=') < 1);
205
+ if (malformed !== undefined) {
206
+ io.error(`constant column ${JSON.stringify(malformed)} is not Name=Value\n\n${USAGE}`);
207
+ return 2;
208
+ }
209
+ const tags = constants.map((a) => {
210
+ const at = a.indexOf('=');
211
+ return [a.slice(0, at), a.slice(at + 1)];
212
+ });
213
+ const scrape = await scrapeFamily(fetcher, code, brand, tags);
214
+ wrote(out, brand, scrape, io);
215
+ return 0;
216
+ }
217
+ async function regofix(argv, io, fetcher) {
218
+ const [what, ...rest] = argv;
219
+ if (what === 'holders') {
220
+ if (rest.length !== 1) {
221
+ io.error(USAGE);
222
+ return 2;
223
+ }
224
+ const scrape = await scrapeHolders(fetcher, 'BT/PG', 'BT', {
225
+ warn: io.error,
226
+ });
227
+ wrote(rest[0], 'regofix', { ...scrape, source: SEARCH_URL }, io);
228
+ return 0;
229
+ }
230
+ if (what === 'collets') {
231
+ if (rest.length !== 2) {
232
+ io.error(USAGE);
233
+ return 2;
234
+ }
235
+ const [group, out] = rest;
236
+ const scrape = await scrapeCollets(fetcher, group, BT30_COLLET_SIZES, {
237
+ warn: io.error,
238
+ });
239
+ wrote(out, 'regofix', { ...scrape, source: SEARCH_URL }, io);
240
+ return 0;
241
+ }
242
+ io.error(`unknown subcommand ${JSON.stringify(what)}\n\n${USAGE}`);
243
+ return 2;
244
+ }
245
+ async function destinytool(argv, io, fetcher) {
246
+ if (argv.length !== 1) {
247
+ io.error(USAGE);
248
+ return 2;
249
+ }
250
+ const scrape = await scrapeEndMills(fetcher);
251
+ wrote(argv[0], 'destinytool', { ...scrape, source: DOCUMENTS_URL }, io);
252
+ return 0;
253
+ }
254
+ function threadPitch(argv, io) {
255
+ if (argv.length === 0) {
256
+ io.error(USAGE);
257
+ return 2;
258
+ }
259
+ for (const name of namesIn(argv, FAMILIES, 'family')) {
260
+ const path = familyCsv(name);
261
+ const updated = addThreadPitch(readCsv(name, path));
262
+ writeCsv(path, updated);
263
+ io.log(`${name}: ${updated.rows.length} rows updated`);
264
+ }
265
+ return 0;
266
+ }
267
+ async function cad(argv, io, fetcher) {
268
+ if (argv.length === 0) {
269
+ io.error(USAGE);
270
+ return 2;
271
+ }
272
+ for (const name of namesIn(argv, HOLDER_FAMILIES, 'holder')) {
273
+ // `annotateCadUrls` is Kennametal's CDS lookup, not a vendor-neutral one —
274
+ // it queries product-config.net and rewrites `CAD_COLUMN` on every row. Run
275
+ // against a REGO-FIX holder it would post that vendor's SKUs to Kennametal
276
+ // and blank the STEP URLs the REGO-FIX scrape had already filled in.
277
+ // `mirror-cad` reads the column and *is* neutral; this writes it and is not.
278
+ const brand = familyBrand(familyConfig(name));
279
+ if (!AEM_BRANDS.includes(brand)) {
280
+ io.error(`${name}: the cad step is ${[...AEM_BRANDS].sort().join('/')}-only — ` +
281
+ `${brand} publishes its own CAD URLs with the scrape`);
282
+ return 2;
283
+ }
284
+ const path = familyCsv(name);
285
+ const { scrape, found } = await annotateCadUrls(fetcher, readCsv(name, path));
286
+ writeCsv(path, scrape);
287
+ io.log(`${name}: ${found} CAD models`);
288
+ }
289
+ return 0;
290
+ }
291
+ async function materials(argv, io, fetcher) {
292
+ if (argv.length === 0) {
293
+ io.error(USAGE);
294
+ return 2;
295
+ }
296
+ // The family code and brand come from config, so a re-run needs neither
297
+ // typed again.
298
+ for (const name of namesIn(argv, FAMILIES, 'family')) {
299
+ const cfg = boundFamily(name);
300
+ if (cfg.familyCode === undefined) {
301
+ throw new ScraperConfigError(name, 'has no familyCode — the material sweep re-queries the family page ' +
302
+ 'and cannot without one');
303
+ }
304
+ const path = familyCsv(name);
305
+ const found = await groupsByMaterial(fetcher, cfg.familyCode, {
306
+ brand: (cfg.brand ?? 'kennametal'),
307
+ });
308
+ const { scrape, matched } = addMaterialGroups(readCsv(name, path), found);
309
+ writeCsv(path, scrape);
310
+ io.log(`${name}: ${matched} rows with a material group`);
311
+ }
312
+ return 0;
313
+ }
314
+ async function mirrorCad(argv, io, fetcher) {
315
+ if (argv.length === 0) {
316
+ io.error(USAGE);
317
+ return 2;
318
+ }
319
+ for (const name of namesIn(argv, HOLDER_FAMILIES, 'holder')) {
320
+ const path = familyCsv(name);
321
+ const brand = familyBrand(familyConfig(name));
322
+ const written = await mirrorFamilySteps(fetcher, readCsv(name, path).rows, stepDir(brand), undefined, io.error);
323
+ const total = written.reduce((sum, f) => sum + f.bytes, 0);
324
+ io.log(`${name}: ${written.length} STEP files, ${Math.floor(total / 1024)} KB`);
325
+ }
326
+ return 0;
327
+ }
328
+ /** The process entry point: run, and turn a refusal into an exit code. */
329
+ export async function main(argv = process.argv.slice(2)) {
330
+ try {
331
+ return await run(argv);
332
+ }
333
+ catch (error) {
334
+ if (error instanceof ScraperConfigError || error instanceof VendorResponseError) {
335
+ STDOUT.error(error.message);
336
+ return 2;
337
+ }
338
+ throw error;
339
+ }
340
+ }
@@ -0,0 +1,47 @@
1
+ /**
2
+ * CSV, to the letter of what Python's `csv` module writes.
3
+ *
4
+ * Node has no CSV in its standard library and this package needs about sixty
5
+ * lines of one, so it is here rather than a dependency — the same call
6
+ * `uuid5.ts` makes, and for the same reason: a dependency whose whole surface
7
+ * is two functions is a supply chain for no gain.
8
+ *
9
+ * **Byte-compatible with the Python this replaces, deliberately.** Scraped
10
+ * CSVs already exist on the machines that keep a corpus, and a re-scrape has
11
+ * to produce the same file or every diff is noise. That means three details
12
+ * that are not the obvious defaults:
13
+ *
14
+ * 1. **`\r\n` line endings.** Python's `csv.writer` uses them regardless of
15
+ * platform; `open(path, 'w', newline='')` is what stops the runtime
16
+ * translating them, not a choice to emit `\n`.
17
+ * 2. **Minimal quoting.** A field is quoted only when it contains a comma, a
18
+ * quote or a newline — `QUOTE_MINIMAL`, Python's default.
19
+ * 3. **Doubled quotes**, not backslash escapes.
20
+ *
21
+ * The reader accepts either line ending, because a CSV that has been through
22
+ * an editor is still a CSV.
23
+ */
24
+ import type { ScrapedRow } from '../scrape.js';
25
+ /**
26
+ * Rows to CSV text, in `header` order.
27
+ *
28
+ * A row missing one of the header's columns writes an empty cell rather than
29
+ * failing: a mixed-unit collet family has `D1_mm` on its metric rows and
30
+ * `D1_in` on its inch ones, and both columns are in the union header.
31
+ */
32
+ export declare function toCsv(header: readonly string[], rows: readonly ScrapedRow[]): string;
33
+ /** What {@link parseCsv} answers with. */
34
+ export interface ParsedCsv {
35
+ header: string[];
36
+ rows: ScrapedRow[];
37
+ }
38
+ /**
39
+ * CSV text back into a header and rows.
40
+ *
41
+ * A row longer than the header keeps its extra cells under no name, and a
42
+ * shorter one leaves the missing columns empty — the same shape Python's
43
+ * `DictReader` produces, and the reason neither is an error is that these
44
+ * files are re-read by the annotate steps, which have to be able to say what
45
+ * changed rather than refuse the file.
46
+ */
47
+ export declare function parseCsv(text: string): ParsedCsv;
@@ -0,0 +1,123 @@
1
+ /**
2
+ * CSV, to the letter of what Python's `csv` module writes.
3
+ *
4
+ * Node has no CSV in its standard library and this package needs about sixty
5
+ * lines of one, so it is here rather than a dependency — the same call
6
+ * `uuid5.ts` makes, and for the same reason: a dependency whose whole surface
7
+ * is two functions is a supply chain for no gain.
8
+ *
9
+ * **Byte-compatible with the Python this replaces, deliberately.** Scraped
10
+ * CSVs already exist on the machines that keep a corpus, and a re-scrape has
11
+ * to produce the same file or every diff is noise. That means three details
12
+ * that are not the obvious defaults:
13
+ *
14
+ * 1. **`\r\n` line endings.** Python's `csv.writer` uses them regardless of
15
+ * platform; `open(path, 'w', newline='')` is what stops the runtime
16
+ * translating them, not a choice to emit `\n`.
17
+ * 2. **Minimal quoting.** A field is quoted only when it contains a comma, a
18
+ * quote or a newline — `QUOTE_MINIMAL`, Python's default.
19
+ * 3. **Doubled quotes**, not backslash escapes.
20
+ *
21
+ * The reader accepts either line ending, because a CSV that has been through
22
+ * an editor is still a CSV.
23
+ */
24
+ const NEEDS_QUOTING = /[",\r\n]/;
25
+ /** One field, quoted only if it has to be. */
26
+ function field(value) {
27
+ return NEEDS_QUOTING.test(value) ? `"${value.replaceAll('"', '""')}"` : value;
28
+ }
29
+ /**
30
+ * Rows to CSV text, in `header` order.
31
+ *
32
+ * A row missing one of the header's columns writes an empty cell rather than
33
+ * failing: a mixed-unit collet family has `D1_mm` on its metric rows and
34
+ * `D1_in` on its inch ones, and both columns are in the union header.
35
+ */
36
+ export function toCsv(header, rows) {
37
+ const lines = [header.map(field).join(',')];
38
+ for (const row of rows) {
39
+ lines.push(header.map((column) => field(row[column] ?? '')).join(','));
40
+ }
41
+ return `${lines.join('\r\n')}\r\n`;
42
+ }
43
+ /**
44
+ * CSV text back into a header and rows.
45
+ *
46
+ * A row longer than the header keeps its extra cells under no name, and a
47
+ * shorter one leaves the missing columns empty — the same shape Python's
48
+ * `DictReader` produces, and the reason neither is an error is that these
49
+ * files are re-read by the annotate steps, which have to be able to say what
50
+ * changed rather than refuse the file.
51
+ */
52
+ export function parseCsv(text) {
53
+ const records = parseRecords(text);
54
+ const header = records.shift() ?? [];
55
+ const rows = records.map((cells) => {
56
+ const row = {};
57
+ header.forEach((column, index) => {
58
+ row[column] = cells[index] ?? '';
59
+ });
60
+ return row;
61
+ });
62
+ return { header, rows };
63
+ }
64
+ /** The raw grid, before a header is applied. */
65
+ function parseRecords(text) {
66
+ const records = [];
67
+ let record = [];
68
+ let cell = '';
69
+ let quoted = false;
70
+ let started = false;
71
+ const endCell = () => {
72
+ record.push(cell);
73
+ cell = '';
74
+ };
75
+ const endRecord = () => {
76
+ endCell();
77
+ records.push(record);
78
+ record = [];
79
+ started = false;
80
+ };
81
+ for (let i = 0; i < text.length; i += 1) {
82
+ const char = text[i];
83
+ if (quoted) {
84
+ if (char === '"') {
85
+ if (text[i + 1] === '"') {
86
+ cell += '"';
87
+ i += 1;
88
+ }
89
+ else {
90
+ quoted = false;
91
+ }
92
+ }
93
+ else {
94
+ cell += char;
95
+ }
96
+ continue;
97
+ }
98
+ if (char === '"' && cell === '') {
99
+ quoted = true;
100
+ started = true;
101
+ }
102
+ else if (char === ',') {
103
+ endCell();
104
+ started = true;
105
+ }
106
+ else if (char === '\r' || char === '\n') {
107
+ if (char === '\r' && text[i + 1] === '\n')
108
+ i += 1;
109
+ // A blank line between records is not a record. Python's reader skips
110
+ // it, and a trailing newline would otherwise add a row of one empty cell
111
+ // to every file this package writes.
112
+ if (started || cell !== '' || record.length > 0)
113
+ endRecord();
114
+ }
115
+ else {
116
+ cell += char;
117
+ started = true;
118
+ }
119
+ }
120
+ if (started || cell !== '' || record.length > 0)
121
+ endRecord();
122
+ return records;
123
+ }
@@ -0,0 +1,16 @@
1
+ /**
2
+ * The half of this package that touches the filesystem.
3
+ *
4
+ * A scrape returns rows; turning them into a CSV, writing the provenance
5
+ * sidecar that goes beside one, resolving where a vendor's files live and
6
+ * mirroring the vendor's CAD binaries all need `fs`, and a backend that only
7
+ * wants records should not have to import any of it.
8
+ *
9
+ * So they are a separate entry point — `@toolpath/tool-scraper/node` — and the
10
+ * main one stays what a library ought to be: functions in, values out.
11
+ */
12
+ export * from './cad-mirror.js';
13
+ export * from './cli.js';
14
+ export * from './csv.js';
15
+ export * from './paths.js';
16
+ export * from './receipts.js';
@@ -0,0 +1,16 @@
1
+ /**
2
+ * The half of this package that touches the filesystem.
3
+ *
4
+ * A scrape returns rows; turning them into a CSV, writing the provenance
5
+ * sidecar that goes beside one, resolving where a vendor's files live and
6
+ * mirroring the vendor's CAD binaries all need `fs`, and a backend that only
7
+ * wants records should not have to import any of it.
8
+ *
9
+ * So they are a separate entry point — `@toolpath/tool-scraper/node` — and the
10
+ * main one stays what a library ought to be: functions in, values out.
11
+ */
12
+ export * from './cad-mirror.js';
13
+ export * from './cli.js';
14
+ export * from './csv.js';
15
+ export * from './paths.js';
16
+ export * from './receipts.js';
@@ -0,0 +1,13 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * The executable. Nothing but the shebang and the call.
4
+ *
5
+ * Separate from `cli.ts` because that module is also imported —
6
+ * `@toolpath/tool-scraper/node` re-exports `run` so a consumer can drive a
7
+ * command without a subprocess, and a top-level "if this is the entry point"
8
+ * guard in an imported module is both fragile and a side effect. It was fragile
9
+ * here in exactly the way that matters: run through the `bin` symlink,
10
+ * `process.argv[1]` is the symlink's name and the guard never matched, so the
11
+ * installed command printed nothing and exited 0.
12
+ */
13
+ export {};
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * The executable. Nothing but the shebang and the call.
4
+ *
5
+ * Separate from `cli.ts` because that module is also imported —
6
+ * `@toolpath/tool-scraper/node` re-exports `run` so a consumer can drive a
7
+ * command without a subprocess, and a top-level "if this is the entry point"
8
+ * guard in an imported module is both fragile and a side effect. It was fragile
9
+ * here in exactly the way that matters: run through the `bin` symlink,
10
+ * `process.argv[1]` is the symlink's name and the guard never matched, so the
11
+ * installed command printed nothing and exited 0.
12
+ */
13
+ import { main } from './cli.js';
14
+ process.exitCode = await main();
@@ -0,0 +1,60 @@
1
+ /**
2
+ * Where a scrape lands.
3
+ *
4
+ * **Scraped output is never committed.** A CSV is a vendor's data and a
5
+ * working file, not source, and this repository is public — which is a second
6
+ * reason, independent of size, to keep it out. Git was carrying the provenance
7
+ * of those CSVs for free; now that it is not, every scrape writes a
8
+ * {@link receipts} sidecar beside its file.
9
+ *
10
+ * This module is in `node/` rather than beside the config tables: resolving a
11
+ * root needs `process.env` and a path relative to this file's own location,
12
+ * and a family table that needed either would be unimportable in anything that
13
+ * only wants to read records.
14
+ */
15
+ import type { BrandName } from '../identity.js';
16
+ /**
17
+ * Where scraped CSVs are read from and written to.
18
+ *
19
+ * Set it when the package is installed rather than run from a checkout: the
20
+ * default below is derived from this file's own location, which is right in a
21
+ * working tree and meaningless inside `node_modules`. Every command prints the
22
+ * resolved root for exactly that reason — a scrape that wrote somewhere
23
+ * surprising should say so on the way, not be discovered later.
24
+ */
25
+ export declare const SCRAPE_ROOT_ENV = "TOOLPATH_SCRAPE_ROOT";
26
+ /** `packages/tool-scraper/scrape-out`, which `.gitignore` already covers. */
27
+ export declare const DEFAULT_SCRAPE_ROOT: string;
28
+ /** The directory holding every vendor's scraped CSVs. */
29
+ export declare function scrapeRoot(): string;
30
+ /**
31
+ * One line naming the resolved root and how it was resolved.
32
+ *
33
+ * Printed by every command. The distinction it carries is the one that matters
34
+ * when a scrape goes somewhere unexpected: whether the path came from the
35
+ * environment or from this package's own location.
36
+ */
37
+ export declare function describeRoot(): string;
38
+ /**
39
+ * Where one vendor's scraped CSVs live — the receipts.
40
+ *
41
+ * Per brand rather than per adapter, and the distinction is worth holding on
42
+ * to: an adapter is a fact about *code*, a scraped table is a fact about who
43
+ * published it. WIDIA's tables are WIDIA's even though Kennametal's adapter is
44
+ * what fetched them.
45
+ */
46
+ export declare function csvDir(brand: BrandName): string;
47
+ /**
48
+ * One vendor's mirrored STEP models.
49
+ *
50
+ * Nothing is redistributed from here: these are a local working copy for
51
+ * measuring a holder, and only a derived profile is ever meant to leave.
52
+ */
53
+ export declare function stepDir(brand: BrandName): string;
54
+ /**
55
+ * Where one family's CSV lives, resolved through its own brand.
56
+ *
57
+ * Takes a bare CSV name rather than a path, so a caller cannot pass a file
58
+ * from somewhere else and have it silently treated as this family's receipt.
59
+ */
60
+ export declare function familyCsv(name: string): string;