@toolpath/tool-scraper 0.1.0 → 2.0.0

package/dist/measure.d.ts

@@ -0,0 +1,112 @@
+/**
+ * Reading a machinist's number, and the one constant between the two systems.
+ *
+ * Three vendors publish dimensions the way a printed catalog does — `.250`,
+ * `3/4`, `1-1/2` — and until this module each adapter had its own reader for
+ * that grammar. They disagreed on real inputs: REGO-FIX's `parseSize` handled
+ * no mixed number at all, so `1-1/2` reached `Number()` as `NaN` and was
+ * refused; Destiny Tool's `parseFractionInches` branched on a `.` first, so
+ * `1.5-1/2` was refused there and read as 2 by Harvey's. And they disagreed on
+ * *how* they refused — two threw `RangeError`, one answered `null`.
+ *
+ * That is the same shape `fetch.createFetcher` and `scrape.pause` were pulled
+ * out for, and the same argument: four copies of four lines is not expensive,
+ * four copies of the **decisions** in them is, because the decisions are the
+ * part that has to be consistent. A dimension string is the last thing in this
+ * package that should mean two things.
+ *
+ * ## The reader answers `null`; the caller decides what that costs
+ *
+ * {@link fractionValue} is the primitive, and it never throws: whether an
+ * unreadable cell is a skipped row, a warning or a refusal is a vendor's call,
+ * and every adapter here already makes it differently for good reasons. Harvey
+ * skips a `-` cell because the column does not apply to that row; REGO-FIX
+ * refuses, because a collet with no size is not a part. Pushing that decision
+ * into the reader would take it away from the module that knows.
+ *
+ * ## The two decisions about a *read* cell live here too
+ *
+ * {@link fractionValue} answers what a token says. {@link asLength} and
+ * {@link asCount} answer what that costs, and they are core for the same reason
+ * the grammar is: a cell stating `mm` in a family published in inches is
+ * converted-and-warned, and a cell stating an angle in a length column is
+ * refused, and those two calls have to be the same call whoever is being
+ * scraped. They were a verbatim copy in `vendors/harvey/value.ts` and
+ * `vendors/emuge/value.ts` — down to the wording of both warnings — which is
+ * two operators reading the same sentence out of two files that were free to
+ * drift apart.
+ *
+ * A vendor's own reader still owns the **grammar**, because that genuinely
+ * differs: Harvey writes a mixed number `1-1/2` and annotates it `(1/8)`, EMUGE
+ * writes `1 1/2` and admits no hyphen at all. What it hands here is
+ * {@link Measured}, which is the part they have in common.
+ *
+ * **`MM_PER_INCH` lives here and not in a vendor.** It was declared and
+ * exported twice — `vendors/harvey/value.ts` and `vendors/regofix/scrape.ts`
+ * — so `@toolpath/tool-scraper/vendors/harvey` and `.../vendors/regofix` each
+ * published their own copy of 25.4. That is exactly what `conventions.CAD_COLUMN`
+ * was moved up for, and what `tests/vendor-boundary.test.ts` now refuses by name.
+ */
+import type { UnitSystem } from './conventions.js';
+import { type Warn } from './scrape.js';
+/** Exact by definition: the inch has been 25.4 mm since 1959. */
+export declare const MM_PER_INCH = 25.4;
+/**
+ * `.250` -> 0.25, `3/4` -> 0.75, `1-1/2` -> 1.5, `2` -> 2.
+ *
+ * Null where the token is not one of those four shapes — including a division
+ * by zero, which arrives finite-looking as `Infinity` and would otherwise
+ * travel into a row as a dimension.
+ *
+ * Unit-free on purpose: what system the number is in is the caller's, from the
+ * family's declared `unit` or from a unit the cell states outright. A reader
+ * that guessed would be the mistake `conventions.UNIT_SUFFIX` exists to prevent.
+ */
+export declare function fractionValue(token: string): number | null;
+/** `value`, converted from `from` to `to`. A no-op when they agree. */
+export declare function convertLength(value: number, from: UnitSystem, to: UnitSystem): number;
+/**
+ * What a value states about itself: a length system, or degrees.
+ *
+ * Degrees are inside this union rather than beside it because a cell states one
+ * thing about itself — "this is an angle" competes with "this is millimetres"
+ * and is never both. `vendors/harvey/value.ts` carries the same fact as a
+ * separate `degrees` boolean, which is that vendor's published shape and stays.
+ */
+export type StatedUnit = UnitSystem | 'degrees';
+/**
+ * One cell as a vendor's own reader left it.
+ *
+ * The part every vendor's reader has in common, and all {@link asLength} and
+ * {@link asCount} need. A reader is free to answer more — Harvey's also carries
+ * the parenthesised annotation, the `(1.5x)` ratio and the non-numeric code —
+ * and anything with these two fields is accepted here.
+ */
+export interface Measured {
+    /** The number the cell states, in {@link Measured.stated}. Null where none. */
+    readonly value: number | null;
+    /** The unit the cell names outright. Null where it names none. */
+    readonly stated: StatedUnit | null;
+}
+/**
+ * A read cell as a length in `unit`, or null where it publishes none.
+ *
+ * A cell that states its own unit and disagrees with the family's is
+ * **converted and warned about** rather than refused: the value is right, the
+ * column's suffix is right, and the parts this happens on are real tools
+ * somebody can order. Refusing would drop them; trusting the column's unit
+ * would publish a 3-inch shank.
+ *
+ * An angle reaching a dimensional column is **refused**, because that is a
+ * header that has moved rather than a value that is odd.
+ *
+ * `display` is passed beside `parsed` only so the warnings can quote the cell
+ * the vendor really wrote, which is the part an operator needs to go look at.
+ */
+export declare function asLength(parsed: Measured, display: string, unit: UnitSystem, what: string, warn?: Warn): number | null;
+/**
+ * A read cell as a whole count — a flute or tooth number. Null where there is
+ * none, and null for a fraction, which is a column that has moved rather than a
+ * tool with two and a half flutes.
+ */
+export declare function asCount(parsed: Measured): number | null;

package/dist/measure.js

@@ -0,0 +1,130 @@
+/**
+ * Reading a machinist's number, and the one constant between the two systems.
+ *
+ * Three vendors publish dimensions the way a printed catalog does — `.250`,
+ * `3/4`, `1-1/2` — and until this module each adapter had its own reader for
+ * that grammar. They disagreed on real inputs: REGO-FIX's `parseSize` handled
+ * no mixed number at all, so `1-1/2` reached `Number()` as `NaN` and was
+ * refused; Destiny Tool's `parseFractionInches` branched on a `.` first, so
+ * `1.5-1/2` was refused there and read as 2 by Harvey's. And they disagreed on
+ * *how* they refused — two threw `RangeError`, one answered `null`.
+ *
+ * That is the same shape `fetch.createFetcher` and `scrape.pause` were pulled
+ * out for, and the same argument: four copies of four lines is not expensive,
+ * four copies of the **decisions** in them is, because the decisions are the
+ * part that has to be consistent. A dimension string is the last thing in this
+ * package that should mean two things.
+ *
+ * ## The reader answers `null`; the caller decides what that costs
+ *
+ * {@link fractionValue} is the primitive, and it never throws: whether an
+ * unreadable cell is a skipped row, a warning or a refusal is a vendor's call,
+ * and every adapter here already makes it differently for good reasons. Harvey
+ * skips a `-` cell because the column does not apply to that row; REGO-FIX
+ * refuses, because a collet with no size is not a part. Pushing that decision
+ * into the reader would take it away from the module that knows.
+ *
+ * ## The two decisions about a *read* cell live here too
+ *
+ * {@link fractionValue} answers what a token says. {@link asLength} and
+ * {@link asCount} answer what that costs, and they are core for the same reason
+ * the grammar is: a cell stating `mm` in a family published in inches is
+ * converted-and-warned, and a cell stating an angle in a length column is
+ * refused, and those two calls have to be the same call whoever is being
+ * scraped. They were a verbatim copy in `vendors/harvey/value.ts` and
+ * `vendors/emuge/value.ts` — down to the wording of both warnings — which is
+ * two operators reading the same sentence out of two files that were free to
+ * drift apart.
+ *
+ * A vendor's own reader still owns the **grammar**, because that genuinely
+ * differs: Harvey writes a mixed number `1-1/2` and annotates it `(1/8)`, EMUGE
+ * writes `1 1/2` and admits no hyphen at all. What it hands here is
+ * {@link Measured}, which is the part they have in common.
+ *
+ * **`MM_PER_INCH` lives here and not in a vendor.** It was declared and
+ * exported twice — `vendors/harvey/value.ts` and `vendors/regofix/scrape.ts`
+ * — so `@toolpath/tool-scraper/vendors/harvey` and `.../vendors/regofix` each
+ * published their own copy of 25.4. That is exactly what `conventions.CAD_COLUMN`
+ * was moved up for, and what `tests/vendor-boundary.test.ts` now refuses by name.
+ */
+import { consoleWarn } from './scrape.js';
+/** Exact by definition: the inch has been 25.4 mm since 1959. */
+export const MM_PER_INCH = 25.4;
+/**
+ * A decimal, a simple fraction or a mixed number — and nothing else.
+ *
+ * Deliberately strict, because it is used without a pre-guard. The whole-number
+ * part of a mixed number is `\d+` rather than a decimal so that a **range** like
+ * `.035-.040` — which Destiny Tool really does publish, in description text —
+ * cannot parse as `0.035 + 0.040`. A range is not a measurement, and reading one
+ * as a sum is the kind of quiet wrong number this package exists to avoid.
+ */
+const NUMERIC = /^(?:(\d+)-)?(\d*\.?\d+)(?:\/(\d*\.?\d+))?$/;
+/**
+ * `.250` -> 0.25, `3/4` -> 0.75, `1-1/2` -> 1.5, `2` -> 2.
+ *
+ * Null where the token is not one of those four shapes — including a division
+ * by zero, which arrives finite-looking as `Infinity` and would otherwise
+ * travel into a row as a dimension.
+ *
+ * Unit-free on purpose: what system the number is in is the caller's, from the
+ * family's declared `unit` or from a unit the cell states outright. A reader
+ * that guessed would be the mistake `conventions.UNIT_SUFFIX` exists to prevent.
+ */
+export function fractionValue(token) {
+    const parsed = NUMERIC.exec(token.trim());
+    if (parsed === null)
+        return null;
+    const [, whole, numerator, denominator] = parsed;
+    const part = denominator === undefined ? Number(numerator) : Number(numerator) / Number(denominator);
+    const value = (whole === undefined ? 0 : Number(whole)) + part;
+    return Number.isFinite(value) ? value : null;
+}
+/** `value`, converted from `from` to `to`. A no-op when they agree. */
+export function convertLength(value, from, to) {
+    if (from === to)
+        return value;
+    return to === 'inches' ? value / MM_PER_INCH : value * MM_PER_INCH;
+}
+/**
+ * A read cell as a length in `unit`, or null where it publishes none.
+ *
+ * A cell that states its own unit and disagrees with the family's is
+ * **converted and warned about** rather than refused: the value is right, the
+ * column's suffix is right, and the parts this happens on are real tools
+ * somebody can order. Refusing would drop them; trusting the column's unit
+ * would publish a 3-inch shank.
+ *
+ * An angle reaching a dimensional column is **refused**, because that is a
+ * header that has moved rather than a value that is odd.
+ *
+ * `display` is passed beside `parsed` only so the warnings can quote the cell
+ * the vendor really wrote, which is the part an operator needs to go look at.
+ */
+export function asLength(parsed, display, unit, what, warn = consoleWarn) {
+    const { value, stated } = parsed;
+    if (value === null)
+        return null;
+    if (stated === 'degrees') {
+        warn(`  WARNING: ${what}: ${JSON.stringify(display)} is an angle in a column ` +
+            `read as a length — skipped`);
+        return null;
+    }
+    if (stated !== null && stated !== unit) {
+        warn(`  WARNING: ${what}: ${JSON.stringify(display)} states ${stated} in a ` +
+            `family published in ${unit} — converted`);
+        return convertLength(value, stated, unit);
+    }
+    return value;
+}
+/**
+ * A read cell as a whole count — a flute or tooth number. Null where there is
+ * none, and null for a fraction, which is a column that has moved rather than a
+ * tool with two and a half flutes.
+ */
+export function asCount(parsed) {
+    const { value } = parsed;
+    if (value === null || !Number.isInteger(value))
+        return null;
+    return value;
+}

package/dist/node/cli.d.ts

@@ -5,6 +5,9 @@
  * toolpath-scrape kennametal      family page -> CSV
  * toolpath-scrape regofix         ProductFinder index -> toolholding CSV
  * toolpath-scrape destinytool     Firestore products -> End Mill CSV
+ * toolpath-scrape harvey          one product page -> CSV
+ * toolpath-scrape maritool        leaf categories -> toolholding CSV
+ * toolpath-scrape emuge           one catalog category -> CSV
  * toolpath-scrape thread-pitch    add the derived Thread Pitch column
  * toolpath-scrape cad             add the vendor CAD model column
  * toolpath-scrape materials       add the ISO workpiece-group column

package/dist/node/cli.js

@@ -5,6 +5,9 @@
  * toolpath-scrape kennametal      family page -> CSV
  * toolpath-scrape regofix         ProductFinder index -> toolholding CSV
  * toolpath-scrape destinytool     Firestore products -> End Mill CSV
+ * toolpath-scrape harvey          one product page -> CSV
+ * toolpath-scrape maritool        leaf categories -> toolholding CSV
+ * toolpath-scrape emuge           one catalog category -> CSV
  * toolpath-scrape thread-pitch    add the derived Thread Pitch column
  * toolpath-scrape cad             add the vendor CAD model column
  * toolpath-scrape materials       add the ISO workpiece-group column
@@ -23,14 +26,15 @@
  * or an assembly catalog is a different product, and none of it is in this
  * tree.
  */
-import { readFileSync, writeFileSync } from 'node:fs';
-import { basename } from 'node:path';
+import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { basename, dirname } from 'node:path';
 import { ScraperConfigError, VendorResponseError } from '../errors.js';
 import { familyBrand } from '../family.js';
 import { ALL_FAMILIES, FAMILIES, HOLDER_FAMILIES, familyConfig } from '../families/index.js';
 import { createFetcher } from '../fetch.js';
 import { AEM_BRANDS } from '../identity.js';
 import { boundFamily } from '../registry.js';
+import { pause, REQUEST_DELAY_MS } from '../scrape.js';
 import { mirrorFamilySteps } from './cad-mirror.js';
 import { parseCsv, toCsv } from './csv.js';
 import { describeRoot, familyCsv, stepDir } from './paths.js';
@@ -40,7 +44,15 @@ import { annotateCadUrls } from '../vendors/kennametal/cad.js';
 import { addMaterialGroups, groupsByMaterial } from '../vendors/kennametal/materials.js';
 import { addThreadPitch } from '../vendors/kennametal/thread-column.js';
 import { scrapeEndMills, DOCUMENTS_URL } from '../vendors/destinytool/scrape.js';
+import { PRODUCT_PAGES } from '../families/harvey.js';
+import { CATEGORY_ROOTS, discoverProducts } from '../vendors/harvey/catalog.js';
+import { scrapeProduct } from '../vendors/harvey/scrape.js';
 import { SEARCH_URL, scrapeCollets, scrapeHolders } from '../vendors/regofix/scrape.js';
+import { LEAVES as MARITOOL_LEAVES } from '../families/maritool.js';
+import { CATEGORY_ROOTS as MARITOOL_ROOTS, describe as describeCategory, discoverCategories, leavesOf, } from '../vendors/maritool/catalog.js';
+import { scrapeHolders as scrapeMaritoolHolders } from '../vendors/maritool/scrape.js';
+import { SCRAPE_TARGETS as EMUGE_TARGETS } from '../families/emuge.js';
+import { scrapeCategory } from '../vendors/emuge/scrape.js';
 /**
  * The PG series a BT 30 holder can take. PG 32 and PG 48 collets exist and no
  * BT 30 holder in this catalog accepts one, so scraping them would add parts
@@ -67,6 +79,35 @@ const USAGE = `usage: toolpath-scrape <command> [args]
       Pages the whole Destiny Tool \`products\` Firestore collection and writes
       every End Mill row.
 
+  harvey FAMILY.csv [more.csv ...]
+      One Harvey Tool product page -> a CSV. The page to fetch and the unit
+      system come from the family's own config, so neither is typed again.
+      One HTML row is up to nine orderable parts; expect more rows out than
+      the vendor's table appears to have.
+
+  harvey --catalog
+      Walks the four Harvey category trees and prints every product page it
+      finds, one per line. For noticing a page Harvey has added — a scrape
+      needs none of it.
+
+  maritool FAMILY.csv [more.csv ...]
+      One MariTool taper -> a CSV. The leaf categories to page through come
+      from the family's own config, so none of them is typed again. One
+      request per listing page, then one per part for its Product
+      Specifications table.
+
+  maritool --catalog
+      Walks the five MariTool taper trees and prints every category it finds,
+      one per line, with its product count. For rechecking the leaf cPaths in
+      \`families/maritool.ts\` — a scrape needs none of it.
+
+  emuge FAMILY.csv [more.csv ...]
+      One EMUGE-FRANKEN catalog category -> a CSV. The category, the facet
+      narrowing it and the unit system all come from the family's own config,
+      so none of them is typed again. One request per page of grouped
+      products, one per group for its orderable parts, and one per 30 parts
+      for the fields only a per-part record carries.
+
   thread-pitch TAP.csv [more.csv ...]
       Adds a Thread Pitch column derived from D1-TDZ, in place. Safe to re-run.
 
@@ -158,6 +199,12 @@ export async function run(argv, io = STDOUT, fetcher = createFetcher()) {
             return regofix(rest, io, fetcher);
         case 'destinytool':
             return destinytool(rest, io, fetcher);
+        case 'harvey':
+            return harvey(rest, io, fetcher);
+        case 'maritool':
+            return maritool(rest, io, fetcher);
+        case 'emuge':
+            return emuge(rest, io, fetcher);
         case 'thread-pitch':
             return threadPitch(rest, io);
         case 'cad':
@@ -251,6 +298,109 @@ async function destinytool(argv, io, fetcher) {
     wrote(argv[0], 'destinytool', { ...scrape, source: DOCUMENTS_URL }, io);
     return 0;
 }
+/**
+ * One or more Harvey families, or the catalog walk.
+ *
+ * Paced between pages by the package's shared politeness delay, which the
+ * per-page scrape does not do for itself: one family is one request, and a
+ * caller scraping one has nothing to wait for.
+ */
+async function harvey(argv, io, fetcher) {
+    if (argv[0] === '--catalog') {
+        const found = await discoverProducts(fetcher, CATEGORY_ROOTS, { warn: io.error });
+        for (const path of found)
+            io.log(path);
+        io.log(`${found.length} product pages under ${CATEGORY_ROOTS.length} category trees`);
+        return 0;
+    }
+    if (argv.length === 0) {
+        io.error(USAGE);
+        return 2;
+    }
+    const names = namesIn(argv, PRODUCT_PAGES, 'Harvey family');
+    for (const [index, name] of names.entries()) {
+        if (index > 0)
+            await pause(REQUEST_DELAY_MS);
+        const cfg = boundFamily(name);
+        // Never undefined: `namesIn` refused anything `PRODUCT_PAGES` does not key,
+        // and `tests/harvey-families.test.ts` holds the two tables to the same keys.
+        const page = PRODUCT_PAGES[name];
+        if (cfg.unit === undefined) {
+            throw new ScraperConfigError(name, 'declares no unit — every Harvey family publishes one');
+        }
+        const scrape = await scrapeProduct(fetcher, page, { unit: cfg.unit, warn: io.error });
+        const out = familyCsv(name);
+        // This command resolves its own output path rather than taking one, so the
+        // brand's directory may not exist yet — unlike `kennametal`, where the path
+        // is typed and its directory is the caller's to have made.
+        mkdirSync(dirname(out), { recursive: true });
+        wrote(out, 'harvey', scrape, io);
+    }
+    return 0;
+}
+/**
+ * One or more MariTool families, or the catalog walk.
+ *
+ * The scrape paces itself between every request it makes, so unlike `harvey`
+ * there is nothing to pace between families here.
+ */
+async function maritool(argv, io, fetcher) {
+    if (argv[0] === '--catalog') {
+        const found = await discoverCategories(fetcher, MARITOOL_ROOTS, { warn: io.error });
+        for (const category of found)
+            io.log(describeCategory(category));
+        const leaves = leavesOf(found);
+        io.log(`${found.length} categories under ${MARITOOL_ROOTS.length} taper trees, ` +
+            `${leaves.length} of them leaves, ` +
+            `${leaves.reduce((sum, leaf) => sum + leaf.products, 0)} products`);
+        return 0;
+    }
+    if (argv.length === 0) {
+        io.error(USAGE);
+        return 2;
+    }
+    for (const name of namesIn(argv, MARITOOL_LEAVES, 'MariTool family')) {
+        // Never undefined: `namesIn` refused anything `MARITOOL_LEAVES` does not
+        // key, and `tests/maritool.test.ts` holds the two tables to the same keys.
+        const leaves = MARITOOL_LEAVES[name];
+        const scrape = await scrapeMaritoolHolders(fetcher, leaves, { warn: io.error });
+        const out = familyCsv(name);
+        // This command resolves its own output path rather than taking one, so the
+        // brand's directory may not exist yet.
+        mkdirSync(dirname(out), { recursive: true });
+        wrote(out, 'maritool', scrape, io);
+    }
+    return 0;
+}
+/**
+ * One or more EMUGE-FRANKEN families.
+ *
+ * The scrape paces itself between every request it makes, so — as with
+ * `maritool` and unlike `harvey` — there is nothing to pace between families
+ * here.
+ */
+async function emuge(argv, io, fetcher) {
+    if (argv.length === 0) {
+        io.error(USAGE);
+        return 2;
+    }
+    for (const name of namesIn(argv, EMUGE_TARGETS, 'EMUGE-FRANKEN family')) {
+        const cfg = boundFamily(name);
+        // Never undefined: `namesIn` refused anything `EMUGE_TARGETS` does not key,
+        // and `tests/emuge-families.test.ts` holds the two tables to the same keys.
+        const target = EMUGE_TARGETS[name];
+        if (cfg.unit === undefined) {
+            throw new ScraperConfigError(name, 'declares no unit — every EMUGE-FRANKEN family publishes one');
+        }
+        const scrape = await scrapeCategory(fetcher, target, { unit: cfg.unit, warn: io.error });
+        const out = familyCsv(name);
+        // This command resolves its own output path rather than taking one, so the
+        // brand's directory may not exist yet.
+        mkdirSync(dirname(out), { recursive: true });
+        wrote(out, 'emuge', scrape, io);
+    }
+    return 0;
+}
 function threadPitch(argv, io) {
     if (argv.length === 0) {
         io.error(USAGE);