@toolpath/tool-scraper 0.1.0 → 2.0.0

package/dist/vendors/harvey/literal.d.ts

@@ -0,0 +1,68 @@
+/**
+ * A JavaScript object literal inlined in a page -> JSON. No Harvey knowledge.
+ *
+ * Harvey Tool serves its whole variant table as `var tableData1 = [{...}]` in a
+ * ~557 KB `<script>` block, with bare identifier keys that `JSON.parse` refuses.
+ * This module is the smallest thing that turns one into JSON, and it is separate
+ * from every other file here because it knows nothing about tools: it takes a
+ * document and a variable name and hands back a value.
+ *
+ * ## The obvious regex is a silent data corruptor
+ *
+ * The reflex is `/([{,])\s*(\w+)\s*:/` -> `'$1"$2":'` and then `JSON.parse`. It
+ * is wrong, and wrong in the worst way — it produces a document that parses.
+ *
+ * Harvey's cells carry HTML with quotes and colons in them:
+ *
+ * ```js
+ * t:"color:#70C0FF"
+ * d:"<a href=\"/products/tool-details-24502\">24502</a>"
+ * ```
+ *
+ * `color:` inside that first string is not an object key, and a `{x:1}` in a
+ * product description is not an object. A regex has no way to know, so it
+ * rewrites text inside strings and the result is a scrape that still produces
+ * rows, still passes a row count, and has wrong data in it.
+ *
+ * So this walks the source once, tracking whether it is inside a string and
+ * whether the last character was an escape, and quotes a bare key only when it
+ * is genuinely at an object-key position. That is about sixty lines and it is
+ * the difference between a parser and a coincidence.
+ *
+ * ## Two passes over the same scan
+ *
+ * {@link findLiteral} locates `var <name> =` by index and then brace-matches
+ * forward with the same string-aware rules, so a whole 720 KB document is never
+ * handed to a regex engine more than once per variable. 18 MB of HTML goes
+ * through this code on a full scrape; a repeated whole-document match is what
+ * turns that into minutes of CPU.
+ */
+/**
+ * The source text of the bracketed value assigned to `var <name>`.
+ *
+ * Returns null when the document declares no such variable, which is an
+ * ordinary answer: Harvey emits `cols1` through `cols10` on every page and
+ * leaves the unused ones empty, and a page with one table simply has no
+ * `tableData2`.
+ *
+ * Throws when the assignment is there but its brackets never balance, because
+ * that is a truncated response rather than an absent one, and returning null
+ * would report it as a page with fewer tables.
+ */
+export declare function findLiteral(source: string, name: string): string | null;
+/**
+ * A JavaScript object literal as JSON text: bare keys quoted, everything else
+ * byte for byte.
+ *
+ * Only *keys* are rewritten. A bare word at a value position — `undefined`, a
+ * single-quoted string — is left where it is and `JSON.parse` refuses it, which
+ * is the right outcome: the page changed shape, and inventing a reading for a
+ * token this has never seen is how a scraper starts authoring data.
+ */
+export declare function toJson(literal: string, what: string): string;
+/**
+ * The value of `var <name>` in `source`, parsed.
+ *
+ * Null where the variable is absent — see {@link findLiteral}.
+ */
+export declare function readLiteral<T>(source: string, name: string): T | null;

package/dist/vendors/harvey/literal.js

@@ -0,0 +1,214 @@
+/**
+ * A JavaScript object literal inlined in a page -> JSON. No Harvey knowledge.
+ *
+ * Harvey Tool serves its whole variant table as `var tableData1 = [{...}]` in a
+ * ~557 KB `<script>` block, with bare identifier keys that `JSON.parse` refuses.
+ * This module is the smallest thing that turns one into JSON, and it is separate
+ * from every other file here because it knows nothing about tools: it takes a
+ * document and a variable name and hands back a value.
+ *
+ * ## The obvious regex is a silent data corruptor
+ *
+ * The reflex is `/([{,])\s*(\w+)\s*:/` -> `'$1"$2":'` and then `JSON.parse`. It
+ * is wrong, and wrong in the worst way — it produces a document that parses.
+ *
+ * Harvey's cells carry HTML with quotes and colons in them:
+ *
+ * ```js
+ * t:"color:#70C0FF"
+ * d:"<a href=\"/products/tool-details-24502\">24502</a>"
+ * ```
+ *
+ * `color:` inside that first string is not an object key, and a `{x:1}` in a
+ * product description is not an object. A regex has no way to know, so it
+ * rewrites text inside strings and the result is a scrape that still produces
+ * rows, still passes a row count, and has wrong data in it.
+ *
+ * So this walks the source once, tracking whether it is inside a string and
+ * whether the last character was an escape, and quotes a bare key only when it
+ * is genuinely at an object-key position. That is about sixty lines and it is
+ * the difference between a parser and a coincidence.
+ *
+ * ## Two passes over the same scan
+ *
+ * {@link findLiteral} locates `var <name> =` by index and then brace-matches
+ * forward with the same string-aware rules, so a whole 720 KB document is never
+ * handed to a regex engine more than once per variable. 18 MB of HTML goes
+ * through this code on a full scrape; a repeated whole-document match is what
+ * turns that into minutes of CPU.
+ */
+import { VendorResponseError } from '../../errors.js';
+/** Opening brackets, and the closer each one expects. */
+const CLOSERS = { '{': '}', '[': ']' };
+/** True where `ch` can begin a bare JavaScript identifier. */
+function identifierStart(ch) {
+    return /[A-Za-z_$]/.test(ch);
+}
+/** True where `ch` can continue one. */
+function identifierPart(ch) {
+    return /[A-Za-z0-9_$]/.test(ch);
+}
+/**
+ * The source text of the bracketed value assigned to `var <name>`.
+ *
+ * Returns null when the document declares no such variable, which is an
+ * ordinary answer: Harvey emits `cols1` through `cols10` on every page and
+ * leaves the unused ones empty, and a page with one table simply has no
+ * `tableData2`.
+ *
+ * Throws when the assignment is there but its brackets never balance, because
+ * that is a truncated response rather than an absent one, and returning null
+ * would report it as a page with fewer tables.
+ */
+export function findLiteral(source, name) {
+    const declaration = new RegExp(`\\bvar\\s+${name}\\s*=\\s*`, 'g');
+    const match = declaration.exec(source);
+    if (match === null)
+        return null;
+    // The regex has already consumed the whitespace after `=`, so the literal
+    // starts here or nowhere. Searching forward for the next bracket instead
+    // would happily find one inside the *next* variable's string.
+    const start = match.index + match[0].length;
+    if (source[start] !== '[' && source[start] !== '{') {
+        throw new VendorResponseError(name, 'is assigned something that is not an array or object');
+    }
+    const end = matchBracket(source, start, name);
+    return source.slice(start, end + 1);
+}
+/**
+ * The index of the bracket closing the one at `start`, string-aware.
+ *
+ * The whole reason this is not a depth counter over the raw text: a `}` inside
+ * a `d` cell's HTML would close an object that is still open, and every
+ * following column would land one key to the left.
+ */
+function matchBracket(source, start, what) {
+    const stack = [];
+    let inString = false;
+    let escaped = false;
+    for (let i = start; i < source.length; i++) {
+        const ch = source[i];
+        if (inString) {
+            if (escaped)
+                escaped = false;
+            else if (ch === '\\')
+                escaped = true;
+            else if (ch === '"')
+                inString = false;
+            continue;
+        }
+        if (ch === '"') {
+            inString = true;
+        }
+        else if (ch === '{' || ch === '[') {
+            stack.push(CLOSERS[ch]);
+        }
+        else if (ch === '}' || ch === ']') {
+            const expected = stack.pop();
+            if (expected !== ch) {
+                throw new VendorResponseError(what, `has ${JSON.stringify(ch)} at offset ${i - start} where ` +
+                    `${expected === undefined ? 'nothing was open' : JSON.stringify(expected)} was due`);
+            }
+            if (stack.length === 0)
+                return i;
+        }
+    }
+    throw new VendorResponseError(what, 'is not closed before the end of the document — a truncated response');
+}
+/**
+ * A JavaScript object literal as JSON text: bare keys quoted, everything else
+ * byte for byte.
+ *
+ * Only *keys* are rewritten. A bare word at a value position — `undefined`, a
+ * single-quoted string — is left where it is and `JSON.parse` refuses it, which
+ * is the right outcome: the page changed shape, and inventing a reading for a
+ * token this has never seen is how a scraper starts authoring data.
+ */
+export function toJson(literal, what) {
+    const out = [];
+    // Whether the innermost bracket is an object, and whether the next token in
+    // it is a key. Both are needed: `[{a:1}]` is at key position after `{` but
+    // not after `[`, and `{a:[1,2]}` is not at key position after that comma.
+    const objects = [];
+    let expectKey = false;
+    let inString = false;
+    let escaped = false;
+    for (let i = 0; i < literal.length; i++) {
+        const ch = literal[i];
+        if (inString) {
+            out.push(ch);
+            if (escaped)
+                escaped = false;
+            else if (ch === '\\')
+                escaped = true;
+            else if (ch === '"')
+                inString = false;
+            continue;
+        }
+        if (ch === '"') {
+            inString = true;
+            expectKey = false;
+            out.push(ch);
+            continue;
+        }
+        if (ch === '{' || ch === '[') {
+            objects.push(ch === '{');
+            expectKey = ch === '{';
+            out.push(ch);
+            continue;
+        }
+        if (ch === '}' || ch === ']') {
+            objects.pop();
+            expectKey = false;
+            out.push(ch);
+            continue;
+        }
+        if (ch === ',') {
+            expectKey = objects[objects.length - 1] === true;
+            out.push(ch);
+            continue;
+        }
+        if (ch === ':') {
+            expectKey = false;
+            out.push(ch);
+            continue;
+        }
+        if (/\s/.test(ch)) {
+            out.push(ch);
+            continue;
+        }
+        if (expectKey && identifierStart(ch)) {
+            let end = i;
+            while (end < literal.length && identifierPart(literal[end]))
+                end++;
+            out.push(JSON.stringify(literal.slice(i, end)));
+            i = end - 1;
+            expectKey = false;
+            continue;
+        }
+        if (ch === "'") {
+            // Never seen on a Harvey page, and re-escaping one into a JSON string is
+            // guesswork about what the vendor meant. Refuse loudly instead.
+            throw new VendorResponseError(what, `has a single-quoted string at offset ${i}`);
+        }
+        expectKey = false;
+        out.push(ch);
+    }
+    return out.join('');
+}
+/**
+ * The value of `var <name>` in `source`, parsed.
+ *
+ * Null where the variable is absent — see {@link findLiteral}.
+ */
+export function readLiteral(source, name) {
+    const literal = findLiteral(source, name);
+    if (literal === null)
+        return null;
+    try {
+        return JSON.parse(toJson(literal, name));
+    }
+    catch (error) {
+        throw new VendorResponseError(name, `is not a readable object literal: ${error.message}`);
+    }
+}

package/dist/vendors/harvey/records.d.ts

@@ -0,0 +1,79 @@
+/**
+ * Harvey rows -> {@link ToolRecord}.
+ *
+ * The CSV a Harvey scrape writes holds the vendor's own **display strings** —
+ * `.250 (1/4)`, `1-1/2`, `3 mm`, `-` — rather than parsed numbers, because the
+ * file is the receipt and Harvey's own fractional and metric annotations are
+ * part of what it published. So this module is where a cell becomes a number,
+ * through `value.ts`; the closest precedent in this package is Destiny Tool's
+ * fractional-inch reader, not Kennametal's, whose columns are already decimals.
+ *
+ * ## What Harvey does not publish
+ *
+ * **No second identifier.** One `Tool #` per part, which fills both the material
+ * number and the catalog number on a record — see
+ * `conventions.IDENTITY_DEVIATIONS.harvey`.
+ *
+ * **No carbide grade.** `substrate` comes from the family's `bmc` fact and the
+ * `COATING` column fills `coating`, exactly as Destiny Tool's coating id does.
+ *
+ * **No workpiece-material index a scrape can reach.** Nothing in a variant
+ * table rates a tool to ISO 513 groups, so every Harvey record is labelled
+ * `records.UNSPECIFIED` — *we do not know what this tool is for*, which is a
+ * different claim from Kennametal's swept taps being rated for nothing.
+ *
+ * The per-part page does publish one, and it is **per part and not per family**:
+ * a 192-request probe on 2026-08-29 found `harvey_endmill_005.csv` splitting by
+ * coating column, uncoated tools rated for steel, stainless, cast iron and
+ * titanium where the amorphous-diamond-coated tools of the same geometry are
+ * rated for aluminium, wood and composites alone. That is correct metallurgy —
+ * diamond cannot cut ferrous — and flattening it to one answer per family would
+ * put a diamond-coated end mill under steel. So there is no family fact to
+ * write, and the two keyseat pages titled for a material class do not get one
+ * either: the "For Non - Ferrous Materials" line's own part pages rate it for
+ * steel and stainless steel, so the title does not predict the index.
+ * `docs/HARVEY_PRODUCT_TABLE.md` §1.5 has the measurement and what reaching it
+ * would cost.
+ *
+ * **No corner radius on a ball nose.** A ball family publishes no radius column
+ * at all, so `RE` comes from the family's `profile` fact — see
+ * {@link cornerRadius}. That is a per-family constant with provenance rather
+ * than a string match on a description, because Harvey states the profile once,
+ * in the page title, for the whole product line.
+ */
+import { type BoundFamily, type RecordMappers } from '../../family.js';
+import { type ColumnMap, type ToolRecord } from '../../records.js';
+import { type MapperOptions, type ScrapedRow } from '../../scrape.js';
+/** The `profile` fact value that means a ball nose. Harvey's own word. */
+export declare const BALL_PROFILE = "Ball";
+/**
+ * The corner radius, in priority order.
+ *
+ * 1. The family's own radius column where it has one — `CORNER RADIUS` on the
+ *    corner-radius lines, `RADIUS` on the full-radius keyseat cutters.
+ * 2. `DC / 2` on a ball nose. Harvey publishes no radius column on any of its
+ *    twelve ball families, and the radius of a ball end *is* half the diameter,
+ *    so this is arithmetic rather than a guess — the `profile` fact is what says
+ *    the family is one.
+ * 3. `0` — a real square end — otherwise.
+ *
+ * A mapped column whose cell is blank falls through to 2 or 3 rather than
+ * refusing the row: `RE` is optional on the endmill contract precisely because
+ * a square-end row's blank radius is an answer.
+ */
+export declare function cornerRadius(row: ScrapedRow, family: BoundFamily, columns: ColumnMap, what: string, dc: number, options: MapperOptions): number;
+/**
+ * The flute count.
+ *
+ * One column whichever way the table encoded it: `vendors/harvey/scrape.ts`
+ * writes `FLUTES` from the row's own column on a `TOOL #` table and from the
+ * coating group's sub-label on a matrix one, so nothing downstream has to know
+ * which shape the page used.
+ *
+ * Null is a real answer on the two deburring families, which publish
+ * right- and left-hand tooth counts and no flute count at all.
+ */
+export declare function flutes(row: ScrapedRow, family: BoundFamily, columns: ColumnMap): number | null;
+/** One orderable Harvey tool. */
+export declare function endmillRecord(row: ScrapedRow, family: BoundFamily, columns: ColumnMap, options?: MapperOptions): ToolRecord;
+export declare const RECORD_MAPPERS: RecordMappers;

package/dist/vendors/harvey/records.js

@@ -0,0 +1,163 @@
+/**
+ * Harvey rows -> {@link ToolRecord}.
+ *
+ * The CSV a Harvey scrape writes holds the vendor's own **display strings** —
+ * `.250 (1/4)`, `1-1/2`, `3 mm`, `-` — rather than parsed numbers, because the
+ * file is the receipt and Harvey's own fractional and metric annotations are
+ * part of what it published. So this module is where a cell becomes a number,
+ * through `value.ts`; the closest precedent in this package is Destiny Tool's
+ * fractional-inch reader, not Kennametal's, whose columns are already decimals.
+ *
+ * ## What Harvey does not publish
+ *
+ * **No second identifier.** One `Tool #` per part, which fills both the material
+ * number and the catalog number on a record — see
+ * `conventions.IDENTITY_DEVIATIONS.harvey`.
+ *
+ * **No carbide grade.** `substrate` comes from the family's `bmc` fact and the
+ * `COATING` column fills `coating`, exactly as Destiny Tool's coating id does.
+ *
+ * **No workpiece-material index a scrape can reach.** Nothing in a variant
+ * table rates a tool to ISO 513 groups, so every Harvey record is labelled
+ * `records.UNSPECIFIED` — *we do not know what this tool is for*, which is a
+ * different claim from Kennametal's swept taps being rated for nothing.
+ *
+ * The per-part page does publish one, and it is **per part and not per family**:
+ * a 192-request probe on 2026-08-29 found `harvey_endmill_005.csv` splitting by
+ * coating column, uncoated tools rated for steel, stainless, cast iron and
+ * titanium where the amorphous-diamond-coated tools of the same geometry are
+ * rated for aluminium, wood and composites alone. That is correct metallurgy —
+ * diamond cannot cut ferrous — and flattening it to one answer per family would
+ * put a diamond-coated end mill under steel. So there is no family fact to
+ * write, and the two keyseat pages titled for a material class do not get one
+ * either: the "For Non - Ferrous Materials" line's own part pages rate it for
+ * steel and stainless steel, so the title does not predict the index.
+ * `docs/HARVEY_PRODUCT_TABLE.md` §1.5 has the measurement and what reaching it
+ * would cost.
+ *
+ * **No corner radius on a ball nose.** A ball family publishes no radius column
+ * at all, so `RE` comes from the family's `profile` fact — see
+ * {@link cornerRadius}. That is a per-family constant with provenance rather
+ * than a string match on a description, because Harvey states the profile once,
+ * in the page title, for the whole product line.
+ */
+import { columnReaders } from '../../columns.js';
+import { DESCRIPTION_COLUMN } from '../../conventions.js';
+import { VendorResponseError } from '../../errors.js';
+import { fact, familyBrand } from '../../family.js';
+import { BRANDS } from '../../identity.js';
+import { toolRecord } from '../../records.js';
+import { consoleWarn } from '../../scrape.js';
+import { COATING_COLUMN, TOOL_NUMBER_COLUMN } from './scrape.js';
+import { count, dimension } from './value.js';
+/** The `profile` fact value that means a ball nose. Harvey's own word. */
+export const BALL_PROFILE = 'Ball';
+/**
+ * The family's declared unit, refused rather than asserted when it is absent.
+ *
+ * Every Harvey family declares one — the `harvey` CLI command will not scrape a
+ * family without it — so `familyUnits` would return exactly this one and a
+ * tap's two-system case cannot arise here. It was `family.unit!` until
+ * 2026-08-29, which is the same claim with no check behind it: a family added
+ * without the fact would have read `undefined` straight into
+ * `dimensionalColumn` and asked the CSV for a column called `CUTTER DIA._`.
+ */
+function unitOf(family) {
+    return fact(family, 'unit', family.unit);
+}
+/**
+ * The three column readers, over this vendor's grammar.
+ *
+ * `dimension` is the only Harvey-specific half; everything either side of it —
+ * an unmapped column answering undefined, a required field refusing the row and
+ * quoting the cell — is `columns.columnReaders`, shared with EMUGE-FRANKEN's
+ * mapper. All three take the caller's `columns` rather than `family.columns`:
+ * they are the same object through `registry.toRecords`, but `registry`
+ * validates the argument, and a mapper reading a different reference is
+ * validating one map and reading another. This mapper asserted the parameter
+ * was unused with an underscore until 2026-08-29.
+ */
+const { cell, required, optional } = columnReaders(dimension);
+/**
+ * The corner radius, in priority order.
+ *
+ * 1. The family's own radius column where it has one — `CORNER RADIUS` on the
+ *    corner-radius lines, `RADIUS` on the full-radius keyseat cutters.
+ * 2. `DC / 2` on a ball nose. Harvey publishes no radius column on any of its
+ *    twelve ball families, and the radius of a ball end *is* half the diameter,
+ *    so this is arithmetic rather than a guess — the `profile` fact is what says
+ *    the family is one.
+ * 3. `0` — a real square end — otherwise.
+ *
+ * A mapped column whose cell is blank falls through to 2 or 3 rather than
+ * refusing the row: `RE` is optional on the endmill contract precisely because
+ * a square-end row's blank radius is an answer.
+ */
+export function cornerRadius(row, family, columns, what, dc, options) {
+    const stated = optional(row, columns, 'RE', unitOf(family), what, options);
+    if (stated !== null)
+        return stated;
+    return family.profile === BALL_PROFILE ? dc / 2 : 0;
+}
+/**
+ * The flute count.
+ *
+ * One column whichever way the table encoded it: `vendors/harvey/scrape.ts`
+ * writes `FLUTES` from the row's own column on a `TOOL #` table and from the
+ * coating group's sub-label on a matrix one, so nothing downstream has to know
+ * which shape the page used.
+ *
+ * Null is a real answer on the two deburring families, which publish
+ * right- and left-hand tooth counts and no flute count at all.
+ */
+export function flutes(row, family, columns) {
+    const raw = cell(row, columns, 'NOF', unitOf(family));
+    return raw === undefined ? null : count(raw);
+}
+/** One orderable Harvey tool. */
+export function endmillRecord(row, family, columns, options = {}) {
+    const warn = options.warn ?? consoleWarn;
+    const what = row[TOOL_NUMBER_COLUMN] ?? '';
+    if (what === '') {
+        throw new VendorResponseError(family.id, `has a row with no ${TOOL_NUMBER_COLUMN}`);
+    }
+    const opts = { warn };
+    const unit = unitOf(family);
+    const dc = required(row, columns, 'DC', unit, what, opts);
+    const fluteLength = required(row, columns, 'LCF', unit, what, opts);
+    // Harvey's reach columns are the distance from the tip to the full shank,
+    // which is what `shoulder-length` names. A family with no reach column is a
+    // plain tool whose usable length below the shank is its flute length — the
+    // same convention Destiny Tool's mapper uses.
+    const reach = optional(row, columns, 'shoulder-length', unit, what, opts);
+    const neck = optional(row, columns, 'shoulder-diameter', unit, what, opts);
+    const geometry = {
+        DC: dc,
+        RE: cornerRadius(row, family, columns, what, dc, opts),
+        SFDM: required(row, columns, 'SFDM', unit, what, opts),
+        OAL: required(row, columns, 'OAL', unit, what, opts),
+        LCF: fluteLength,
+        'shoulder-length': reach ?? fluteLength,
+        'shoulder-diameter': neck ?? dc,
+    };
+    const nof = flutes(row, family, columns);
+    if (nof !== null)
+        geometry.NOF = nof;
+    return toolRecord({
+        brand: familyBrand(family),
+        vendor: BRANDS[familyBrand(family)].vendor,
+        materialNumber: what,
+        catalogNumber: what,
+        description: row[DESCRIPTION_COLUMN] ?? '',
+        kind: 'endmill',
+        unit,
+        substrate: fact(family, 'bmc', family.bmc),
+        coating: row[COATING_COLUMN] ?? '',
+        // `materialGroups` and its source are left to the factory's `null`: see the
+        // module docstring for what a Harvey part page publishes and why none of it
+        // can be stated per family.
+        coolantThrough: fact(family, 'coolantThrough', family.coolantThrough),
+        geometry,
+    });
+}
+export const RECORD_MAPPERS = { endmill: endmillRecord };