@toolpath/tool-scraper 0.1.0 → 2.1.0

package/dist/vendors/harvey/lexicon.d.ts

@@ -0,0 +1,73 @@
+/**
+ * What Harvey's own column labels mean, and which of them carry a unit.
+ *
+ * 50 distinct header shapes across the 52 product pages collapse to the table
+ * below, because the variance is in *spelling* rather than in vocabulary:
+ * `CUTTER DIA.` and `CUTTER DIAMETER` are the same column, and the keyseat
+ * families call the length of cut `CUTTER WIDTH`.
+ *
+ * **This module does not map a label to a canonical ISO name.** That mapping is
+ * per family and lives in `families/harvey.ts`, where it belongs: a family
+ * declares which of *its* labels is `LCF`, and the core appends the unit suffix.
+ * What is left over — is this label one this package has seen, and does its
+ * column carry a length — is the same on every page, and is here.
+ *
+ * ## An unknown label is a hard failure
+ *
+ * A label nothing here recognises stops the scrape naming the family. The
+ * alternative is a column that reaches the CSV under whatever Harvey renamed it
+ * to, silently no longer matching the family's column map, and a family whose
+ * `DC` is quietly absent from every record. The cost of the strict rule is that
+ * a vendor's copy edit breaks a scrape; the cost of the loose one is that it
+ * does not.
+ */
+/**
+ * What a column holds, which is only ever asked so that a length gets a `_mm` /
+ * `_in` suffix and a tooth count does not.
+ *
+ * `code` is for a column whose values are not numbers at all — the keyseat
+ * `TYPE`, whose `I`/`II`/`III` nobody at Harvey has yet explained.
+ */
+export type LabelKind = 'dimension' | 'count' | 'angle' | 'code';
+/**
+ * Every geometry top label the 52 product pages publish, and what it holds.
+ *
+ * Written out rather than pattern-matched. A regex over "anything containing
+ * DIA" would have swallowed `RADIAL DOC` and `NECK DIA.` into the same rule and
+ * quietly given one of them the other's meaning, and the whole point of the
+ * strict check below is that a label this package has not seen gets looked at
+ * by a person.
+ */
+export declare const LABELS: Readonly<Record<string, LabelKind>>;
+/** The top label of the column holding the add-to-cart control. Dropped. */
+export declare const CART_LABEL = "Add to Cart";
+/** The top label of the column stating a row's flute count, where there is one. */
+export declare const FLUTES_LABEL = "FLUTES";
+/** The sub-label a coating group uses when the row states the flute count. */
+export declare const TOOL_NUMBER_LABEL = "TOOL #";
+/** The sub-label ending every coating group. */
+export declare const PRICE_LABEL = "PRICE";
+/**
+ * The eight coating names Harvey publishes across the whole catalog.
+ *
+ * A closed list, checked, because the coating is *synthesised* from a column's
+ * position rather than lifted from a cell — nothing in a row says which coating
+ * a tool number belongs to. A ninth coating is Harvey extending its catalog and
+ * is worth a person looking; a coating group whose header stopped being a
+ * coating is the grid having shifted, and that is the failure this catches.
+ */
+export declare const COATINGS: readonly string[];
+/** The flute count a group sub-label states, or null where it states none. */
+export declare function flutesInLabel(sub: string): number | null;
+/**
+ * What the column under `label` holds, refusing a label nothing here knows.
+ *
+ * `family` is in the message because that is the one thing the person reading
+ * the failure needs and cannot work out from the label: 52 pages publish these
+ * labels, and only one of them changed.
+ */
+export declare function labelKind(family: string, label: string): LabelKind;
+/** Whether the column under `label` gets a `_mm` / `_in` suffix. */
+export declare function isDimensional(family: string, label: string): boolean;
+/** Refuse a coating group header that is not one of the eight. */
+export declare function checkCoating(family: string, coating: string): string;

package/dist/vendors/harvey/lexicon.js

@@ -0,0 +1,126 @@
+/**
+ * What Harvey's own column labels mean, and which of them carry a unit.
+ *
+ * 50 distinct header shapes across the 52 product pages collapse to the table
+ * below, because the variance is in *spelling* rather than in vocabulary:
+ * `CUTTER DIA.` and `CUTTER DIAMETER` are the same column, and the keyseat
+ * families call the length of cut `CUTTER WIDTH`.
+ *
+ * **This module does not map a label to a canonical ISO name.** That mapping is
+ * per family and lives in `families/harvey.ts`, where it belongs: a family
+ * declares which of *its* labels is `LCF`, and the core appends the unit suffix.
+ * What is left over — is this label one this package has seen, and does its
+ * column carry a length — is the same on every page, and is here.
+ *
+ * ## An unknown label is a hard failure
+ *
+ * A label nothing here recognises stops the scrape naming the family. The
+ * alternative is a column that reaches the CSV under whatever Harvey renamed it
+ * to, silently no longer matching the family's column map, and a family whose
+ * `DC` is quietly absent from every record. The cost of the strict rule is that
+ * a vendor's copy edit breaks a scrape; the cost of the loose one is that it
+ * does not.
+ */
+import { ScraperConfigError } from '../../errors.js';
+/**
+ * Every geometry top label the 52 product pages publish, and what it holds.
+ *
+ * Written out rather than pattern-matched. A regex over "anything containing
+ * DIA" would have swallowed `RADIAL DOC` and `NECK DIA.` into the same rule and
+ * quietly given one of them the other's meaning, and the whole point of the
+ * strict check below is that a label this package has not seen gets looked at
+ * by a person.
+ */
+export const LABELS = {
+    // Mapped to canonical names by each family's own column map.
+    'CUTTER DIA.': 'dimension',
+    'CUTTER DIAMETER': 'dimension',
+    LOC: 'dimension',
+    'LENGTH OF CUT': 'dimension',
+    'CUTTER WIDTH': 'dimension',
+    'SHANK DIA.': 'dimension',
+    'SHANK DIAMETER': 'dimension',
+    OAL: 'dimension',
+    'OVERALL LENGTH': 'dimension',
+    'CORNER RADIUS': 'dimension',
+    RADIUS: 'dimension',
+    'NECK DIA.': 'dimension',
+    'NECK LENGTH': 'dimension',
+    'OVERALL REACH': 'dimension',
+    FLUTES: 'count',
+    // Kept verbatim and mapped to nothing. Each is either a measurement this
+    // package has no canonical name for, or one whose meaning Harvey has not
+    // stated — see `docs/HARVEY_PRODUCT_TABLE.md` §5.1 and §6.
+    'RADIAL DOC*': 'dimension',
+    'Radial DOC*': 'dimension',
+    'Radial DOC**': 'dimension',
+    'Interference Depth At Wall Angle*': 'dimension',
+    'RIGHT HAND TEETH': 'count',
+    'LEFT HAND TEETH': 'count',
+    'ANGLE PER SIDE': 'angle',
+    'EFFECTIVE WALL ANGLE*': 'angle',
+    'EFF WALL ANGLE': 'angle',
+    TYPE: 'code',
+};
+/** The top label of the column holding the add-to-cart control. Dropped. */
+export const CART_LABEL = 'Add to Cart';
+/** The top label of the column stating a row's flute count, where there is one. */
+export const FLUTES_LABEL = 'FLUTES';
+/** The sub-label a coating group uses when the row states the flute count. */
+export const TOOL_NUMBER_LABEL = 'TOOL #';
+/** The sub-label ending every coating group. */
+export const PRICE_LABEL = 'PRICE';
+/** `2 FL`, `4FL`, `5 FL` — a per-part flute count in a group's sub-label. */
+const FLUTE_LABEL = /^(\d+)\s*FL$/i;
+/**
+ * The eight coating names Harvey publishes across the whole catalog.
+ *
+ * A closed list, checked, because the coating is *synthesised* from a column's
+ * position rather than lifted from a cell — nothing in a row says which coating
+ * a tool number belongs to. A ninth coating is Harvey extending its catalog and
+ * is worth a person looking; a coating group whose header stopped being a
+ * coating is the grid having shifted, and that is the failure this catches.
+ */
+export const COATINGS = [
+    'UNCOATED',
+    'AlTiN COATED',
+    'AMORPHOUS DIAMOND COATED',
+    'AlTiN NANO COATED',
+    'Ti NANO COATED',
+    'TiB2 COATED',
+    'BALL END UNCOATED',
+    'AlTiN NANO BALL END COATED',
+];
+/** The flute count a group sub-label states, or null where it states none. */
+export function flutesInLabel(sub) {
+    const match = FLUTE_LABEL.exec(sub);
+    return match ? Number.parseInt(match[1], 10) : null;
+}
+/**
+ * What the column under `label` holds, refusing a label nothing here knows.
+ *
+ * `family` is in the message because that is the one thing the person reading
+ * the failure needs and cannot work out from the label: 52 pages publish these
+ * labels, and only one of them changed.
+ */
+export function labelKind(family, label) {
+    const kind = LABELS[label];
+    if (kind === undefined) {
+        throw new ScraperConfigError(family, `publishes a column headed ${JSON.stringify(label)}, which this adapter ` +
+            `does not know — add it to vendors/harvey/lexicon.ts with what it ` +
+            `holds, and to the family's column map if it is a canonical field`);
+    }
+    return kind;
+}
+/** Whether the column under `label` gets a `_mm` / `_in` suffix. */
+export function isDimensional(family, label) {
+    return labelKind(family, label) === 'dimension';
+}
+/** Refuse a coating group header that is not one of the eight. */
+export function checkCoating(family, coating) {
+    if (!COATINGS.includes(coating)) {
+        throw new ScraperConfigError(family, `has a tool-number group headed ${JSON.stringify(coating)}, which is not ` +
+            `one of the coatings this adapter knows (${COATINGS.join(', ')})`);
+    }
+    return coating;
+}

package/dist/vendors/harvey/literal.d.ts

@@ -0,0 +1,68 @@
+/**
+ * A JavaScript object literal inlined in a page -> JSON. No Harvey knowledge.
+ *
+ * Harvey Tool serves its whole variant table as `var tableData1 = [{...}]` in a
+ * ~557 KB `<script>` block, with bare identifier keys that `JSON.parse` refuses.
+ * This module is the smallest thing that turns one into JSON, and it is separate
+ * from every other file here because it knows nothing about tools: it takes a
+ * document and a variable name and hands back a value.
+ *
+ * ## The obvious regex is a silent data corruptor
+ *
+ * The reflex is `/([{,])\s*(\w+)\s*:/` -> `'$1"$2":'` and then `JSON.parse`. It
+ * is wrong, and wrong in the worst way — it produces a document that parses.
+ *
+ * Harvey's cells carry HTML with quotes and colons in them:
+ *
+ * ```js
+ * t:"color:#70C0FF"
+ * d:"<a href=\"/products/tool-details-24502\">24502</a>"
+ * ```
+ *
+ * `color:` inside that first string is not an object key, and a `{x:1}` in a
+ * product description is not an object. A regex has no way to know, so it
+ * rewrites text inside strings and the result is a scrape that still produces
+ * rows, still passes a row count, and has wrong data in it.
+ *
+ * So this walks the source once, tracking whether it is inside a string and
+ * whether the last character was an escape, and quotes a bare key only when it
+ * is genuinely at an object-key position. That is about sixty lines and it is
+ * the difference between a parser and a coincidence.
+ *
+ * ## Two passes over the same scan
+ *
+ * {@link findLiteral} locates `var <name> =` by index and then brace-matches
+ * forward with the same string-aware rules, so a whole 720 KB document is never
+ * handed to a regex engine more than once per variable. 18 MB of HTML goes
+ * through this code on a full scrape; a repeated whole-document match is what
+ * turns that into minutes of CPU.
+ */
+/**
+ * The source text of the bracketed value assigned to `var <name>`.
+ *
+ * Returns null when the document declares no such variable, which is an
+ * ordinary answer: Harvey emits `cols1` through `cols10` on every page and
+ * leaves the unused ones empty, and a page with one table simply has no
+ * `tableData2`.
+ *
+ * Throws when the assignment is there but its brackets never balance, because
+ * that is a truncated response rather than an absent one, and returning null
+ * would report it as a page with fewer tables.
+ */
+export declare function findLiteral(source: string, name: string): string | null;
+/**
+ * A JavaScript object literal as JSON text: bare keys quoted, everything else
+ * byte for byte.
+ *
+ * Only *keys* are rewritten. A bare word at a value position — `undefined`, a
+ * single-quoted string — is left where it is and `JSON.parse` refuses it, which
+ * is the right outcome: the page changed shape, and inventing a reading for a
+ * token this has never seen is how a scraper starts authoring data.
+ */
+export declare function toJson(literal: string, what: string): string;
+/**
+ * The value of `var <name>` in `source`, parsed.
+ *
+ * Null where the variable is absent — see {@link findLiteral}.
+ */
+export declare function readLiteral<T>(source: string, name: string): T | null;

package/dist/vendors/harvey/literal.js

@@ -0,0 +1,214 @@
+/**
+ * A JavaScript object literal inlined in a page -> JSON. No Harvey knowledge.
+ *
+ * Harvey Tool serves its whole variant table as `var tableData1 = [{...}]` in a
+ * ~557 KB `<script>` block, with bare identifier keys that `JSON.parse` refuses.
+ * This module is the smallest thing that turns one into JSON, and it is separate
+ * from every other file here because it knows nothing about tools: it takes a
+ * document and a variable name and hands back a value.
+ *
+ * ## The obvious regex is a silent data corruptor
+ *
+ * The reflex is `/([{,])\s*(\w+)\s*:/` -> `'$1"$2":'` and then `JSON.parse`. It
+ * is wrong, and wrong in the worst way — it produces a document that parses.
+ *
+ * Harvey's cells carry HTML with quotes and colons in them:
+ *
+ * ```js
+ * t:"color:#70C0FF"
+ * d:"<a href=\"/products/tool-details-24502\">24502</a>"
+ * ```
+ *
+ * `color:` inside that first string is not an object key, and a `{x:1}` in a
+ * product description is not an object. A regex has no way to know, so it
+ * rewrites text inside strings and the result is a scrape that still produces
+ * rows, still passes a row count, and has wrong data in it.
+ *
+ * So this walks the source once, tracking whether it is inside a string and
+ * whether the last character was an escape, and quotes a bare key only when it
+ * is genuinely at an object-key position. That is about sixty lines and it is
+ * the difference between a parser and a coincidence.
+ *
+ * ## Two passes over the same scan
+ *
+ * {@link findLiteral} locates `var <name> =` by index and then brace-matches
+ * forward with the same string-aware rules, so a whole 720 KB document is never
+ * handed to a regex engine more than once per variable. 18 MB of HTML goes
+ * through this code on a full scrape; a repeated whole-document match is what
+ * turns that into minutes of CPU.
+ */
+import { VendorResponseError } from '../../errors.js';
+/** Opening brackets, and the closer each one expects. */
+const CLOSERS = { '{': '}', '[': ']' };
+/** True where `ch` can begin a bare JavaScript identifier. */
+function identifierStart(ch) {
+    return /[A-Za-z_$]/.test(ch);
+}
+/** True where `ch` can continue one. */
+function identifierPart(ch) {
+    return /[A-Za-z0-9_$]/.test(ch);
+}
+/**
+ * The source text of the bracketed value assigned to `var <name>`.
+ *
+ * Returns null when the document declares no such variable, which is an
+ * ordinary answer: Harvey emits `cols1` through `cols10` on every page and
+ * leaves the unused ones empty, and a page with one table simply has no
+ * `tableData2`.
+ *
+ * Throws when the assignment is there but its brackets never balance, because
+ * that is a truncated response rather than an absent one, and returning null
+ * would report it as a page with fewer tables.
+ */
+export function findLiteral(source, name) {
+    const declaration = new RegExp(`\\bvar\\s+${name}\\s*=\\s*`, 'g');
+    const match = declaration.exec(source);
+    if (match === null)
+        return null;
+    // The regex has already consumed the whitespace after `=`, so the literal
+    // starts here or nowhere. Searching forward for the next bracket instead
+    // would happily find one inside the *next* variable's string.
+    const start = match.index + match[0].length;
+    if (source[start] !== '[' && source[start] !== '{') {
+        throw new VendorResponseError(name, 'is assigned something that is not an array or object');
+    }
+    const end = matchBracket(source, start, name);
+    return source.slice(start, end + 1);
+}
+/**
+ * The index of the bracket closing the one at `start`, string-aware.
+ *
+ * The whole reason this is not a depth counter over the raw text: a `}` inside
+ * a `d` cell's HTML would close an object that is still open, and every
+ * following column would land one key to the left.
+ */
+function matchBracket(source, start, what) {
+    const stack = [];
+    let inString = false;
+    let escaped = false;
+    for (let i = start; i < source.length; i++) {
+        const ch = source[i];
+        if (inString) {
+            if (escaped)
+                escaped = false;
+            else if (ch === '\\')
+                escaped = true;
+            else if (ch === '"')
+                inString = false;
+            continue;
+        }
+        if (ch === '"') {
+            inString = true;
+        }
+        else if (ch === '{' || ch === '[') {
+            stack.push(CLOSERS[ch]);
+        }
+        else if (ch === '}' || ch === ']') {
+            const expected = stack.pop();
+            if (expected !== ch) {
+                throw new VendorResponseError(what, `has ${JSON.stringify(ch)} at offset ${i - start} where ` +
+                    `${expected === undefined ? 'nothing was open' : JSON.stringify(expected)} was due`);
+            }
+            if (stack.length === 0)
+                return i;
+        }
+    }
+    throw new VendorResponseError(what, 'is not closed before the end of the document — a truncated response');
+}
+/**
+ * A JavaScript object literal as JSON text: bare keys quoted, everything else
+ * byte for byte.
+ *
+ * Only *keys* are rewritten. A bare word at a value position — `undefined`, a
+ * single-quoted string — is left where it is and `JSON.parse` refuses it, which
+ * is the right outcome: the page changed shape, and inventing a reading for a
+ * token this has never seen is how a scraper starts authoring data.
+ */
+export function toJson(literal, what) {
+    const out = [];
+    // Whether the innermost bracket is an object, and whether the next token in
+    // it is a key. Both are needed: `[{a:1}]` is at key position after `{` but
+    // not after `[`, and `{a:[1,2]}` is not at key position after that comma.
+    const objects = [];
+    let expectKey = false;
+    let inString = false;
+    let escaped = false;
+    for (let i = 0; i < literal.length; i++) {
+        const ch = literal[i];
+        if (inString) {
+            out.push(ch);
+            if (escaped)
+                escaped = false;
+            else if (ch === '\\')
+                escaped = true;
+            else if (ch === '"')
+                inString = false;
+            continue;
+        }
+        if (ch === '"') {
+            inString = true;
+            expectKey = false;
+            out.push(ch);
+            continue;
+        }
+        if (ch === '{' || ch === '[') {
+            objects.push(ch === '{');
+            expectKey = ch === '{';
+            out.push(ch);
+            continue;
+        }
+        if (ch === '}' || ch === ']') {
+            objects.pop();
+            expectKey = false;
+            out.push(ch);
+            continue;
+        }
+        if (ch === ',') {
+            expectKey = objects[objects.length - 1] === true;
+            out.push(ch);
+            continue;
+        }
+        if (ch === ':') {
+            expectKey = false;
+            out.push(ch);
+            continue;
+        }
+        if (/\s/.test(ch)) {
+            out.push(ch);
+            continue;
+        }
+        if (expectKey && identifierStart(ch)) {
+            let end = i;
+            while (end < literal.length && identifierPart(literal[end]))
+                end++;
+            out.push(JSON.stringify(literal.slice(i, end)));
+            i = end - 1;
+            expectKey = false;
+            continue;
+        }
+        if (ch === "'") {
+            // Never seen on a Harvey page, and re-escaping one into a JSON string is
+            // guesswork about what the vendor meant. Refuse loudly instead.
+            throw new VendorResponseError(what, `has a single-quoted string at offset ${i}`);
+        }
+        expectKey = false;
+        out.push(ch);
+    }
+    return out.join('');
+}
+/**
+ * The value of `var <name>` in `source`, parsed.
+ *
+ * Null where the variable is absent — see {@link findLiteral}.
+ */
+export function readLiteral(source, name) {
+    const literal = findLiteral(source, name);
+    if (literal === null)
+        return null;
+    try {
+        return JSON.parse(toJson(literal, name));
+    }
+    catch (error) {
+        throw new VendorResponseError(name, `is not a readable object literal: ${error.message}`);
+    }
+}

package/dist/vendors/harvey/records.d.ts

@@ -0,0 +1,79 @@
+/**
+ * Harvey rows -> {@link ToolRecord}.
+ *
+ * The CSV a Harvey scrape writes holds the vendor's own **display strings** —
+ * `.250 (1/4)`, `1-1/2`, `3 mm`, `-` — rather than parsed numbers, because the
+ * file is the receipt and Harvey's own fractional and metric annotations are
+ * part of what it published. So this module is where a cell becomes a number,
+ * through `value.ts`; the closest precedent in this package is Destiny Tool's
+ * fractional-inch reader, not Kennametal's, whose columns are already decimals.
+ *
+ * ## What Harvey does not publish
+ *
+ * **No second identifier.** One `Tool #` per part, which fills both the material
+ * number and the catalog number on a record — see
+ * `conventions.IDENTITY_DEVIATIONS.harvey`.
+ *
+ * **No carbide grade.** `substrate` comes from the family's `bmc` fact and the
+ * `COATING` column fills `coating`, exactly as Destiny Tool's coating id does.
+ *
+ * **No workpiece-material index a scrape can reach.** Nothing in a variant
+ * table rates a tool to ISO 513 groups, so every Harvey record is labelled
+ * `records.UNSPECIFIED` — *we do not know what this tool is for*, which is a
+ * different claim from Kennametal's swept taps being rated for nothing.
+ *
+ * The per-part page does publish one, and it is **per part and not per family**:
+ * a 192-request probe on 2026-08-29 found `harvey_endmill_005.csv` splitting by
+ * coating column, uncoated tools rated for steel, stainless, cast iron and
+ * titanium where the amorphous-diamond-coated tools of the same geometry are
+ * rated for aluminium, wood and composites alone. That is correct metallurgy —
+ * diamond cannot cut ferrous — and flattening it to one answer per family would
+ * put a diamond-coated end mill under steel. So there is no family fact to
+ * write, and the two keyseat pages titled for a material class do not get one
+ * either: the "For Non - Ferrous Materials" line's own part pages rate it for
+ * steel and stainless steel, so the title does not predict the index.
+ * `docs/HARVEY_PRODUCT_TABLE.md` §1.5 has the measurement and what reaching it
+ * would cost.
+ *
+ * **No corner radius on a ball nose.** A ball family publishes no radius column
+ * at all, so `RE` comes from the family's `profile` fact — see
+ * {@link cornerRadius}. That is a per-family constant with provenance rather
+ * than a string match on a description, because Harvey states the profile once,
+ * in the page title, for the whole product line.
+ */
+import { type BoundFamily, type RecordMappers } from '../../family.js';
+import { type ColumnMap, type ToolRecord } from '../../records.js';
+import { type MapperOptions, type ScrapedRow } from '../../scrape.js';
+/** The `profile` fact value that means a ball nose. Harvey's own word. */
+export declare const BALL_PROFILE = "Ball";
+/**
+ * The corner radius, in priority order.
+ *
+ * 1. The family's own radius column where it has one — `CORNER RADIUS` on the
+ *    corner-radius lines, `RADIUS` on the full-radius keyseat cutters.
+ * 2. `DC / 2` on a ball nose. Harvey publishes no radius column on any of its
+ *    twelve ball families, and the radius of a ball end *is* half the diameter,
+ *    so this is arithmetic rather than a guess — the `profile` fact is what says
+ *    the family is one.
+ * 3. `0` — a real square end — otherwise.
+ *
+ * A mapped column whose cell is blank falls through to 2 or 3 rather than
+ * refusing the row: `RE` is optional on the endmill contract precisely because
+ * a square-end row's blank radius is an answer.
+ */
+export declare function cornerRadius(row: ScrapedRow, family: BoundFamily, columns: ColumnMap, what: string, dc: number, options: MapperOptions): number;
+/**
+ * The flute count.
+ *
+ * One column whichever way the table encoded it: `vendors/harvey/scrape.ts`
+ * writes `FLUTES` from the row's own column on a `TOOL #` table and from the
+ * coating group's sub-label on a matrix one, so nothing downstream has to know
+ * which shape the page used.
+ *
+ * Null is a real answer on the two deburring families, which publish
+ * right- and left-hand tooth counts and no flute count at all.
+ */
+export declare function flutes(row: ScrapedRow, family: BoundFamily, columns: ColumnMap): number | null;
+/** One orderable Harvey tool. */
+export declare function endmillRecord(row: ScrapedRow, family: BoundFamily, columns: ColumnMap, options?: MapperOptions): ToolRecord;
+export declare const RECORD_MAPPERS: RecordMappers;