@toolpath/tool-scraper 0.1.0 → 2.1.0

package/dist/vendors/harvey/records.js

@@ -0,0 +1,163 @@
+/**
+ * Harvey rows -> {@link ToolRecord}.
+ *
+ * The CSV a Harvey scrape writes holds the vendor's own **display strings** —
+ * `.250 (1/4)`, `1-1/2`, `3 mm`, `-` — rather than parsed numbers, because the
+ * file is the receipt and Harvey's own fractional and metric annotations are
+ * part of what it published. So this module is where a cell becomes a number,
+ * through `value.ts`; the closest precedent in this package is Destiny Tool's
+ * fractional-inch reader, not Kennametal's, whose columns are already decimals.
+ *
+ * ## What Harvey does not publish
+ *
+ * **No second identifier.** One `Tool #` per part, which fills both the material
+ * number and the catalog number on a record — see
+ * `conventions.IDENTITY_DEVIATIONS.harvey`.
+ *
+ * **No carbide grade.** `substrate` comes from the family's `bmc` fact and the
+ * `COATING` column fills `coating`, exactly as Destiny Tool's coating id does.
+ *
+ * **No workpiece-material index a scrape can reach.** Nothing in a variant
+ * table rates a tool to ISO 513 groups, so every Harvey record is labelled
+ * `records.UNSPECIFIED` — *we do not know what this tool is for*, which is a
+ * different claim from Kennametal's swept taps being rated for nothing.
+ *
+ * The per-part page does publish one, and it is **per part and not per family**:
+ * a 192-request probe on 2026-08-29 found `harvey_endmill_005.csv` splitting by
+ * coating column, uncoated tools rated for steel, stainless, cast iron and
+ * titanium where the amorphous-diamond-coated tools of the same geometry are
+ * rated for aluminium, wood and composites alone. That is correct metallurgy —
+ * diamond cannot cut ferrous — and flattening it to one answer per family would
+ * put a diamond-coated end mill under steel. So there is no family fact to
+ * write, and the two keyseat pages titled for a material class do not get one
+ * either: the "For Non - Ferrous Materials" line's own part pages rate it for
+ * steel and stainless steel, so the title does not predict the index.
+ * `docs/HARVEY_PRODUCT_TABLE.md` §1.5 has the measurement and what reaching it
+ * would cost.
+ *
+ * **No corner radius on a ball nose.** A ball family publishes no radius column
+ * at all, so `RE` comes from the family's `profile` fact — see
+ * {@link cornerRadius}. That is a per-family constant with provenance rather
+ * than a string match on a description, because Harvey states the profile once,
+ * in the page title, for the whole product line.
+ */
+import { columnReaders } from '../../columns.js';
+import { DESCRIPTION_COLUMN } from '../../conventions.js';
+import { VendorResponseError } from '../../errors.js';
+import { fact, familyBrand } from '../../family.js';
+import { BRANDS } from '../../identity.js';
+import { toolRecord } from '../../records.js';
+import { consoleWarn } from '../../scrape.js';
+import { COATING_COLUMN, TOOL_NUMBER_COLUMN } from './scrape.js';
+import { count, dimension } from './value.js';
+/** The `profile` fact value that means a ball nose. Harvey's own word. */
+export const BALL_PROFILE = 'Ball';
+/**
+ * The family's declared unit, refused rather than asserted when it is absent.
+ *
+ * Every Harvey family declares one — the `harvey` CLI command will not scrape a
+ * family without it — so `familyUnits` would return exactly this one and a
+ * tap's two-system case cannot arise here. It was `family.unit!` until
+ * 2026-08-29, which is the same claim with no check behind it: a family added
+ * without the fact would have read `undefined` straight into
+ * `dimensionalColumn` and asked the CSV for a column called `CUTTER DIA._`.
+ */
+function unitOf(family) {
+    return fact(family, 'unit', family.unit);
+}
+/**
+ * The three column readers, over this vendor's grammar.
+ *
+ * `dimension` is the only Harvey-specific half; everything either side of it —
+ * an unmapped column answering undefined, a required field refusing the row and
+ * quoting the cell — is `columns.columnReaders`, shared with EMUGE-FRANKEN's
+ * mapper. All three take the caller's `columns` rather than `family.columns`:
+ * they are the same object through `registry.toRecords`, but `registry`
+ * validates the argument, and a mapper reading a different reference is
+ * validating one map and reading another. This mapper asserted the parameter
+ * was unused with an underscore until 2026-08-29.
+ */
+const { cell, required, optional } = columnReaders(dimension);
+/**
+ * The corner radius, in priority order.
+ *
+ * 1. The family's own radius column where it has one — `CORNER RADIUS` on the
+ *    corner-radius lines, `RADIUS` on the full-radius keyseat cutters.
+ * 2. `DC / 2` on a ball nose. Harvey publishes no radius column on any of its
+ *    twelve ball families, and the radius of a ball end *is* half the diameter,
+ *    so this is arithmetic rather than a guess — the `profile` fact is what says
+ *    the family is one.
+ * 3. `0` — a real square end — otherwise.
+ *
+ * A mapped column whose cell is blank falls through to 2 or 3 rather than
+ * refusing the row: `RE` is optional on the endmill contract precisely because
+ * a square-end row's blank radius is an answer.
+ */
+export function cornerRadius(row, family, columns, what, dc, options) {
+    const stated = optional(row, columns, 'RE', unitOf(family), what, options);
+    if (stated !== null)
+        return stated;
+    return family.profile === BALL_PROFILE ? dc / 2 : 0;
+}
+/**
+ * The flute count.
+ *
+ * One column whichever way the table encoded it: `vendors/harvey/scrape.ts`
+ * writes `FLUTES` from the row's own column on a `TOOL #` table and from the
+ * coating group's sub-label on a matrix one, so nothing downstream has to know
+ * which shape the page used.
+ *
+ * Null is a real answer on the two deburring families, which publish
+ * right- and left-hand tooth counts and no flute count at all.
+ */
+export function flutes(row, family, columns) {
+    const raw = cell(row, columns, 'NOF', unitOf(family));
+    return raw === undefined ? null : count(raw);
+}
+/** One orderable Harvey tool. */
+export function endmillRecord(row, family, columns, options = {}) {
+    const warn = options.warn ?? consoleWarn;
+    const what = row[TOOL_NUMBER_COLUMN] ?? '';
+    if (what === '') {
+        throw new VendorResponseError(family.id, `has a row with no ${TOOL_NUMBER_COLUMN}`);
+    }
+    const opts = { warn };
+    const unit = unitOf(family);
+    const dc = required(row, columns, 'DC', unit, what, opts);
+    const fluteLength = required(row, columns, 'LCF', unit, what, opts);
+    // Harvey's reach columns are the distance from the tip to the full shank,
+    // which is what `shoulder-length` names. A family with no reach column is a
+    // plain tool whose usable length below the shank is its flute length — the
+    // same convention Destiny Tool's mapper uses.
+    const reach = optional(row, columns, 'shoulder-length', unit, what, opts);
+    const neck = optional(row, columns, 'shoulder-diameter', unit, what, opts);
+    const geometry = {
+        DC: dc,
+        RE: cornerRadius(row, family, columns, what, dc, opts),
+        SFDM: required(row, columns, 'SFDM', unit, what, opts),
+        OAL: required(row, columns, 'OAL', unit, what, opts),
+        LCF: fluteLength,
+        'shoulder-length': reach ?? fluteLength,
+        'shoulder-diameter': neck ?? dc,
+    };
+    const nof = flutes(row, family, columns);
+    if (nof !== null)
+        geometry.NOF = nof;
+    return toolRecord({
+        brand: familyBrand(family),
+        vendor: BRANDS[familyBrand(family)].vendor,
+        materialNumber: what,
+        catalogNumber: what,
+        description: row[DESCRIPTION_COLUMN] ?? '',
+        kind: 'endmill',
+        unit,
+        substrate: fact(family, 'bmc', family.bmc),
+        coating: row[COATING_COLUMN] ?? '',
+        // `materialGroups` and its source are left to the factory's `null`: see the
+        // module docstring for what a Harvey part page publishes and why none of it
+        // can be stated per family.
+        coolantThrough: fact(family, 'coolantThrough', family.coolantThrough),
+        geometry,
+    });
+}
+export const RECORD_MAPPERS = { endmill: endmillRecord };

package/dist/vendors/harvey/scrape.d.ts

@@ -0,0 +1,187 @@
+/**
+ * One Harvey Tool product page -> one row per orderable part.
+ *
+ * The page is server-rendered HTML with its variant data inlined as a
+ * JavaScript literal; `docs/HARVEY_PRODUCT_TABLE.md` records how that was found
+ * and what was tried first. This module is the join between the two halves — a
+ * `<thead>` in the DOM and a `tableData<N>` in a `<script>` — plus the one
+ * structural thing Harvey's table does that no other vendor here does.
+ *
+ * ## One HTML row is up to nine orderable parts
+ *
+ * The table is a matrix. The `a*` cells are the geometry, shared; after them
+ * comes a coating × flute grid where each coating is a `colspan`-ed header and
+ * every non-empty cell under it is a different tool number at the same geometry.
+ * **5,033 HTML rows become 12,799 CSV rows**, and that explosion is the whole
+ * risk in this adapter: a coating or flute count read off the wrong column
+ * produces a CSV that is the right length and wrong throughout.
+ *
+ * ## Harvey checks it for us, 5,033 times per scrape
+ *
+ * Every row carries an `atc` cell whose `j` is the add-to-cart payload, and that
+ * payload independently lists every tool number on the row **in the same order
+ * as the non-empty tool-number cells**:
+ *
+ * ```json
+ * [{"T":"690508","C":"690508","Q":"1"},{"T":"679608","C":"679608","Q":"1"}]
+ * ```
+ *
+ * So the explosion is checked against the vendor's own list on every row rather
+ * than once against a hand count, and a grid that has shifted by one column
+ * cannot pass. That check is {@link checkCartPayload}, and it is the reason the
+ * declared `rows` counts in `families/harvey.ts` being seeded rather than
+ * hand-counted is survivable.
+ *
+ * ## Two columns are synthesised, and neither has a cell to copy
+ *
+ * `Coating` is a fact about which *header* a tool number sits under, and
+ * `FLUTES` is too on the matrix tables. Both are encoded in column position and
+ * nowhere else, which is why they are built here rather than lifted, and why the
+ * coating vocabulary is a closed list in `lexicon.ts`.
+ *
+ * `FLUTES` is one column either way — synthesised from the sub-label on a matrix
+ * table and lifted from the row's own column on a `TOOL #` one — so the vendor's
+ * `FLUTES` column is taken out of the geometry list rather than written twice.
+ *
+ * Everything else in the CSV is the vendor's own display string, verbatim —
+ * `.250 (1/4)` reaches the file as `.250 (1/4)`. `value.ts` is what resolves one
+ * when a record is built.
+ */
+import { type UnitSystem } from '../../conventions.js';
+import type { Fetcher } from '../../fetch.js';
+import { type ScrapeResult, type Warn } from '../../scrape.js';
+import { type HeaderColumn } from './header.js';
+export declare const BASE = "https://www.harveytool.com";
+/** `cols1`..`cols10` and `tableData1`..`tableData10` exist on every page. */
+export declare const MAX_TABLES = 10;
+/** The CSV columns this adapter synthesises or lifts from the page itself. */
+export declare const TOOL_NUMBER_COLUMN = "Tool #";
+export declare const COATING_COLUMN = "Coating";
+export declare const FLUTES_COLUMN = "FLUTES";
+export declare const PRICE_COLUMN = "PRICE_USD";
+/**
+ * The suffix an unlabelled annotation column takes, after the column it
+ * annotates.
+ *
+ * Harvey gives these no header at all — see `docs/HARVEY_PRODUCT_TABLE.md` §5.3
+ * — so a name has to be supplied, and naming one for what it annotates is the
+ * only honest option available. Dropping them instead would lose a published
+ * column; leaving them unnamed would collide with the column they follow.
+ */
+export declare const RATIO_SUFFIX = "RATIO";
+/**
+ * The suffix an unlabelled column takes when it is *not* a ratio.
+ *
+ * Four such columns exist, on two families, and Harvey renders them in white
+ * text: two carry the badge `LONG` on a handful of rows and two are entirely
+ * empty. Naming them `RATIO` would be a claim about their contents that is
+ * false, so which suffix a column gets is decided from the whole page's data
+ * rather than from its header — see {@link ratioColumnKeys}.
+ */
+export declare const NOTE_SUFFIX = "NOTE";
+/** One cell of a `tableData<N>` row. `d` is the display HTML — the value. */
+interface Cell {
+    d?: string | null;
+    j?: string | null;
+}
+type DataRow = Record<string, Cell | undefined>;
+/** One geometry column: which key holds it, and what the CSV calls it. */
+interface GeometryColumn {
+    key: string;
+    /** The CSV column, unit suffix already applied where the label carries one. */
+    column: string;
+    /** Harvey's own top label, before disambiguation. */
+    label: string;
+}
+/** One coating group: its tool-number columns, and the price they share. */
+interface CoatingGroup {
+    coating: string;
+    parts: {
+        key: string;
+        flutes: number | null;
+    }[];
+    /** Every group on all 80 tables has exactly one, and it is last. */
+    priceKey: string;
+}
+/** One table's shape, worked out once and reused for all of its rows. */
+interface TablePlan {
+    geometry: GeometryColumn[];
+    /** The `FLUTES` column's key, where the table publishes one. */
+    flutesKey: string | null;
+    /** True where any group states a flute count in its own sub-label. */
+    matrix: boolean;
+    groups: CoatingGroup[];
+}
+/** `<a href="...">14916</a>` -> `14916`. These cells carry no entities. */
+export declare function cellText(html: string | null | undefined): string;
+/** `$148.40 ` -> `148.40`. The column already says which currency. */
+export declare function priceOf(html: string | null | undefined): string;
+/** Read one table's plan off its `cols<N>` keys and its flattened header. */
+export declare function planTable(family: string, keys: string[], header: HeaderColumn[], unit: UnitSystem, options?: {
+    warn?: Warn;
+    ratioKeys?: ReadonlySet<string>;
+}): TablePlan;
+/** One entry of a row's add-to-cart payload. */
+export interface CartEntry {
+    /** The part number with no footnote marker — what the CSV records. */
+    number: string;
+    /** The same string as printed in the cell, footnote marker and all. */
+    printed: string;
+}
+/**
+ * Every part the row's own add-to-cart payload lists, in cell order.
+ *
+ * `T` and `C` differ on the 36 parts whose printed number carries a footnote
+ * marker: `C` keeps it, `T` does not. Both are read, and each does a different
+ * job — `C` is what the cell says and is therefore what the cell can be checked
+ * against, `T` is Harvey's own clean part number and is therefore what the CSV
+ * records. Deriving the second from the first with a regex would work today and
+ * be this package's guess rather than the vendor's statement.
+ *
+ * `Q` is `"1"` on all 12,799 entries and is not read.
+ */
+export declare function cartEntries(family: string, row: DataRow): CartEntry[];
+/** Refuse an explosion the vendor's own payload does not agree with. */
+export declare function checkCartPayload(family: string, printed: string[], cart: CartEntry[]): void;
+/**
+ * The unlabelled columns of a page whose cells are all ratio annotations.
+ *
+ * Decided across the whole page rather than per table, because the tables merge
+ * into one CSV and a column cannot be named two things. `EndMill-006` is why: a
+ * column is `LONG` on four rows of its first table and empty throughout its
+ * second.
+ */
+export declare function ratioColumnKeys(tables: {
+    keys: string[];
+    rows: DataRow[];
+    header: HeaderColumn[];
+}[]): Set<string>;
+/** Options a page scrape accepts. */
+export interface ScrapeOptions {
+    /** The unit system the family is published in. Decides every `_mm`/`_in`. */
+    unit: UnitSystem;
+    warn?: Warn;
+}
+/**
+ * One product page's HTML -> one row per orderable part.
+ *
+ * Pure, and separate from {@link scrapeProduct} so the whole matrix explosion is
+ * testable from a fixture string with no fetcher anywhere near it.
+ *
+ * A page's tables differ only in published tolerance — which is dropped — so
+ * they merge into one CSV. Their column labels are required to agree, because
+ * two tables that no longer publish the same columns is a page that has been
+ * restructured, and merging them would interleave two different geometries under
+ * one header.
+ */
+export declare function parseProductPage(html: string, source: string, options: ScrapeOptions): ScrapeResult;
+/** One product page's URL. */
+export declare function productUrl(path: string): string;
+/**
+ * Scrape one product page.
+ *
+ * The one network call in this module, and the seam every test replaces —
+ * everything below it parses a string.
+ */
+export declare function scrapeProduct(fetcher: Fetcher, path: string, options: ScrapeOptions): Promise<ScrapeResult>;
+export {};