@toolpath/tool-scraper 0.1.0

package/dist/node/paths.js

@@ -0,0 +1,80 @@
+/**
+ * Where a scrape lands.
+ *
+ * **Scraped output is never committed.** A CSV is a vendor's data and a
+ * working file, not source, and this repository is public — which is a second
+ * reason, independent of size, to keep it out. Git was carrying the provenance
+ * of those CSVs for free; now that it is not, every scrape writes a
+ * {@link receipts} sidecar beside its file.
+ *
+ * This module is in `node/` rather than beside the config tables: resolving a
+ * root needs `process.env` and a path relative to this file's own location,
+ * and a family table that needed either would be unimportable in anything that
+ * only wants to read records.
+ */
+import { dirname, join, resolve } from 'node:path';
+import { homedir } from 'node:os';
+import { fileURLToPath } from 'node:url';
+import { familyBrand } from '../family.js';
+import { familyConfig } from '../families/index.js';
+/**
+ * Where scraped CSVs are read from and written to.
+ *
+ * Set it when the package is installed rather than run from a checkout: the
+ * default below is derived from this file's own location, which is right in a
+ * working tree and meaningless inside `node_modules`. Every command prints the
+ * resolved root for exactly that reason — a scrape that wrote somewhere
+ * surprising should say so on the way, not be discovered later.
+ */
+export const SCRAPE_ROOT_ENV = 'TOOLPATH_SCRAPE_ROOT';
+/** `packages/tool-scraper/scrape-out`, which `.gitignore` already covers. */
+export const DEFAULT_SCRAPE_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../scrape-out');
+/** `~` expanded, the way a shell would. */
+function expandHome(path) {
+    return path.startsWith('~/') ? join(homedir(), path.slice(2)) : path;
+}
+/** The directory holding every vendor's scraped CSVs. */
+export function scrapeRoot() {
+    const override = process.env[SCRAPE_ROOT_ENV];
+    return override ? resolve(expandHome(override)) : DEFAULT_SCRAPE_ROOT;
+}
+/**
+ * One line naming the resolved root and how it was resolved.
+ *
+ * Printed by every command. The distinction it carries is the one that matters
+ * when a scrape goes somewhere unexpected: whether the path came from the
+ * environment or from this package's own location.
+ */
+export function describeRoot() {
+    const how = process.env[SCRAPE_ROOT_ENV] ? 'set' : 'default';
+    return `scrape root: ${scrapeRoot()} (${SCRAPE_ROOT_ENV} ${how})`;
+}
+/**
+ * Where one vendor's scraped CSVs live — the receipts.
+ *
+ * Per brand rather than per adapter, and the distinction is worth holding on
+ * to: an adapter is a fact about *code*, a scraped table is a fact about who
+ * published it. WIDIA's tables are WIDIA's even though Kennametal's adapter is
+ * what fetched them.
+ */
+export function csvDir(brand) {
+    return join(scrapeRoot(), brand, 'csv');
+}
+/**
+ * One vendor's mirrored STEP models.
+ *
+ * Nothing is redistributed from here: these are a local working copy for
+ * measuring a holder, and only a derived profile is ever meant to leave.
+ */
+export function stepDir(brand) {
+    return join(scrapeRoot(), brand, 'step');
+}
+/**
+ * Where one family's CSV lives, resolved through its own brand.
+ *
+ * Takes a bare CSV name rather than a path, so a caller cannot pass a file
+ * from somewhere else and have it silently treated as this family's receipt.
+ */
+export function familyCsv(name) {
+    return join(csvDir(familyBrand(familyConfig(name))), name);
+}

package/dist/node/receipts.d.ts

@@ -0,0 +1,100 @@
+/**
+ * What a scrape did, written beside what it produced.
+ *
+ * The CSVs are not committed — they are vendor data, and a public repository
+ * is the wrong place for it — so `git log` cannot answer "when was this
+ * scraped, from which URL, under which family code, how many rows". Each
+ * scrape writes a sidecar instead. It is cheap now and effectively impossible
+ * to backfill: nothing in a CSV records the URL it came from, and a re-scrape
+ * a month later answers a different question than the one asked.
+ *
+ * **The row count is the interesting field.** `families/` states how many rows
+ * a human counted at scrape time, and a receipt states how many the scrape
+ * actually wrote. Every other count in a pipeline like this is computed from
+ * the same file it is checking, so a scrape that silently lost rows agrees
+ * with itself; two independently-arrived-at numbers do not.
+ */
+import type { BrandName } from '../identity.js';
+/**
+ * The suffix a receipt takes, beside the CSV it describes.
+ *
+ * A suffix rather than a parallel directory, so a CSV and its receipt cannot
+ * be separated by moving one — and so a directory listing shows immediately
+ * which scrapes have one.
+ */
+export declare const SUFFIX = ".scrape.json";
+/**
+ * This package's version, or `unknown` when it cannot be read.
+ *
+ * Not an error: the package runs perfectly well from a source checkout, and
+ * refusing to record a scrape over a missing version string would be the tail
+ * wagging the dog. `unknown` is the honest value and reads as one.
+ */
+export declare function scraperVersion(): string;
+/** One scrape: what was fetched, from where, when, and how much of it. */
+export interface Receipt {
+    /**
+     * The CSV this describes, by name — so a receipt read on its own says what
+     * it belongs to.
+     */
+    csv: string;
+    brand: BrandName;
+    /**
+     * The URL the rows came from. A request, not a page: this is the thing to
+     * re-issue when a column changes shape, and finding it again is the most
+     * expensive part of adding a vendor.
+     */
+    source: string;
+    /** How many rows were written. See the module docstring. */
+    rows: number;
+    /** UTC, ISO 8601, to the second. */
+    scrapedAt: string;
+    scraper: string;
+    /**
+     * The vendor's own family code where there is one. REGO-FIX and Destiny Tool
+     * have none — their scrape target is a set of index filters — so it is null
+     * rather than an empty string, which would read as a code the vendor left
+     * blank.
+     */
+    familyCode: string | null;
+}
+/** Where the receipt for `csvPath` goes. */
+export declare function pathFor(csvPath: string): string;
+/** What {@link write} needs to record. */
+export interface ReceiptInput {
+    brand: BrandName;
+    source: string;
+    rows: number;
+    familyCode?: string | null;
+    /** The moment to stamp. Injectable so a test is not a clock. */
+    now?: Date;
+}
+/**
+ * Record a scrape beside its CSV, replacing any earlier receipt.
+ *
+ * Replacing rather than appending: a receipt describes the file sitting next
+ * to it, and a history of scrapes that produced files no longer there is a
+ * log, not a receipt.
+ */
+export declare function write(csvPath: string, input: ReceiptInput): string;
+/**
+ * The receipt beside `csvPath`, or null when the scrape predates them.
+ *
+ * Null rather than a throw: a CSV somebody scraped before this existed is
+ * still a usable CSV, and refusing it would be refusing data over its
+ * paperwork.
+ */
+export declare function read(csvPath: string): Receipt | null;
+/**
+ * Refuse a scrape whose row count disagrees with the declared one.
+ *
+ * The one check two independent numbers make possible. A silently truncated
+ * response, a facet that started filtering, a vendor discontinuing half a
+ * family — all of them produce a CSV that parses cleanly and is wrong, and
+ * none of them is visible from the file alone.
+ *
+ * Throwing rather than warning, because the declared count is a human's
+ * statement about the vendor's own page: if the scrape now disagrees with it,
+ * one of the two needs updating and neither can be guessed at from here.
+ */
+export declare function checkRows(family: string, declared: number, receipt: Receipt): void;

package/dist/node/receipts.js

@@ -0,0 +1,107 @@
+/**
+ * What a scrape did, written beside what it produced.
+ *
+ * The CSVs are not committed — they are vendor data, and a public repository
+ * is the wrong place for it — so `git log` cannot answer "when was this
+ * scraped, from which URL, under which family code, how many rows". Each
+ * scrape writes a sidecar instead. It is cheap now and effectively impossible
+ * to backfill: nothing in a CSV records the URL it came from, and a re-scrape
+ * a month later answers a different question than the one asked.
+ *
+ * **The row count is the interesting field.** `families/` states how many rows
+ * a human counted at scrape time, and a receipt states how many the scrape
+ * actually wrote. Every other count in a pipeline like this is computed from
+ * the same file it is checking, so a scrape that silently lost rows agrees
+ * with itself; two independently-arrived-at numbers do not.
+ */
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname, basename, join } from 'node:path';
+import { createRequire } from 'node:module';
+import { VendorResponseError } from '../errors.js';
+import { compare } from '../order.js';
+/**
+ * The suffix a receipt takes, beside the CSV it describes.
+ *
+ * A suffix rather than a parallel directory, so a CSV and its receipt cannot
+ * be separated by moving one — and so a directory listing shows immediately
+ * which scrapes have one.
+ */
+export const SUFFIX = '.scrape.json';
+/**
+ * This package's version, or `unknown` when it cannot be read.
+ *
+ * Not an error: the package runs perfectly well from a source checkout, and
+ * refusing to record a scrape over a missing version string would be the tail
+ * wagging the dog. `unknown` is the honest value and reads as one.
+ */
+export function scraperVersion() {
+    try {
+        const require = createRequire(import.meta.url);
+        const manifest = require('../../package.json');
+        return manifest.version ?? 'unknown';
+    }
+    catch {
+        return 'unknown';
+    }
+}
+/** Where the receipt for `csvPath` goes. */
+export function pathFor(csvPath) {
+    return join(dirname(csvPath), basename(csvPath) + SUFFIX);
+}
+/**
+ * Record a scrape beside its CSV, replacing any earlier receipt.
+ *
+ * Replacing rather than appending: a receipt describes the file sitting next
+ * to it, and a history of scrapes that produced files no longer there is a
+ * log, not a receipt.
+ */
+export function write(csvPath, input) {
+    const stamped = input.now ?? new Date();
+    const receipt = {
+        csv: basename(csvPath),
+        brand: input.brand,
+        source: input.source,
+        rows: input.rows,
+        scrapedAt: `${stamped.toISOString().slice(0, 19)}+00:00`,
+        scraper: scraperVersion(),
+        familyCode: input.familyCode ?? null,
+    };
+    const out = pathFor(csvPath);
+    mkdirSync(dirname(out), { recursive: true });
+    // Sorted keys, so a re-scrape's receipt diffs only where it differs.
+    const ordered = Object.fromEntries(Object.entries(receipt).sort(([a], [b]) => compare(a, b)));
+    writeFileSync(out, `${JSON.stringify(ordered, null, 2)}\n`);
+    return out;
+}
+/**
+ * The receipt beside `csvPath`, or null when the scrape predates them.
+ *
+ * Null rather than a throw: a CSV somebody scraped before this existed is
+ * still a usable CSV, and refusing it would be refusing data over its
+ * paperwork.
+ */
+export function read(csvPath) {
+    const path = pathFor(csvPath);
+    if (!existsSync(path))
+        return null;
+    return JSON.parse(readFileSync(path, 'utf8'));
+}
+/**
+ * Refuse a scrape whose row count disagrees with the declared one.
+ *
+ * The one check two independent numbers make possible. A silently truncated
+ * response, a facet that started filtering, a vendor discontinuing half a
+ * family — all of them produce a CSV that parses cleanly and is wrong, and
+ * none of them is visible from the file alone.
+ *
+ * Throwing rather than warning, because the declared count is a human's
+ * statement about the vendor's own page: if the scrape now disagrees with it,
+ * one of the two needs updating and neither can be guessed at from here.
+ */
+export function checkRows(family, declared, receipt) {
+    if (receipt.rows !== declared) {
+        throw new VendorResponseError(family, `the scrape wrote ${receipt.rows} rows where this family declares ` +
+            `${declared} — either the vendor changed the family or the scrape ` +
+            `lost rows. Re-count and update \`rows\`, or find out which.`);
+    }
+}

package/dist/order.d.ts

@@ -0,0 +1,10 @@
+/**
+ * One string comparator, so every ordered output in this package is ordered
+ * the same way.
+ *
+ * By code unit rather than `localeCompare`: an assumptions document, a receipt
+ * and a scraped row order all have to be byte-stable for a diff or a gate to
+ * read them, and collation is a machine's locale setting rather than a
+ * property of the data.
+ */
+export declare function compare(a: string, b: string): number;

package/dist/order.js

@@ -0,0 +1,12 @@
+/**
+ * One string comparator, so every ordered output in this package is ordered
+ * the same way.
+ *
+ * By code unit rather than `localeCompare`: an assumptions document, a receipt
+ * and a scraped row order all have to be byte-stable for a diff or a gate to
+ * read them, and collation is a machine's locale setting rather than a
+ * property of the data.
+ */
+export function compare(a, b) {
+    return a < b ? -1 : a > b ? 1 : 0;
+}

package/dist/provenance.d.ts

@@ -0,0 +1,125 @@
+/**
+ * Where a per-family fact came from, as data rather than as a comment.
+ *
+ * The rule this enforces: **when a vendor label is unclear, ask** — record the
+ * answer and its date, and never guess and flag it afterwards. It was kept
+ * scrupulously in the source package, and kept in prose: `JG 2026-08-06`,
+ * breadcrumb citations, the three paragraphs deriving the KenDrill point angle
+ * by least squares. All of it invisible to code, and therefore unenforceable
+ * against a stranger who adds a family next month.
+ *
+ * A {@link Fact} is the index into that prose. It carries the value the
+ * pipeline uses plus **how it was arrived at**, which turns the cultural rule
+ * into a gate: an assumed fact with no note, date and initials does not
+ * compile, and a vendor-stated one with no citation does not either. An
+ * assumptions document is generated from the same data, so every guess is on
+ * one page instead of scattered across 700 lines of config comments.
+ *
+ * ## The three sources, and why the distinction is not cosmetic
+ *
+ * - **`vendor-stated`** — the vendor published it. Needs a `cite`: which
+ *   column, breadcrumb, tagline or facet said so, specific enough to re-check
+ *   with one request. `contact: 'face'` cites *"Shank - SK BT Taper Face
+ *   Contact"*.
+ * - **`derived`** — this repo's arithmetic over vendor inputs. Needs a `note`
+ *   saying what was computed from what. The KenDrill 142° point angle is the
+ *   worked example: no page states it, but `L5 = D1 / (2·tan(SIG/2))` over 49
+ *   rows does. A derived fact is checkable.
+ * - **`assumed`** — nobody said it and nothing proves it. Needs a note, a date
+ *   and initials, because the only thing standing behind it is a person on a
+ *   day. These are what an assumptions document exists to list: every place
+ *   the catalog would be wrong if the guess were wrong.
+ *
+ * The prose does not go away. A fact's note is a sentence; the KenDrill
+ * derivation stays in `families/kennametal.ts` and the runbook where it has
+ * room.
+ *
+ * ## What the types do, and what {@link checkFact} still has to
+ *
+ * A union discriminated on `source` rather than one shape with five optional
+ * fields, so "an assumed fact needs `by`" is a compile error at the point
+ * somebody writes the family.
+ *
+ * The gate stays, because a type cannot say a string is non-empty or that it
+ * is shaped `YYYY-MM-DD` — and `cite: ''` would otherwise satisfy the
+ * compiler while citing nothing.
+ */
+/** What a per-family constant can be. */
+export type FactValue = string | number | boolean;
+/**
+ * How a fact was arrived at. Closed, and ordered from strongest to weakest.
+ *
+ * An assumptions document sorts by it, so a reader meets the checkable facts
+ * before the guesses and finishes on the ones only a person stands behind —
+ * `derived` first, then `assumed`. (`vendor-stated` leads the order but is
+ * filtered out of that document entirely; see {@link assumptions}.)
+ */
+export declare const SOURCES: readonly ["vendor-stated", "derived", "assumed"];
+export type FactSource = (typeof SOURCES)[number];
+/**
+ * One per-family constant, with its provenance.
+ *
+ * `value` is what the pipeline uses — the registry projects it onto the family
+ * config under the fact's own name, so readers say `cfg.pointAngle` and never
+ * learn about provenance. That projection is what keeps this from being a
+ * second source of truth: the fact is the only authored copy, and a family
+ * that also set the plain key is refused.
+ */
+export type Fact<T extends FactValue = FactValue> = {
+    readonly value: T;
+    readonly source: 'vendor-stated';
+    /** What the vendor said, and where. */
+    readonly cite: string;
+} | {
+    readonly value: T;
+    readonly source: 'derived';
+    /** What was worked out, from what. */
+    readonly note: string;
+} | {
+    readonly value: T;
+    readonly source: 'assumed';
+    /** What was guessed, and why. */
+    readonly note: string;
+    /** When a person last checked it, `YYYY-MM-DD`. */
+    readonly checked: string;
+    /**
+     * Who. Required for the same reason the date is: an assumption is only
+     * as good as someone being willing to be named beside it.
+     */
+    readonly by: string;
+};
+/**
+ * Refuse a fact that does not carry what its source kind requires.
+ *
+ * Called for every fact of every family when the registry binds, so the
+ * failure names the family and the key rather than surfacing as a thin
+ * assumptions-document row nobody reads.
+ */
+export declare function checkFact(family: string, key: string, fact: Fact): void;
+/** One row of an assumptions document. */
+export interface Assumption {
+    table: string;
+    family: string;
+    key: string;
+    value: FactValue;
+    source: FactSource;
+    note: string;
+    checked: string | null;
+    by: string | null;
+}
+/** A config table's shape, as far as provenance is concerned. */
+export interface FactBearing {
+    facts?: Record<string, Fact>;
+}
+/**
+ * Every non-vendor-stated fact in the catalog, flattened and sorted.
+ *
+ * `vendor-stated` facts are excluded because the document's purpose is the
+ * list of things that would be wrong if somebody guessed wrong — a citation is
+ * a different kind of claim and has its own re-check path (one `curl`).
+ *
+ * Sorted by source then family then key — so the derived facts lead, the
+ * assumed ones close the list, and the output is byte-stable for whatever
+ * gate reads it.
+ */
+export declare function assumptions(tables: Record<string, Record<string, FactBearing>>): Assumption[];

package/dist/provenance.js

@@ -0,0 +1,133 @@
+/**
+ * Where a per-family fact came from, as data rather than as a comment.
+ *
+ * The rule this enforces: **when a vendor label is unclear, ask** — record the
+ * answer and its date, and never guess and flag it afterwards. It was kept
+ * scrupulously in the source package, and kept in prose: `JG 2026-08-06`,
+ * breadcrumb citations, the three paragraphs deriving the KenDrill point angle
+ * by least squares. All of it invisible to code, and therefore unenforceable
+ * against a stranger who adds a family next month.
+ *
+ * A {@link Fact} is the index into that prose. It carries the value the
+ * pipeline uses plus **how it was arrived at**, which turns the cultural rule
+ * into a gate: an assumed fact with no note, date and initials does not
+ * compile, and a vendor-stated one with no citation does not either. An
+ * assumptions document is generated from the same data, so every guess is on
+ * one page instead of scattered across 700 lines of config comments.
+ *
+ * ## The three sources, and why the distinction is not cosmetic
+ *
+ * - **`vendor-stated`** — the vendor published it. Needs a `cite`: which
+ *   column, breadcrumb, tagline or facet said so, specific enough to re-check
+ *   with one request. `contact: 'face'` cites *"Shank - SK BT Taper Face
+ *   Contact"*.
+ * - **`derived`** — this repo's arithmetic over vendor inputs. Needs a `note`
+ *   saying what was computed from what. The KenDrill 142° point angle is the
+ *   worked example: no page states it, but `L5 = D1 / (2·tan(SIG/2))` over 49
+ *   rows does. A derived fact is checkable.
+ * - **`assumed`** — nobody said it and nothing proves it. Needs a note, a date
+ *   and initials, because the only thing standing behind it is a person on a
+ *   day. These are what an assumptions document exists to list: every place
+ *   the catalog would be wrong if the guess were wrong.
+ *
+ * The prose does not go away. A fact's note is a sentence; the KenDrill
+ * derivation stays in `families/kennametal.ts` and the runbook where it has
+ * room.
+ *
+ * ## What the types do, and what {@link checkFact} still has to
+ *
+ * A union discriminated on `source` rather than one shape with five optional
+ * fields, so "an assumed fact needs `by`" is a compile error at the point
+ * somebody writes the family.
+ *
+ * The gate stays, because a type cannot say a string is non-empty or that it
+ * is shaped `YYYY-MM-DD` — and `cite: ''` would otherwise satisfy the
+ * compiler while citing nothing.
+ */
+import { ScraperConfigError } from './errors.js';
+import { compare } from './order.js';
+/**
+ * How a fact was arrived at. Closed, and ordered from strongest to weakest.
+ *
+ * An assumptions document sorts by it, so a reader meets the checkable facts
+ * before the guesses and finishes on the ones only a person stands behind —
+ * `derived` first, then `assumed`. (`vendor-stated` leads the order but is
+ * filtered out of that document entirely; see {@link assumptions}.)
+ */
+export const SOURCES = ['vendor-stated', 'derived', 'assumed'];
+const DATE = /^\d{4}-\d{2}-\d{2}$/;
+/** What a note has to say, per source kind. */
+const WANTED = {
+    derived: 'computed from what',
+    assumed: 'guessed, and why',
+};
+/**
+ * Refuse a fact that does not carry what its source kind requires.
+ *
+ * Called for every fact of every family when the registry binds, so the
+ * failure names the family and the key rather than surfacing as a thin
+ * assumptions-document row nobody reads.
+ */
+export function checkFact(family, key, fact) {
+    const where = `${family}: ${key}`;
+    if (!SOURCES.includes(fact.source)) {
+        throw new ScraperConfigError(where, `source ${JSON.stringify(fact.source)} is not one of ${SOURCES.join(', ')}`);
+    }
+    if (fact.source === 'vendor-stated') {
+        if (!fact.cite) {
+            throw new ScraperConfigError(where, 'a vendor-stated fact needs a `cite` — which column, breadcrumb, ' +
+                'tagline or facet says so');
+        }
+        return;
+    }
+    if (!fact.note) {
+        throw new ScraperConfigError(where, `a ${fact.source} fact needs a \`note\` saying what was ${WANTED[fact.source]}`);
+    }
+    if (fact.source === 'assumed') {
+        // The whole weight of an assumption is a person on a day. Without both, an
+        // assumptions document lists a guess nobody can be asked about.
+        if (!fact.checked || !DATE.test(fact.checked)) {
+            throw new ScraperConfigError(where, `an assumed fact needs \`checked\` as YYYY-MM-DD, not ${JSON.stringify(fact.checked)}`);
+        }
+        if (!fact.by) {
+            throw new ScraperConfigError(where, 'an assumed fact needs `by`');
+        }
+    }
+}
+/**
+ * Every non-vendor-stated fact in the catalog, flattened and sorted.
+ *
+ * `vendor-stated` facts are excluded because the document's purpose is the
+ * list of things that would be wrong if somebody guessed wrong — a citation is
+ * a different kind of claim and has its own re-check path (one `curl`).
+ *
+ * Sorted by source then family then key — so the derived facts lead, the
+ * assumed ones close the list, and the output is byte-stable for whatever
+ * gate reads it.
+ */
+export function assumptions(tables) {
+    const rows = [];
+    for (const [table, families] of Object.entries(tables)) {
+        for (const [family, cfg] of Object.entries(families)) {
+            // Unsorted: the sort below orders the whole list by source, family and
+            // key, which subsumes any order the facts arrive in.
+            for (const [key, fact] of Object.entries(cfg.facts ?? {})) {
+                if (fact.source === 'vendor-stated')
+                    continue;
+                rows.push({
+                    table,
+                    family,
+                    key,
+                    value: fact.value,
+                    source: fact.source,
+                    note: fact.note,
+                    checked: fact.source === 'assumed' ? fact.checked : null,
+                    by: fact.source === 'assumed' ? fact.by : null,
+                });
+            }
+        }
+    }
+    return rows.sort((a, b) => SOURCES.indexOf(a.source) - SOURCES.indexOf(b.source) ||
+        compare(a.family, b.family) ||
+        compare(a.key, b.key));
+}