@toolpath/tool-scraper 2.0.0 → 2.1.0

package/dist/vendors/kennametal/scrape.js

@@ -24,8 +24,12 @@
  * characters.
  */
 import { Parser } from 'htmlparser2';
+import { FAMILY_TITLE_COLUMN } from '../../conventions.js';
 import { VendorResponseError } from '../../errors.js';
 import { BRANDS } from '../../identity.js';
+import { pause, REQUEST_DELAY_MS } from '../../scrape.js';
+import { fetchFamily } from './family.js';
+import { PRODUCT_LINE_COLUMN } from './records.js';
 export const BASE = 'https://www.{host}/us/en/products/fam/_jcr_content/root/' +
     'responsivegrid/{node}.variants.{code}.html' +
     '?query={query}&uom=metric';
@@ -194,16 +198,37 @@ export function columnNames(header) {
  * `tags` is a sequence of `[name, value]` pairs appended to every row as
  * constant columns — used to tag facts the table doesn't state, e.g. the
  * thread system on a tap family.
+ *
+ * `options.familyTitle` adds two more of exactly that kind. They are tags
+ * rather than parsed columns because that is what they are: one string per
+ * family, constant down its whole table, which is the case the `tags` seam was
+ * built for.
  */
-export async function scrapeFamily(fetcher, code, brand = 'kennametal', tags = []) {
+export async function scrapeFamily(fetcher, code, brand = 'kennametal', tags = [], options = {}) {
     const url = variantsUrl(code, brand);
     const { header, rows: dataRows } = parseVariantTable(await fetcher.text(url));
     if (header === null) {
         throw new VendorResponseError(`family ${code}`, 'the vendor returned no variants');
     }
+    // After the table, so a family the vendor no longer publishes fails on the
+    // table it has no rows for rather than on a title nobody would have read.
+    const titled = [];
+    if (options.familyTitle === true) {
+        await pause(options.delayMs ?? REQUEST_DELAY_MS);
+        const { title, productLine } = await fetchFamily(fetcher, code, brand);
+        // A page with no heading writes no columns, rather than a column of empty
+        // strings that reads as a vendor stating an empty name. `unionHeader` is
+        // not in play here — this header is positional — so an absent tag is
+        // simply a narrower CSV.
+        if (title !== '')
+            titled.push([FAMILY_TITLE_COLUMN, title]);
+        if (productLine !== null)
+            titled.push([PRODUCT_LINE_COLUMN, productLine]);
+    }
     const names = columnNames(header);
     const kept = names.filter((name) => name !== null);
-    const csvHeader = [...kept, ...tags.map(([name]) => name)];
+    const allTags = [...tags, ...titled];
+    const csvHeader = [...kept, ...allTags.map(([name]) => name)];
     const rows = dataRows.map((row) => {
         // `dataRows` is filtered to rows exactly as long as the header, and
         // `names` has one entry per header cell, so a length mismatch here means
@@ -218,7 +243,7 @@ export async function scrapeFamily(fetcher, code, brand = 'kennametal', tags = [
             if (name !== null)
                 out[name] = row[index]?.[0] ?? '';
         });
-        for (const [name, value] of tags)
+        for (const [name, value] of allTags)
             out[name] = value;
         return out;
     });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@toolpath/tool-scraper",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "Scrape cutting-tool geometry from vendor catalogs into records and CSVs",
   "license": "MIT",
   "engines": {