@toolpath/tool-scraper 0.1.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/README.md +41 -11
  2. package/dist/columns.d.ts +62 -0
  3. package/dist/columns.js +62 -0
  4. package/dist/conventions.d.ts +89 -15
  5. package/dist/conventions.js +108 -15
  6. package/dist/families/emuge.d.ts +183 -0
  7. package/dist/families/emuge.js +159 -0
  8. package/dist/families/harvey.d.ts +1782 -0
  9. package/dist/families/harvey.js +1328 -0
  10. package/dist/families/index.js +5 -2
  11. package/dist/families/kennametal.d.ts +21 -0
  12. package/dist/families/kennametal.js +10 -0
  13. package/dist/families/maritool.d.ts +120 -0
  14. package/dist/families/maritool.js +175 -0
  15. package/dist/family.d.ts +35 -1
  16. package/dist/family.js +28 -0
  17. package/dist/identity.d.ts +18 -0
  18. package/dist/identity.js +46 -0
  19. package/dist/index.d.ts +16 -0
  20. package/dist/index.js +16 -0
  21. package/dist/measure.d.ts +112 -0
  22. package/dist/measure.js +130 -0
  23. package/dist/node/cli.d.ts +3 -0
  24. package/dist/node/cli.js +152 -2
  25. package/dist/node/main.js +0 -0
  26. package/dist/records.d.ts +169 -12
  27. package/dist/records.js +116 -5
  28. package/dist/registry.d.ts +31 -1
  29. package/dist/registry.js +40 -1
  30. package/dist/scrape.d.ts +15 -0
  31. package/dist/scrape.js +24 -0
  32. package/dist/vendors/destinytool/records.d.ts +13 -3
  33. package/dist/vendors/destinytool/records.js +41 -37
  34. package/dist/vendors/emuge/index.d.ts +17 -0
  35. package/dist/vendors/emuge/index.js +17 -0
  36. package/dist/vendors/emuge/records.d.ts +150 -0
  37. package/dist/vendors/emuge/records.js +375 -0
  38. package/dist/vendors/emuge/scrape.d.ts +227 -0
  39. package/dist/vendors/emuge/scrape.js +358 -0
  40. package/dist/vendors/emuge/value.d.ts +75 -0
  41. package/dist/vendors/emuge/value.js +116 -0
  42. package/dist/vendors/harvey/catalog.d.ts +53 -0
  43. package/dist/vendors/harvey/catalog.js +120 -0
  44. package/dist/vendors/harvey/header.d.ts +89 -0
  45. package/dist/vendors/harvey/header.js +185 -0
  46. package/dist/vendors/harvey/index.d.ts +21 -0
  47. package/dist/vendors/harvey/index.js +21 -0
  48. package/dist/vendors/harvey/lexicon.d.ts +73 -0
  49. package/dist/vendors/harvey/lexicon.js +126 -0
  50. package/dist/vendors/harvey/literal.d.ts +68 -0
  51. package/dist/vendors/harvey/literal.js +214 -0
  52. package/dist/vendors/harvey/records.d.ts +79 -0
  53. package/dist/vendors/harvey/records.js +163 -0
  54. package/dist/vendors/harvey/scrape.d.ts +187 -0
  55. package/dist/vendors/harvey/scrape.js +483 -0
  56. package/dist/vendors/harvey/value.d.ts +74 -0
  57. package/dist/vendors/harvey/value.js +119 -0
  58. package/dist/vendors/kennametal/records.d.ts +13 -4
  59. package/dist/vendors/kennametal/records.js +61 -26
  60. package/dist/vendors/maritool/catalog.d.ts +81 -0
  61. package/dist/vendors/maritool/catalog.js +132 -0
  62. package/dist/vendors/maritool/index.d.ts +10 -0
  63. package/dist/vendors/maritool/index.js +10 -0
  64. package/dist/vendors/maritool/scrape.d.ts +297 -0
  65. package/dist/vendors/maritool/scrape.js +593 -0
  66. package/dist/vendors/regofix/scrape.d.ts +8 -11
  67. package/dist/vendors/regofix/scrape.js +21 -36
  68. package/package.json +31 -7
@@ -0,0 +1,74 @@
1
+ /**
2
+ * One display cell -> a number, or an honest reason there is none.
3
+ *
4
+ * Harvey publishes values the way a printed catalog does, because the table is
5
+ * a printed catalog: `.1250 (1/8)` is a decimal with the fraction a machinist
6
+ * orders by beside it, `1-1/2` is a mixed number, `3 mm` is a metric shank on
7
+ * an imperial page, and `-` means the column does not apply to that row.
8
+ *
9
+ * **One rule covers the whole grammar: the value is the leading token, and
10
+ * anything in parentheses is Harvey's own equivalent annotation.** Every shape
11
+ * measured across all 12,799 parts is in `docs/HARVEY_PRODUCT_TABLE.md` §4;
12
+ * this module is that section as code.
13
+ *
14
+ * ## Read `d`, never `v`
15
+ *
16
+ * Each cell also carries `v`, which looks pre-parsed and is the wrong field: it
17
+ * changes unit basis *within a single row*. On the metric ball page, `v` is the
18
+ * inch equivalent for the cutter diameter and the raw millimetre number for the
19
+ * length of cut. Anything reading it gets a clean conversion with wrong numbers
20
+ * in it — the exact failure `conventions.UNIT_SUFFIX` exists to prevent.
21
+ *
22
+ * ## A stated unit wins over the column's
23
+ *
24
+ * 46 cells across six otherwise-imperial families state a metric value outright
25
+ * — `3 mm` in a `SHANK DIAMETER` column on an inch page, a metric-shank tool
26
+ * listed among imperial ones. {@link dimension} converts those into the family's
27
+ * declared unit and warns, naming the part. It does **not** treat a
28
+ * parenthesised `(3 mm)` that way: that is an annotation on an inch value, and
29
+ * converting it would turn `.1181 in` into `3 in`.
30
+ */
31
+ import type { UnitSystem } from '../../conventions.js';
32
+ import { type Warn } from '../../scrape.js';
33
+ /** What one display cell says. */
34
+ export interface HarveyValue {
35
+ /** The leading numeric token, in whatever {@link stated} says. */
36
+ readonly value: number | null;
37
+ /** The unit the token states outright. Null where it states none. */
38
+ readonly stated: UnitSystem | null;
39
+ /** True where the token is an angle in degrees rather than a length. */
40
+ readonly degrees: boolean;
41
+ /** Harvey's parenthesised equivalent, verbatim — `1/8`, `3 mm`, `N.P.T.`. */
42
+ readonly annotation: string | null;
43
+ /** The multiplier a `(1.5x)` cell states, and nothing else. */
44
+ readonly ratio: number | null;
45
+ /** Text with no numeric reading at all: `I`, `II`, `III`, `LONG`. */
46
+ readonly code: string | null;
47
+ }
48
+ /**
49
+ * One cell's display text, read.
50
+ *
51
+ * An empty cell and a `-` both come back as {@link NOTHING}. They mean slightly
52
+ * different things to a reader of the catalog — "nothing published" and "does
53
+ * not apply" — and the same thing to anything building a record, which is that
54
+ * there is no number here and 0 is not a substitute for one.
55
+ */
56
+ export declare function parseValue(display: string): HarveyValue;
57
+ /**
58
+ * One cell as a length in `unit`, or null where it publishes none.
59
+ *
60
+ * `measure.asLength` makes both calls — convert-and-warn a cell whose stated
61
+ * unit disagrees with the family's, refuse an angle in a dimensional column —
62
+ * because they are the same two calls for every vendor and were a verbatim copy
63
+ * here and in `vendors/emuge/value.ts` until 2026-09-01. What is Harvey's is
64
+ * above: that `degrees` is a field of its own rather than a member of
65
+ * `stated`, and that a parenthesised `(3 mm)` is an annotation on an inch value
66
+ * and never the value itself.
67
+ *
68
+ * The 46 cells this converts are metric-shank tools listed among imperial ones
69
+ * — real parts somebody can order, which is why they are converted rather than
70
+ * dropped.
71
+ */
72
+ export declare function dimension(display: string, unit: UnitSystem, what: string, warn?: Warn): number | null;
73
+ /** One cell as a whole count — a flute or tooth number. Null where blank. */
74
+ export declare function count(display: string): number | null;
@@ -0,0 +1,119 @@
1
+ /**
2
+ * One display cell -> a number, or an honest reason there is none.
3
+ *
4
+ * Harvey publishes values the way a printed catalog does, because the table is
5
+ * a printed catalog: `.1250 (1/8)` is a decimal with the fraction a machinist
6
+ * orders by beside it, `1-1/2` is a mixed number, `3 mm` is a metric shank on
7
+ * an imperial page, and `-` means the column does not apply to that row.
8
+ *
9
+ * **One rule covers the whole grammar: the value is the leading token, and
10
+ * anything in parentheses is Harvey's own equivalent annotation.** Every shape
11
+ * measured across all 12,799 parts is in `docs/HARVEY_PRODUCT_TABLE.md` §4;
12
+ * this module is that section as code.
13
+ *
14
+ * ## Read `d`, never `v`
15
+ *
16
+ * Each cell also carries `v`, which looks pre-parsed and is the wrong field: it
17
+ * changes unit basis *within a single row*. On the metric ball page, `v` is the
18
+ * inch equivalent for the cutter diameter and the raw millimetre number for the
19
+ * length of cut. Anything reading it gets a clean conversion with wrong numbers
20
+ * in it — the exact failure `conventions.UNIT_SUFFIX` exists to prevent.
21
+ *
22
+ * ## A stated unit wins over the column's
23
+ *
24
+ * 46 cells across six otherwise-imperial families state a metric value outright
25
+ * — `3 mm` in a `SHANK DIAMETER` column on an inch page, a metric-shank tool
26
+ * listed among imperial ones. {@link dimension} converts those into the family's
27
+ * declared unit and warns, naming the part. It does **not** treat a
28
+ * parenthesised `(3 mm)` that way: that is an annotation on an inch value, and
29
+ * converting it would turn `.1181 in` into `3 in`.
30
+ */
31
+ import { asCount, asLength, fractionValue } from '../../measure.js';
32
+ import { consoleWarn } from '../../scrape.js';
33
+ /**
34
+ * Footnote references Harvey appends to a value or a tool number. 62 cells
35
+ * carry one; each points at a note printed under the table, and none of them is
36
+ * part of the number.
37
+ */
38
+ const FOOTNOTE = /[*!†]+$/;
39
+ /** Harvey's own equivalent, in parentheses at the end: `(1/8)`, `(3 mm)`. */
40
+ const ANNOTATION = /\s*\(([^)]*)\)\s*$/;
41
+ /** `(1.5x)`, `(30x)` — the vendor's reach- or length-to-diameter ratio. */
42
+ const RATIO = /^(\d*\.?\d+)x$/;
43
+ /**
44
+ * A mixed number, a simple fraction or a decimal, with an optional unit.
45
+ *
46
+ * The alternation is ordered longest-first on purpose: `\d*\.?\d+` alone
47
+ * matches the `3` of `3-3/4` and would leave the rest unread, which parses a
48
+ * 3.75-inch tool as a 3-inch one.
49
+ */
50
+ const NUMBER = /^(\d+-\d+\/\d+|\d*\.?\d+\/\d+|\d*\.?\d+)\s*(mm|°)?$/;
51
+ const NOTHING = {
52
+ value: null,
53
+ stated: null,
54
+ degrees: false,
55
+ annotation: null,
56
+ ratio: null,
57
+ code: null,
58
+ };
59
+ /**
60
+ * One cell's display text, read.
61
+ *
62
+ * An empty cell and a `-` both come back as {@link NOTHING}. They mean slightly
63
+ * different things to a reader of the catalog — "nothing published" and "does
64
+ * not apply" — and the same thing to anything building a record, which is that
65
+ * there is no number here and 0 is not a substitute for one.
66
+ */
67
+ export function parseValue(display) {
68
+ const text = display.trim().replace(FOOTNOTE, '').trim();
69
+ if (text === '' || text === '-')
70
+ return NOTHING;
71
+ const annotated = ANNOTATION.exec(text);
72
+ const annotation = annotated?.[1]?.trim() ?? null;
73
+ const head = annotated ? text.slice(0, annotated.index).trim() : text;
74
+ if (head === '') {
75
+ const ratio = annotation === null ? null : RATIO.exec(annotation);
76
+ if (ratio)
77
+ return { ...NOTHING, ratio: Number(ratio[1]) };
78
+ return { ...NOTHING, annotation };
79
+ }
80
+ const number = NUMBER.exec(head);
81
+ // `measure.fractionValue` is stricter than {@link NUMBER} in one place — it
82
+ // refuses a division by zero — so a token that matched here can still have no
83
+ // reading, and that is a code cell rather than a number.
84
+ const value = number === null ? null : fractionValue(number[1]);
85
+ if (number === null || value === null)
86
+ return { ...NOTHING, annotation, code: head };
87
+ const unit = number[2];
88
+ return {
89
+ value,
90
+ stated: unit === 'mm' ? 'millimeters' : null,
91
+ degrees: unit === '°',
92
+ annotation,
93
+ ratio: null,
94
+ code: null,
95
+ };
96
+ }
97
+ /**
98
+ * One cell as a length in `unit`, or null where it publishes none.
99
+ *
100
+ * `measure.asLength` makes both calls — convert-and-warn a cell whose stated
101
+ * unit disagrees with the family's, refuse an angle in a dimensional column —
102
+ * because they are the same two calls for every vendor and were a verbatim copy
103
+ * here and in `vendors/emuge/value.ts` until 2026-09-01. What is Harvey's is
104
+ * above: that `degrees` is a field of its own rather than a member of
105
+ * `stated`, and that a parenthesised `(3 mm)` is an annotation on an inch value
106
+ * and never the value itself.
107
+ *
108
+ * The 46 cells this converts are metric-shank tools listed among imperial ones
109
+ * — real parts somebody can order, which is why they are converted rather than
110
+ * dropped.
111
+ */
112
+ export function dimension(display, unit, what, warn = consoleWarn) {
113
+ const { value, stated, degrees } = parseValue(display);
114
+ return asLength({ value, stated: degrees ? 'degrees' : stated }, display, unit, what, warn);
115
+ }
116
+ /** One cell as a whole count — a flute or tooth number. Null where blank. */
117
+ export function count(display) {
118
+ return asCount(parseValue(display));
119
+ }
@@ -15,10 +15,11 @@
15
15
  * Number` are Kennametal's header text — the labels
16
16
  * `conventions.IDENTITY_COLUMNS` took as the convention because this vendor
17
17
  * was first, and which Destiny Tool then did not follow.
18
- * - **Which column is the grade.** A drill and an end mill carry a carbide
19
- * `Grade`; a tap has no carbide grade and carries `Coating`, the surface
20
- * treatment, in the record's `grade` field. That is Kennametal's table
21
- * shape, not a rule about taps everywhere.
18
+ * - **Which column is the coating.** `Coating` on every table that publishes
19
+ * one, which is every tap table and some of the others. The carbide `Grade`
20
+ * a drill and an end mill table also carry reaches no record: it is what the
21
+ * tool is made of in Kennametal's own vocabulary, nothing downstream reads
22
+ * it, and `substrate` already carries the cutting material as a fact.
22
23
  * - **That a tap's unit system is per row.** `Thread System` is a constant tag
23
24
  * column this package appends at scrape time, and a metric and an inch tap
24
25
  * can sit in one family — so a tap's `unit` is read per row where a drill's
@@ -43,6 +44,14 @@ import type { ScrapedRow } from '../../scrape.js';
43
44
  */
44
45
  export declare const MATERIAL_NUMBER = "Material Number";
45
46
  export declare const CATALOG_NUMBER = "ISO Catalog Number";
47
+ /**
48
+ * The surface-treatment column, on the tables that publish one.
49
+ *
50
+ * Every tap table does; a drill or an end mill table may not, which is why the
51
+ * mappers read it with a `?? ''` rather than through a required column — an
52
+ * absent coating column is a table that states no coating, not a scrape fault.
53
+ */
54
+ export declare const COATING = "Coating";
46
55
  /**
47
56
  * A drill, in the family's native unit system per its `unit` fact.
48
57
  *
@@ -15,10 +15,11 @@
15
15
  * Number` are Kennametal's header text — the labels
16
16
  * `conventions.IDENTITY_COLUMNS` took as the convention because this vendor
17
17
  * was first, and which Destiny Tool then did not follow.
18
- * - **Which column is the grade.** A drill and an end mill carry a carbide
19
- * `Grade`; a tap has no carbide grade and carries `Coating`, the surface
20
- * treatment, in the record's `grade` field. That is Kennametal's table
21
- * shape, not a rule about taps everywhere.
18
+ * - **Which column is the coating.** `Coating` on every table that publishes
19
+ * one, which is every tap table and some of the others. The carbide `Grade`
20
+ * a drill and an end mill table also carry reaches no record: it is what the
21
+ * tool is made of in Kennametal's own vocabulary, nothing downstream reads
22
+ * it, and `substrate` already carries the cutting material as a fact.
22
23
  * - **That a tap's unit system is per row.** `Thread System` is a constant tag
23
24
  * column this package appends at scrape time, and a metric and an inch tap
24
25
  * can sit in one family — so a tap's `unit` is read per row where a drill's
@@ -34,10 +35,10 @@
34
35
  * nose, `LB` and `assemblyGaugeLength` being `OAL` on a bare tool, and every
35
36
  * cutting preset. Those are true of a tool, not of a table.
36
37
  */
37
- import { ScraperConfigError, VendorResponseError } from '../../errors.js';
38
- import { familyBrand } from '../../family.js';
38
+ import { VendorResponseError } from '../../errors.js';
39
+ import { fact, familyBrand } from '../../family.js';
39
40
  import { BRANDS } from '../../identity.js';
40
- import { toolRecord } from '../../records.js';
41
+ import { toolRecord, UNSPECIFIED, } from '../../records.js';
41
42
  import { threadMajorDiameter } from '../../thread.js';
42
43
  import { MATERIALS_COLUMN, materialClasses } from './materials.js';
43
44
  /**
@@ -46,6 +47,14 @@ import { MATERIALS_COLUMN, materialClasses } from './materials.js';
46
47
  */
47
48
  export const MATERIAL_NUMBER = 'Material Number';
48
49
  export const CATALOG_NUMBER = 'ISO Catalog Number';
50
+ /**
51
+ * The surface-treatment column, on the tables that publish one.
52
+ *
53
+ * Every tap table does; a drill or an end mill table may not, which is why the
54
+ * mappers read it with a `?? ''` rather than through a required column — an
55
+ * absent coating column is a table that states no coating, not a scrape fault.
56
+ */
57
+ export const COATING = 'Coating';
49
58
  /**
50
59
  * One canonical dimension, or null when this family maps or publishes none.
51
60
  *
@@ -79,12 +88,30 @@ function require_(row, columns, canonical, unit, what) {
79
88
  }
80
89
  return value;
81
90
  }
82
- /** A per-family constant a mapper cannot proceed without. */
83
- function fact(family, key, value) {
84
- if (value === undefined) {
85
- throw new ScraperConfigError(family.id, `a ${family.kind} family must state ${key} as a fact`);
86
- }
87
- return value;
91
+ /**
92
+ * The workpiece-material groups, and how they were arrived at.
93
+ *
94
+ * **The distinction the column cannot make on its own is present versus
95
+ * blank.** The material groups are not scraped with the variant table: they
96
+ * come from a second CLI step (`toolpath-scrape materials`) that sweeps the
97
+ * `workpieceMaterialDetail` facet and writes {@link MATERIALS_COLUMN} into the
98
+ * CSV. A family that step never ran over has **no such column**, and
99
+ * `node/csv.parseCsv` fills `''` only for cells under a column that is in the
100
+ * header — so the cell is `undefined` on an unswept CSV and `''` on a swept
101
+ * one whose part the vendor rates for nothing.
102
+ *
103
+ * Those are different claims and the record now keeps them apart: an
104
+ * `unspecified` label says this package has no evidence, `[]` says the vendor's
105
+ * own index has none. All
106
+ * 129 taps land on the second after a sweep — Kennametal indexes no tap by
107
+ * workpiece material at all — and reading that as "unconstrained" would put
108
+ * every tap under every material.
109
+ */
110
+ function materialGroups(row) {
111
+ const cell = row[MATERIALS_COLUMN];
112
+ if (cell === undefined)
113
+ return { materialGroups: null, materialGroupsSource: UNSPECIFIED };
114
+ return { materialGroups: materialClasses(cell), materialGroupsSource: 'vendor-stated' };
88
115
  }
89
116
  /**
90
117
  * The `Thread System` tag, refusing anything that is not one of the two.
@@ -129,15 +156,19 @@ export function drillRecord(row, family, columns) {
129
156
  const unit = fact(family, 'unit', family.unit);
130
157
  const what = row[MATERIAL_NUMBER] ?? '';
131
158
  return toolRecord({
159
+ brand: familyBrand(family),
132
160
  vendor: BRANDS[familyBrand(family)].vendor,
133
161
  materialNumber: what,
134
162
  catalogNumber: row[CATALOG_NUMBER] ?? '',
135
- description: row[CATALOG_NUMBER] ?? '',
163
+ // Kennametal publishes no description column on any table. The catalog
164
+ // number is already on the record, and repeating it here said nothing —
165
+ // see `records.ToolRecord.description`.
166
+ description: '',
136
167
  kind: 'drill',
137
168
  unit,
138
169
  substrate: fact(family, 'bmc', family.bmc),
139
- grade: row['Grade'] ?? '',
140
- materialGroups: materialClasses(row[MATERIALS_COLUMN]),
170
+ coating: row[COATING] ?? '',
171
+ ...materialGroups(row),
141
172
  coolantThrough: fact(family, 'coolantThrough', family.coolantThrough),
142
173
  nonFerrous: fact(family, 'nonFerrous', family.nonFerrous),
143
174
  geometry: {
@@ -170,19 +201,21 @@ export function tapRecord(row, family, columns) {
170
201
  const tdz = row['D1-TDZ'] ?? '';
171
202
  const what = row[MATERIAL_NUMBER] ?? '';
172
203
  return toolRecord({
204
+ brand: familyBrand(family),
173
205
  vendor: BRANDS[familyBrand(family)].vendor,
174
206
  materialNumber: what,
175
207
  catalogNumber: row[CATALOG_NUMBER] ?? '',
176
- // The designation is part of what a tap *is*, and the catalog number alone
177
- // does not carry the size.
178
- description: `${row[CATALOG_NUMBER] ?? ''} ${tdz}`,
208
+ // The one Kennametal table that publishes per-part text: the thread
209
+ // designation is part of what a tap *is*, and the catalog number does not
210
+ // carry the size. The catalog number no longer leads it — it is already on
211
+ // the record, and a description that restates one is not a description.
212
+ description: tdz,
179
213
  kind: 'tap',
180
214
  unit,
181
215
  substrate: fact(family, 'bmc', family.bmc),
182
- // A tap has no carbide grade; the record's grade carries the coating.
183
- grade: row['Coating'] ?? '',
184
- materialGroups: materialClasses(row[MATERIALS_COLUMN]),
185
- coolantThrough: false,
216
+ coating: row[COATING] ?? '',
217
+ ...materialGroups(row),
218
+ coolantThrough: fact(family, 'coolantThrough', family.coolantThrough),
186
219
  geometry: {
187
220
  DC: threadMajorDiameter(tdz, system),
188
221
  TP: require_(row, columns, 'TP', unit, what),
@@ -212,15 +245,17 @@ export function endmillRecord(row, family, columns) {
212
245
  const dc = require_(row, columns, 'DC', unit, what);
213
246
  const fluteLength = require_(row, columns, 'LCF', unit, what);
214
247
  return toolRecord({
248
+ brand: familyBrand(family),
215
249
  vendor: BRANDS[familyBrand(family)].vendor,
216
250
  materialNumber: what,
217
251
  catalogNumber: row[CATALOG_NUMBER] ?? '',
218
- description: row[CATALOG_NUMBER] ?? '',
252
+ // No description column here either. See the drill mapper.
253
+ description: '',
219
254
  kind: 'endmill',
220
255
  unit,
221
256
  substrate: fact(family, 'bmc', family.bmc),
222
- grade: row['Grade'] ?? '',
223
- materialGroups: materialClasses(row[MATERIALS_COLUMN]),
257
+ coating: row[COATING] ?? '',
258
+ ...materialGroups(row),
224
259
  coolantThrough: fact(family, 'coolantThrough', family.coolantThrough),
225
260
  geometry: {
226
261
  DC: dc,
@@ -0,0 +1,81 @@
1
+ /**
2
+ * The category tree -> the leaves under it.
3
+ *
4
+ * MariTool publishes no sitemap: `/sitemap.xml` answers with a meta-refresh
5
+ * shell rather than a document, and there is no flat index anywhere on the
6
+ * site. Walking the category tree is the only way to find out what exists —
7
+ * see `docs/MARITOOL_CATALOG.md` §2.
8
+ *
9
+ * **Nothing in a scrape needs this.** `families/maritool.ts` names all 41 leaf
10
+ * cPaths in scope, and a scrape fetches exactly those; re-deriving them at
11
+ * scrape time would put a 199-page crawl in front of every run to rediscover
12
+ * a list that is already written down. It is here for the maintenance question
13
+ * that table cannot answer — *has MariTool added or moved a leaf?* — and the
14
+ * CLI exposes it as `maritool --catalog` for that purpose. It is how the table
15
+ * was built and how it gets rechecked.
16
+ *
17
+ * ## The tree is deeper than it looks
18
+ *
19
+ * Two to four levels, and **no root carries products itself**. `Dual Contact
20
+ * CAT40` (`c23_25_432`) and each of the nine HSK sizes are intermediate nodes
21
+ * with children, so a walk that stopped at depth 2 would find zero HSK holders
22
+ * and zero dual-contact parts — 187 of the 529 in scope. That is why this
23
+ * recurses on the child links rather than reading one level of the sidebar.
24
+ *
25
+ * A category page carries its children as links whose cPath extends its own by
26
+ * exactly one segment, and the sidebar renders unrelated branches beside them,
27
+ * so the prefix test is what separates a child from a sibling of an ancestor.
28
+ */
29
+ import type { Fetcher } from '../../fetch.js';
30
+ import { type Warn } from '../../scrape.js';
31
+ /**
32
+ * The five taper trees this adapter covers.
33
+ *
34
+ * MariTool's catalog is larger — `c23` holds twenty more roots, and the store
35
+ * sells cutting tools besides. These are the trees asked for, and adding one
36
+ * is a decision rather than a widening of a pattern; see the package's rule on
37
+ * not adding vendor scope without being asked.
38
+ *
39
+ * **There is no CAT30.** MariTool's 30-taper is BT30 and ISO30, confirmed
40
+ * against the full sidebar rather than against a landing page. ISO30 (`c23_54`)
41
+ * is absent for a different reason: its three ER parts publish no spec table,
42
+ * so the family would be a receipt of nothing.
43
+ */
44
+ export declare const CATEGORY_ROOTS: readonly string[];
45
+ /** One node of the tree, as the walk found it. */
46
+ export interface Category {
47
+ /** MariTool's own category path, e.g. `23_25_42`. */
48
+ readonly cPath: string;
49
+ /** The vendor's own name for it, off the page title. */
50
+ readonly name: string;
51
+ /** Child cPaths, sorted. Empty on a leaf. */
52
+ readonly children: readonly string[];
53
+ /** What `(of N products)` said, or 0 where the page states no count. */
54
+ readonly products: number;
55
+ }
56
+ /** The category name a page states, stripped of the store's boilerplate. */
57
+ export declare function categoryName(html: string): string;
58
+ /**
59
+ * One category page: its children, its name and its own product count.
60
+ *
61
+ * A child is a link whose cPath extends this one by exactly one segment. The
62
+ * page also links every sibling of every ancestor — the sidebar renders the
63
+ * whole open branch — so taking every `cPath` link would walk the catalog from
64
+ * any starting point and take the twenty out-of-scope roots with it.
65
+ */
66
+ export declare function parseCategory(html: string, cPath: string): Category;
67
+ /**
68
+ * Every category reachable from `roots`, breadth first and paced.
69
+ *
70
+ * Sequential and paced by the package's shared politeness delay. The five
71
+ * roots in scope reach 199 categories, which is a three-minute walk and is run
72
+ * by hand rather than by a scrape.
73
+ */
74
+ export declare function discoverCategories(fetcher: Fetcher, roots?: readonly string[], options?: {
75
+ warn?: Warn;
76
+ delayMs?: number;
77
+ }): Promise<Category[]>;
78
+ /** The leaves of a walk: the categories that carry products themselves. */
79
+ export declare function leavesOf(categories: readonly Category[]): Category[];
80
+ /** One line per category, for a human reading a `--catalog` run. */
81
+ export declare function describe(category: Category): string;
@@ -0,0 +1,132 @@
1
+ /**
2
+ * The category tree -> the leaves under it.
3
+ *
4
+ * MariTool publishes no sitemap: `/sitemap.xml` answers with a meta-refresh
5
+ * shell rather than a document, and there is no flat index anywhere on the
6
+ * site. Walking the category tree is the only way to find out what exists —
7
+ * see `docs/MARITOOL_CATALOG.md` §2.
8
+ *
9
+ * **Nothing in a scrape needs this.** `families/maritool.ts` names all 41 leaf
10
+ * cPaths in scope, and a scrape fetches exactly those; re-deriving them at
11
+ * scrape time would put a 199-page crawl in front of every run to rediscover
12
+ * a list that is already written down. It is here for the maintenance question
13
+ * that table cannot answer — *has MariTool added or moved a leaf?* — and the
14
+ * CLI exposes it as `maritool --catalog` for that purpose. It is how the table
15
+ * was built and how it gets rechecked.
16
+ *
17
+ * ## The tree is deeper than it looks
18
+ *
19
+ * Two to four levels, and **no root carries products itself**. `Dual Contact
20
+ * CAT40` (`c23_25_432`) and each of the nine HSK sizes are intermediate nodes
21
+ * with children, so a walk that stopped at depth 2 would find zero HSK holders
22
+ * and zero dual-contact parts — 187 of the 529 in scope. That is why this
23
+ * recurses on the child links rather than reading one level of the sidebar.
24
+ *
25
+ * A category page carries its children as links whose cPath extends its own by
26
+ * exactly one segment, and the sidebar renders unrelated branches beside them,
27
+ * so the prefix test is what separates a child from a sibling of an ancestor.
28
+ */
29
+ import { Parser } from 'htmlparser2';
30
+ import { compare } from '../../order.js';
31
+ import { consoleWarn, pause, REQUEST_DELAY_MS } from '../../scrape.js';
32
+ import { BASE, categoryUrl } from './scrape.js';
33
+ /**
34
+ * The five taper trees this adapter covers.
35
+ *
36
+ * MariTool's catalog is larger — `c23` holds twenty more roots, and the store
37
+ * sells cutting tools besides. These are the trees asked for, and adding one
38
+ * is a decision rather than a widening of a pattern; see the package's rule on
39
+ * not adding vendor scope without being asked.
40
+ *
41
+ * **There is no CAT30.** MariTool's 30-taper is BT30 and ISO30, confirmed
42
+ * against the full sidebar rather than against a landing page. ISO30 (`c23_54`)
43
+ * is absent for a different reason: its three ER parts publish no spec table,
44
+ * so the family would be a receipt of nothing.
45
+ */
46
+ export const CATEGORY_ROOTS = [
47
+ '23_25', // CAT40
48
+ '23_24', // CAT50
49
+ '23_33', // BT30
50
+ '23_26', // BT40
51
+ '23_46', // HSK
52
+ ];
53
+ /** `.../c23_25_42/index.html` and `index.php?cPath=23_25_42` both give `23_25_42`. */
54
+ const CATEGORY_HREF = /(?:\/c|[?&]cPath=)(\d+(?:_\d+)*)(?:\/index\.html)?(?:[&#]|$)/;
55
+ /** `<title>… Accessories ER Collet Chucks - MariTool</title>` -> the middle. */
56
+ const TITLE = /<title>([^<]*)<\/title>/i;
57
+ /** The vendor's own boilerplate around every category name. */
58
+ const TITLE_NOISE = [/^Tool Holders, Collets and Machine Accessories\s*/i, /\s*-\s*MariTool$/i];
59
+ /** `(of 51 products)`, or 0 — an empty category renders no count at all. */
60
+ const PRODUCT_COUNT = /\(of\s*<b>\s*(\d+)\s*<\/b>\s*products\)/i;
61
+ /** The category name a page states, stripped of the store's boilerplate. */
62
+ export function categoryName(html) {
63
+ let name = TITLE.exec(html)?.[1] ?? '';
64
+ for (const noise of TITLE_NOISE)
65
+ name = name.replace(noise, '');
66
+ return name.replace(/\s+/g, ' ').trim();
67
+ }
68
+ /**
69
+ * One category page: its children, its name and its own product count.
70
+ *
71
+ * A child is a link whose cPath extends this one by exactly one segment. The
72
+ * page also links every sibling of every ancestor — the sidebar renders the
73
+ * whole open branch — so taking every `cPath` link would walk the catalog from
74
+ * any starting point and take the twenty out-of-scope roots with it.
75
+ */
76
+ export function parseCategory(html, cPath) {
77
+ const children = new Set();
78
+ const parser = new Parser({
79
+ onopentag: (tag, attribs) => {
80
+ if (tag !== 'a')
81
+ return;
82
+ const found = CATEGORY_HREF.exec(attribs['href'] ?? '')?.[1];
83
+ if (found === undefined)
84
+ return;
85
+ if (found.startsWith(`${cPath}_`) &&
86
+ found.split('_').length === cPath.split('_').length + 1)
87
+ children.add(found);
88
+ },
89
+ }, { decodeEntities: true });
90
+ parser.write(html);
91
+ parser.end();
92
+ return {
93
+ cPath,
94
+ name: categoryName(html),
95
+ children: [...children].sort(compare),
96
+ products: Number(PRODUCT_COUNT.exec(html)?.[1] ?? 0),
97
+ };
98
+ }
99
+ /**
100
+ * Every category reachable from `roots`, breadth first and paced.
101
+ *
102
+ * Sequential and paced by the package's shared politeness delay. The five
103
+ * roots in scope reach 199 categories, which is a three-minute walk and is run
104
+ * by hand rather than by a scrape.
105
+ */
106
+ export async function discoverCategories(fetcher, roots = CATEGORY_ROOTS, options = {}) {
107
+ const { warn = consoleWarn, delayMs = REQUEST_DELAY_MS } = options;
108
+ const found = new Map();
109
+ const queue = [...roots];
110
+ while (queue.length > 0) {
111
+ const cPath = queue.shift();
112
+ if (found.has(cPath))
113
+ continue;
114
+ const category = parseCategory(await fetcher.text(categoryUrl(cPath)), cPath);
115
+ found.set(cPath, category);
116
+ if (category.children.length === 0 && category.products === 0) {
117
+ warn(` WARNING: c${cPath} (${category.name}) holds neither a subcategory nor a product`);
118
+ }
119
+ queue.push(...category.children);
120
+ await pause(delayMs);
121
+ }
122
+ return [...found.values()].sort((a, b) => compare(a.cPath, b.cPath));
123
+ }
124
+ /** The leaves of a walk: the categories that carry products themselves. */
125
+ export function leavesOf(categories) {
126
+ return categories.filter((category) => category.children.length === 0);
127
+ }
128
+ /** One line per category, for a human reading a `--catalog` run. */
129
+ export function describe(category) {
130
+ const shape = category.children.length > 0 ? `${category.children.length} subcategories` : 'leaf';
131
+ return `${BASE}/index.php?cPath=${category.cPath}\t${category.products}\t${shape}\t${category.name}`;
132
+ }
@@ -0,0 +1,10 @@
1
+ /**
2
+ * MariTool — toolholding only, and the one vendor here with no transport to
3
+ * discover.
4
+ *
5
+ * An osCommerce-family storefront: server-rendered HTML, a paced `GET`, and a
6
+ * roster on the category page with the geometry one request further on. See
7
+ * `docs/MARITOOL_CATALOG.md`.
8
+ */
9
+ export * from './catalog.js';
10
+ export * from './scrape.js';
@@ -0,0 +1,10 @@
1
+ /**
2
+ * MariTool — toolholding only, and the one vendor here with no transport to
3
+ * discover.
4
+ *
5
+ * An osCommerce-family storefront: server-rendered HTML, a paced `GET`, and a
6
+ * roster on the category page with the geometry one request further on. See
7
+ * `docs/MARITOOL_CATALOG.md`.
8
+ */
9
+ export * from './catalog.js';
10
+ export * from './scrape.js';