@nitpicker/crawler 0.18.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (140) hide show
  1. package/lib/append-retry-failed-common-setup-phases.d.ts +15 -0
  2. package/lib/append-retry-failed-common-setup-phases.js +21 -0
  3. package/lib/append-setup-phases.d.ts +17 -0
  4. package/lib/append-setup-phases.js +22 -0
  5. package/lib/archive/archive-accessor.d.ts +31 -5
  6. package/lib/archive/archive-accessor.js +46 -24
  7. package/lib/archive/archive.d.ts +115 -37
  8. package/lib/archive/archive.js +118 -52
  9. package/lib/archive/cache/extract-archive-to-cache.d.ts +10 -1
  10. package/lib/archive/cache/extract-archive-to-cache.js +14 -4
  11. package/lib/archive/create-adjunct-tables.d.ts +12 -2
  12. package/lib/archive/create-adjunct-tables.js +91 -29
  13. package/lib/archive/create-entity-tables.d.ts +5 -1
  14. package/lib/archive/create-entity-tables.js +6 -1
  15. package/lib/archive/database.d.ts +39 -9
  16. package/lib/archive/database.js +63 -18
  17. package/lib/archive/db-ops/lifecycle/init.d.ts +14 -2
  18. package/lib/archive/db-ops/lifecycle/init.js +29 -11
  19. package/lib/archive/db-ops/meta/get-custom-elements-of-page.d.ts +11 -0
  20. package/lib/archive/db-ops/meta/get-custom-elements-of-page.js +15 -0
  21. package/lib/archive/db-ops/meta/get-page-technologies-of-page.d.ts +11 -0
  22. package/lib/archive/db-ops/meta/get-page-technologies-of-page.js +15 -0
  23. package/lib/archive/db-ops/meta/get-technology-signals-of-page.d.ts +11 -0
  24. package/lib/archive/db-ops/meta/get-technology-signals-of-page.js +15 -0
  25. package/lib/archive/db-ops/pages/order/set-url-order.d.ts +6 -1
  26. package/lib/archive/db-ops/pages/order/set-url-order.js +7 -1
  27. package/lib/archive/db-ops/pages/read/build-page-query.js +1 -1
  28. package/lib/archive/db-ops/pages/reset/repromote-external-pages.d.ts +6 -1
  29. package/lib/archive/db-ops/pages/reset/repromote-external-pages.js +12 -3
  30. package/lib/archive/db-ops/pages/reset/reset-failed-pages.d.ts +6 -1
  31. package/lib/archive/db-ops/pages/reset/reset-failed-pages.js +10 -2
  32. package/lib/archive/db-ops/pages/write/insert-custom-elements.d.ts +27 -0
  33. package/lib/archive/db-ops/pages/write/insert-custom-elements.js +40 -0
  34. package/lib/archive/db-ops/pages/write/insert-page.d.ts +24 -4
  35. package/lib/archive/db-ops/pages/write/insert-page.js +40 -4
  36. package/lib/archive/db-ops/pages/write/insert-technologies.d.ts +25 -0
  37. package/lib/archive/db-ops/pages/write/insert-technologies.js +45 -0
  38. package/lib/archive/db-ops/pages/write/link-redirect-sources.d.ts +7 -1
  39. package/lib/archive/db-ops/pages/write/link-redirect-sources.js +8 -2
  40. package/lib/archive/db-ops/pages/write/update-page.d.ts +4 -1
  41. package/lib/archive/db-ops/pages/write/update-page.js +38 -16
  42. package/lib/archive/db-ops/resources/get-resource-url-list.d.ts +12 -1
  43. package/lib/archive/db-ops/resources/get-resource-url-list.js +43 -5
  44. package/lib/archive/filesystem/copy-file-with-progress.d.ts +20 -0
  45. package/lib/archive/filesystem/copy-file-with-progress.js +34 -0
  46. package/lib/archive/filesystem/parse-pax-path.d.ts +18 -0
  47. package/lib/archive/filesystem/parse-pax-path.js +47 -0
  48. package/lib/archive/filesystem/parse-tar-size-field.d.ts +13 -0
  49. package/lib/archive/filesystem/parse-tar-size-field.js +40 -0
  50. package/lib/archive/filesystem/peek-tar-top-dir.d.ts +10 -3
  51. package/lib/archive/filesystem/peek-tar-top-dir.js +166 -16
  52. package/lib/archive/filesystem/tar.d.ts +13 -1
  53. package/lib/archive/filesystem/tar.js +63 -7
  54. package/lib/archive/filesystem/untar.d.ts +13 -0
  55. package/lib/archive/filesystem/untar.js +68 -8
  56. package/lib/archive/init-schema.d.ts +1 -1
  57. package/lib/archive/init-schema.js +6 -4
  58. package/lib/archive/meta/compute-main-contents-denormalized.d.ts +13 -2
  59. package/lib/archive/meta/compute-main-contents-denormalized.js +15 -2
  60. package/lib/archive/meta/technologies/combine-technology-confidence.d.ts +46 -0
  61. package/lib/archive/meta/technologies/combine-technology-confidence.js +93 -0
  62. package/lib/archive/meta/technologies/convert-legacy-page-tags-to-inserts.d.ts +45 -0
  63. package/lib/archive/meta/technologies/convert-legacy-page-tags-to-inserts.js +53 -0
  64. package/lib/archive/meta/technologies/convert-tag-row-to-wappalyzer-signal.d.ts +33 -0
  65. package/lib/archive/meta/technologies/convert-tag-row-to-wappalyzer-signal.js +46 -0
  66. package/lib/archive/meta/technologies/extract-technologies-for-archive.d.ts +30 -0
  67. package/lib/archive/meta/technologies/extract-technologies-for-archive.js +27 -0
  68. package/lib/archive/meta/technologies/match-structural-technology-signals.d.ts +13 -0
  69. package/lib/archive/meta/technologies/match-structural-technology-signals.js +44 -0
  70. package/lib/archive/meta/technologies/normalize-wappalyzer-entries.d.ts +25 -0
  71. package/lib/archive/meta/technologies/normalize-wappalyzer-entries.js +35 -0
  72. package/lib/archive/meta/technologies/technology-signal-definitions.d.ts +43 -0
  73. package/lib/archive/meta/technologies/technology-signal-definitions.js +193 -0
  74. package/lib/archive/meta/technologies/types.d.ts +68 -0
  75. package/lib/archive/meta/technologies/types.js +13 -0
  76. package/lib/archive/meta/types.d.ts +97 -30
  77. package/lib/archive/migrate-content-items-alias-of-id.d.ts +6 -1
  78. package/lib/archive/migrate-content-items-alias-of-id.js +14 -3
  79. package/lib/archive/migrate-content-items-dedupe-cap-event-id.d.ts +6 -1
  80. package/lib/archive/migrate-content-items-dedupe-cap-event-id.js +14 -3
  81. package/lib/archive/migrate-info-main-content-selector.d.ts +6 -1
  82. package/lib/archive/migrate-info-main-content-selector.js +14 -3
  83. package/lib/archive/migrate-info-roots.d.ts +6 -1
  84. package/lib/archive/migrate-info-roots.js +14 -3
  85. package/lib/archive/migrate-inventory-runs-exclude-skipped.d.ts +6 -1
  86. package/lib/archive/migrate-inventory-runs-exclude-skipped.js +14 -3
  87. package/lib/archive/migrate-inventory-runs-invalid-skipped.d.ts +6 -1
  88. package/lib/archive/migrate-inventory-runs-invalid-skipped.js +14 -3
  89. package/lib/archive/migrate-main-contents-columns.d.ts +6 -1
  90. package/lib/archive/migrate-main-contents-columns.js +14 -3
  91. package/lib/archive/migrate-page-meta-body-hash.d.ts +6 -1
  92. package/lib/archive/migrate-page-meta-body-hash.js +14 -3
  93. package/lib/archive/migrate-page-meta-console-error-count.d.ts +6 -1
  94. package/lib/archive/migrate-page-meta-console-error-count.js +14 -3
  95. package/lib/archive/migrate-page-meta-custom-element-count.d.ts +33 -0
  96. package/lib/archive/migrate-page-meta-custom-element-count.js +51 -0
  97. package/lib/archive/migrate-page-tags-to-page-technologies.d.ts +39 -0
  98. package/lib/archive/migrate-page-tags-to-page-technologies.js +79 -0
  99. package/lib/archive/page.d.ts +37 -13
  100. package/lib/archive/page.js +45 -17
  101. package/lib/archive/retarget-legacy-fk-tables.d.ts +27 -14
  102. package/lib/archive/retarget-legacy-fk-tables.js +61 -15
  103. package/lib/archive/types.d.ts +18 -0
  104. package/lib/crawler/assert-puppeteer-shared-with-beholder.d.ts +24 -0
  105. package/lib/crawler/assert-puppeteer-shared-with-beholder.js +36 -0
  106. package/lib/crawler/capture-custom-elements.d.ts +33 -0
  107. package/lib/crawler/capture-custom-elements.js +39 -0
  108. package/lib/crawler/close-browser-safely.d.ts +3 -3
  109. package/lib/crawler/close-browser-safely.js +8 -17
  110. package/lib/crawler/collect-custom-elements.d.ts +33 -0
  111. package/lib/crawler/collect-custom-elements.js +110 -0
  112. package/lib/crawler/crawler.js +101 -46
  113. package/lib/crawler/fetch-destination.js +6 -13
  114. package/lib/crawler/find-package-dir.d.ts +24 -0
  115. package/lib/crawler/find-package-dir.js +39 -0
  116. package/lib/crawler/scan-js-resource-for-license-comment.d.ts +32 -0
  117. package/lib/crawler/scan-js-resource-for-license-comment.js +140 -0
  118. package/lib/crawler/scan-js-resources-for-technology-signals.d.ts +58 -0
  119. package/lib/crawler/scan-js-resources-for-technology-signals.js +196 -0
  120. package/lib/crawler/types.d.ts +23 -0
  121. package/lib/crawler-orchestrator.d.ts +53 -9
  122. package/lib/crawler-orchestrator.js +223 -46
  123. package/lib/crawler.d.ts +11 -1
  124. package/lib/crawler.js +9 -0
  125. package/lib/inventory-setup-phases.d.ts +32 -0
  126. package/lib/inventory-setup-phases.js +45 -0
  127. package/lib/resume-setup-phases.d.ts +16 -0
  128. package/lib/resume-setup-phases.js +24 -0
  129. package/lib/retry-failed-setup-phases.d.ts +19 -0
  130. package/lib/retry-failed-setup-phases.js +24 -0
  131. package/lib/setup-recovery-phase-labels.d.ts +21 -0
  132. package/lib/setup-recovery-phase-labels.js +24 -0
  133. package/lib/types.d.ts +145 -0
  134. package/package.json +12 -9
  135. package/lib/archive/db-ops/meta/get-tags-of-page.d.ts +0 -12
  136. package/lib/archive/db-ops/meta/get-tags-of-page.js +0 -28
  137. package/lib/archive/db-ops/pages/write/insert-tags.d.ts +0 -16
  138. package/lib/archive/db-ops/pages/write/insert-tags.js +0 -34
  139. package/lib/archive/meta/summarize-tags.d.ts +0 -16
  140. package/lib/archive/meta/summarize-tags.js +0 -33
@@ -0,0 +1,193 @@
1
+ /**
2
+ * Fixed fingerprint list for the frontend frameworks this module detects via
3
+ * structural signals (URL path fragments, HTML markers, scoped-attribute
4
+ * conventions). Deliberately a TypeScript array, not an external config
5
+ * file (per the accepted design: precision/coverage is refined by editing
6
+ * this file, not by runtime configuration).
7
+ *
8
+ * Every framework here is ALSO detectable via {@link WAPPALYZER_PROVIDER_TO_TECHNOLOGY}
9
+ * when beholder's Wappalyzer pass independently reports it — the two signal
10
+ * sources compound in `combineTechnologyConfidence` rather than one
11
+ * replacing the other. Wappalyzer detects many more technologies than the
12
+ * frameworks listed here (analytics, CMSes, ...); those pass through with
13
+ * the provider name verbatim as the technology name, uncurated.
14
+ * @module
15
+ */
16
+ export const TECHNOLOGY_SIGNAL_DEFINITIONS = [
17
+ // Next.js
18
+ {
19
+ technology: 'Next.js',
20
+ signalType: 'html-marker',
21
+ pattern: /<script[^>]*\sid="__NEXT_DATA__"/i,
22
+ weight: 70,
23
+ category: 'JavaScript frameworks',
24
+ },
25
+ {
26
+ technology: 'Next.js',
27
+ signalType: 'html-marker',
28
+ pattern: /<div[^>]*\sid="__next"/i,
29
+ weight: 60,
30
+ category: 'JavaScript frameworks',
31
+ },
32
+ {
33
+ technology: 'Next.js',
34
+ signalType: 'url-pattern',
35
+ pattern: /\/_next\//,
36
+ weight: 50,
37
+ category: 'JavaScript frameworks',
38
+ },
39
+ // Astro
40
+ {
41
+ technology: 'Astro',
42
+ signalType: 'html-marker',
43
+ pattern: /<astro-island\b/i,
44
+ weight: 65,
45
+ category: 'Static site generator',
46
+ },
47
+ {
48
+ technology: 'Astro',
49
+ signalType: 'url-pattern',
50
+ pattern: /\/_astro\//,
51
+ weight: 50,
52
+ category: 'Static site generator',
53
+ },
54
+ {
55
+ technology: 'Astro',
56
+ signalType: 'scoped-attr',
57
+ pattern: /\sdata-astro-cid-[a-z0-9]+/i,
58
+ weight: 40,
59
+ category: 'Static site generator',
60
+ },
61
+ // Vue
62
+ {
63
+ technology: 'Vue',
64
+ signalType: 'scoped-attr',
65
+ pattern: /\sdata-v-[a-z0-9]{6,10}\b/i,
66
+ weight: 40,
67
+ category: 'JavaScript frameworks',
68
+ },
69
+ {
70
+ technology: 'Vue',
71
+ signalType: 'weak-marker',
72
+ pattern: /\sid="app"/i,
73
+ weight: 15,
74
+ category: 'JavaScript frameworks',
75
+ },
76
+ {
77
+ technology: 'Vue',
78
+ signalType: 'js-license-comment',
79
+ pattern: /\/\*!\s*\*\s*Vue\.js v\d/i,
80
+ weight: 55,
81
+ category: 'JavaScript frameworks',
82
+ },
83
+ // Nuxt
84
+ {
85
+ technology: 'Nuxt',
86
+ signalType: 'html-marker',
87
+ pattern: /<div[^>]*\sid="__nuxt"/i,
88
+ weight: 60,
89
+ category: 'JavaScript frameworks',
90
+ },
91
+ {
92
+ technology: 'Nuxt',
93
+ signalType: 'url-pattern',
94
+ pattern: /\/_nuxt\//,
95
+ weight: 50,
96
+ category: 'JavaScript frameworks',
97
+ },
98
+ // Svelte / SvelteKit
99
+ {
100
+ technology: 'Svelte',
101
+ signalType: 'scoped-attr',
102
+ pattern: /class="[^"]*\bsvelte-[a-z0-9]{6,8}\b/i,
103
+ weight: 40,
104
+ category: 'JavaScript frameworks',
105
+ },
106
+ {
107
+ technology: 'SvelteKit',
108
+ signalType: 'url-pattern',
109
+ pattern: /\/_app\/immutable\//,
110
+ weight: 50,
111
+ category: 'JavaScript frameworks',
112
+ },
113
+ // Remix
114
+ {
115
+ technology: 'Remix',
116
+ signalType: 'html-marker',
117
+ pattern: /window\.__remixContext\s*=/i,
118
+ weight: 65,
119
+ category: 'JavaScript frameworks',
120
+ },
121
+ // Gatsby
122
+ {
123
+ technology: 'Gatsby',
124
+ signalType: 'html-marker',
125
+ pattern: /<div[^>]*\sid="___gatsby"/i,
126
+ weight: 65,
127
+ category: 'Static site generator',
128
+ },
129
+ {
130
+ technology: 'Gatsby',
131
+ signalType: 'url-pattern',
132
+ pattern: /\/page-data\//,
133
+ weight: 50,
134
+ category: 'Static site generator',
135
+ },
136
+ // Angular
137
+ {
138
+ technology: 'Angular',
139
+ signalType: 'html-marker',
140
+ pattern: /\sng-version="[\d.]+"/i,
141
+ weight: 65,
142
+ category: 'JavaScript frameworks',
143
+ },
144
+ {
145
+ technology: 'Angular',
146
+ signalType: 'scoped-attr',
147
+ pattern: /\s_ngcontent-[a-z0-9-]+/i,
148
+ weight: 40,
149
+ category: 'JavaScript frameworks',
150
+ },
151
+ ];
152
+ /**
153
+ * `<meta name="generator">` prefix matchers, checked case-insensitively
154
+ * against `page_meta.generator` (already extracted — no new HTML scan
155
+ * needed). Distinct from {@link TECHNOLOGY_SIGNAL_DEFINITIONS} because the
156
+ * match strategy is prefix-on-a-known-field, not regex-over-HTML.
157
+ */
158
+ export const GENERATOR_TECHNOLOGY_MATCHERS = [
159
+ { technology: 'Astro', prefix: 'astro', weight: 85, category: 'Static site generator' },
160
+ { technology: 'Nuxt', prefix: 'nuxt', weight: 85, category: 'JavaScript frameworks' },
161
+ {
162
+ technology: 'Gatsby',
163
+ prefix: 'gatsby',
164
+ weight: 85,
165
+ category: 'Static site generator',
166
+ },
167
+ ];
168
+ /**
169
+ * Wappalyzer provider name → normalized technology name, for the curated
170
+ * frameworks that also have {@link TECHNOLOGY_SIGNAL_DEFINITIONS}. Providers
171
+ * NOT in this map still become `page_technologies` rows — the technology
172
+ * name is just the provider name verbatim (see `normalize-wappalyzer-entries.ts`)
173
+ * — this map exists only to fold curated frameworks' Wappalyzer detections
174
+ * onto the same technology name their structural signals use, so the two
175
+ * sources compound instead of appearing as two separate rows.
176
+ */
177
+ export const WAPPALYZER_PROVIDER_TO_TECHNOLOGY = {
178
+ 'Next.js': 'Next.js',
179
+ Astro: 'Astro',
180
+ 'Vue.js': 'Vue',
181
+ 'Nuxt.js': 'Nuxt',
182
+ Nuxt: 'Nuxt',
183
+ Svelte: 'Svelte',
184
+ SvelteKit: 'SvelteKit',
185
+ Remix: 'Remix',
186
+ Gatsby: 'Gatsby',
187
+ Angular: 'Angular',
188
+ AngularJS: 'Angular',
189
+ };
190
+ /** Confidence a lone `wappalyzer` signal contributes, before compounding with any structural signal. */
191
+ export const WAPPALYZER_SIGNAL_WEIGHT = 60;
192
+ /** Confidence threshold (`combineTechnologyConfidence`'s output) at or above which a `page_technologies` row is written at all. Below this, a single weak/ambiguous signal is treated as noise, not a detection. */
193
+ export const TECHNOLOGY_CONFIDENCE_THRESHOLD = 30;
@@ -0,0 +1,68 @@
1
+ /**
2
+ * Domain types for technology-signal extraction (crawl time) and confidence
3
+ * roll-up. Kept in this dedicated `technologies/types.ts` rather than the
4
+ * parent `meta/types.ts` because these types are internal to the extraction
5
+ * pipeline (`match-structural-technology-signals.ts` →
6
+ * `combine-technology-confidence.ts` → `extract-technologies-for-archive.ts`)
7
+ * and never cross the `archive/meta/types.ts` boundary that the persisted
8
+ * `technology_signals` / `page_technologies` row shapes (`TechnologySignalRow`
9
+ * / `PageTechnologyRow`) live in — see the repo-wide `型は types.ts に集約`
10
+ * rule, applied per-module rather than globally so this module's internal
11
+ * vocabulary does not leak into the db-ops layer's public row types.
12
+ */
13
+ /**
14
+ * How a technology signal was detected. `wappalyzer` is beholder's
15
+ * `simple-wappalyzer`-based detection (imprecise on its own for frameworks,
16
+ * but authoritative for the long tail of non-framework technologies —
17
+ * analytics, CMSes, etc. — this module does not otherwise attempt to
18
+ * detect). The rest are structural signals this module detects itself.
19
+ */
20
+ export type TechnologySignalType = 'wappalyzer' | 'meta-generator' | 'html-marker' | 'url-pattern' | 'scoped-attr' | 'weak-marker' | 'js-license-comment';
21
+ /**
22
+ * One un-combined signal for one technology on one page. The unit that
23
+ * `combineTechnologyConfidence` groups (by `technology`) and reduces via
24
+ * noisy-OR into a single {@link PageTechnologyPartial} per technology.
25
+ */
26
+ export interface TechnologySignalPartial {
27
+ /** Normalized technology name (e.g. `'Next.js'`, `'Google Analytics'`). */
28
+ technology: string;
29
+ signalType: TechnologySignalType;
30
+ /** Matched fragment or raw value, truncated to ~200 chars by the caller. `null` when the signal type carries no useful excerpt. */
31
+ evidence: string | null;
32
+ /** This signal's confidence in isolation, 0-100. */
33
+ weight: number;
34
+ /** Wappalyzer/generator-reported category, when this signal type can supply one. */
35
+ category?: string | null;
36
+ /** Wappalyzer/generator-reported version, when this signal type can supply one. */
37
+ version?: string | null;
38
+ }
39
+ /**
40
+ * One technology's combined roll-up for one page, before the `pageId` is
41
+ * known (added by the caller at insert time — same "id injected at write
42
+ * time" pattern as {@link import('../types.js').TagRowForInsert}).
43
+ */
44
+ export interface PageTechnologyPartial {
45
+ technology: string;
46
+ category: string | null;
47
+ version: string | null;
48
+ /** `combineTechnologyConfidence`'s noisy-OR result, 0-100. */
49
+ confidence: number;
50
+ /** Count of distinct `signalType`s that contributed to `confidence`. */
51
+ signalCount: number;
52
+ }
53
+ /**
54
+ * A single fingerprint definition: one technology matched by one signal
55
+ * type via one regular expression, at one isolated weight. Multiple
56
+ * definitions may share a `technology` (different signal types/patterns)
57
+ * — {@link combineTechnologyConfidence} is what lets those compound.
58
+ */
59
+ export interface TechnologySignalDefinition {
60
+ technology: string;
61
+ signalType: 'html-marker' | 'url-pattern' | 'scoped-attr' | 'weak-marker' | 'js-license-comment';
62
+ /** Tested against the page's raw HTML string (or, for `js-license-comment`, a JS resource's leading bytes). */
63
+ pattern: RegExp;
64
+ /** This definition's confidence in isolation, 0-100, when `pattern` matches. */
65
+ weight: number;
66
+ /** Category to attach to the technology when only this signal fired. */
67
+ category?: string;
68
+ }
@@ -0,0 +1,13 @@
1
+ /**
2
+ * Domain types for technology-signal extraction (crawl time) and confidence
3
+ * roll-up. Kept in this dedicated `technologies/types.ts` rather than the
4
+ * parent `meta/types.ts` because these types are internal to the extraction
5
+ * pipeline (`match-structural-technology-signals.ts` →
6
+ * `combine-technology-confidence.ts` → `extract-technologies-for-archive.ts`)
7
+ * and never cross the `archive/meta/types.ts` boundary that the persisted
8
+ * `technology_signals` / `page_technologies` row shapes (`TechnologySignalRow`
9
+ * / `PageTechnologyRow`) live in — see the repo-wide `型は types.ts に集約`
10
+ * rule, applied per-module rather than globally so this module's internal
11
+ * vocabulary does not leak into the db-ops layer's public row types.
12
+ */
13
+ export {};
@@ -122,6 +122,7 @@ export interface PageDenormalizedColumns {
122
122
  * main_content_video_count: 0,
123
123
  * main_content_audio_count: 0,
124
124
  * main_content_canvas_count: 0,
125
+ * main_content_custom_element_count: 0,
125
126
  * scroll_height_desktop: 3200,
126
127
  * scroll_height_mobile: 5400,
127
128
  * };
@@ -157,6 +158,16 @@ export interface MainContentsDenormalizedColumns {
157
158
  main_content_audio_count: number | null;
158
159
  /** Number of canvases within the main region, or `null`. */
159
160
  main_content_canvas_count: number | null;
161
+ /**
162
+ * Number of Web Components (custom elements) within the main region, or
163
+ * `null`. Unlike its siblings above, `null` is not solely "page not
164
+ * rendered" — it also covers "rendered, but nitpicker's own
165
+ * `captureCustomElements` best-effort capture failed" (a distinct state
166
+ * from "captured, zero found" = `0`), since this column is not sourced
167
+ * from beholder's `MainContentsData` at all. See
168
+ * `compute-main-contents-denormalized.ts`.
169
+ */
170
+ main_content_custom_element_count: number | null;
160
171
  /** `document.body.scrollHeight` at the desktop-compact preset, or `null`. */
161
172
  scroll_height_desktop: number | null;
162
173
  /** `document.body.scrollHeight` at the mobile-small preset, or `null`. */
@@ -356,6 +367,27 @@ export interface MainContentCanvasRow {
356
367
  /** IDL bitmap height. */
357
368
  height: number;
358
369
  }
370
+ /**
371
+ * One row in the `page_main_content_custom_elements` table. Unlike its
372
+ * seven `MainContentXxxRow` siblings, the source data is not beholder's
373
+ * `MainContentsData` but nitpicker's own `capture-custom-elements.ts`.
374
+ * @example
375
+ * const row: MainContentCustomElementRow = { id: 1, pageId: 42, order: 0, nodeName: 'MY-WIDGET', elementId: 'widget-1', classList: '["foo"]' };
376
+ */
377
+ export interface MainContentCustomElementRow {
378
+ /** Auto-increment primary key. */
379
+ id: number;
380
+ /** FK to `content_items.id`. */
381
+ pageId: number;
382
+ /** 0-based DOM traversal order within the main content region. */
383
+ order: number;
384
+ /** The element's `nodeName` (e.g. `'MY-WIDGET'`). */
385
+ nodeName: string;
386
+ /** The element's `id` attribute, or `null`. */
387
+ elementId: string | null;
388
+ /** JSON-encoded array of the element's CSS classes, or `null`. */
389
+ classList: string | null;
390
+ }
359
391
  /**
360
392
  * One row in the `page_jsonld` table.
361
393
  *
@@ -474,6 +506,71 @@ export interface TagRow {
474
506
  * };
475
507
  */
476
508
  export type TagRowForInsert = Omit<TagRow, 'id'>;
509
+ /**
510
+ * One row in the `technology_signals` table — one un-combined signal for
511
+ * one technology on one page. See
512
+ * `archive/meta/technologies/types.ts#TechnologySignalPartial` for the
513
+ * pre-insert shape (no `pageId`/`id`) this is built from.
514
+ * @example
515
+ * const row: TechnologySignalRow = {
516
+ * id: 1,
517
+ * pageId: 42,
518
+ * technology: 'Next.js',
519
+ * signalType: 'html-marker',
520
+ * evidence: '<script id="__NEXT_DATA__"',
521
+ * weight: 70,
522
+ * };
523
+ */
524
+ export interface TechnologySignalRow {
525
+ /** Auto-increment primary key. */
526
+ id: number;
527
+ /** FK to `content_items.id`. */
528
+ pageId: number;
529
+ /** Normalized technology name (e.g. `'Next.js'`, `'Google Analytics'`). */
530
+ technology: string;
531
+ /** How this signal was detected. */
532
+ signalType: 'wappalyzer' | 'meta-generator' | 'html-marker' | 'url-pattern' | 'scoped-attr' | 'weak-marker' | 'js-license-comment';
533
+ /** Matched fragment or raw value, truncated to ~200 chars, or `null`. */
534
+ evidence: string | null;
535
+ /** This signal's confidence in isolation, 0-100. */
536
+ weight: number;
537
+ }
538
+ /** Insert shape for {@link TechnologySignalRow}. */
539
+ export type TechnologySignalRowForInsert = Omit<TechnologySignalRow, 'id'>;
540
+ /**
541
+ * One row in the `page_technologies` table — the confidence-combined
542
+ * roll-up of every {@link TechnologySignalRow} for one technology on one
543
+ * page. Read-optimised counterpart of `technology_signals`, analogous to
544
+ * `page_meta.tag_count` being the roll-up of `page_tags` — except here the
545
+ * roll-up is a full row, not just a count, because `confidence` is a
546
+ * per-technology computed value (`combineTechnologyConfidence`), not a
547
+ * simple count.
548
+ * @example
549
+ * const row: PageTechnologyRow = {
550
+ * id: 1,
551
+ * pageId: 42,
552
+ * technology: 'Next.js',
553
+ * category: 'JavaScript frameworks',
554
+ * version: null,
555
+ * confidence: 80,
556
+ * signalCount: 2,
557
+ * };
558
+ */
559
+ export interface PageTechnologyRow {
560
+ /** Auto-increment primary key. */
561
+ id: number;
562
+ /** FK to `content_items.id`. */
563
+ pageId: number;
564
+ technology: string;
565
+ category: string | null;
566
+ version: string | null;
567
+ /** `combineTechnologyConfidence`'s noisy-OR result, 0-100. */
568
+ confidence: number;
569
+ /** Count of distinct `signalType`s that contributed to `confidence`. */
570
+ signalCount: number;
571
+ }
572
+ /** Insert shape for {@link PageTechnologyRow}. */
573
+ export type PageTechnologyRowForInsert = Omit<PageTechnologyRow, 'id'>;
477
574
  /**
478
575
  * Summary of one page's JSON-LD entries returned by `get-page-detail`.
479
576
  *
@@ -495,36 +592,6 @@ export interface JsonLdSummary {
495
592
  /** Number of entries that failed to parse (i.e. have a non-null `parseError`). */
496
593
  parseErrorCount: number;
497
594
  }
498
- /**
499
- * Summary of one page's Wappalyzer tags returned by `get-page-detail`.
500
- * @see summarize-tags.ts
501
- * @example
502
- * const summary: TagsSummary = {
503
- * count: 2,
504
- * providerIds: { 'Google Analytics': ['G-XXXX'], YouTube: [] },
505
- * };
506
- */
507
- export interface TagsSummary {
508
- /** Total tag rows for the page. */
509
- count: number;
510
- /** Provider → list of external IDs (unique, sorted). Providers with no IDs map to `[]`. */
511
- providerIds: Readonly<Record<string, readonly string[]>>;
512
- }
513
- /**
514
- * One row of {@link import('@d-zero/beholder').Meta.tags.detected} after
515
- * archive-side flattening.
516
- *
517
- * Used by `get-tag-inventory` to return per-provider page counts across the
518
- * whole site.
519
- * @example
520
- * const entry: TagInventoryEntry = { provider: 'Google Tag Manager', pageCount: 120 };
521
- */
522
- export interface TagInventoryEntry {
523
- /** Wappalyzer provider name. */
524
- provider: string;
525
- /** Number of distinct pages where the provider was detected. */
526
- pageCount: number;
527
- }
528
595
  /**
529
596
  * Error thrown by `assert-compatible-version` when the archive's
530
597
  * `info.version` is older than the format version this build accepts.
@@ -43,5 +43,10 @@ import type { Knex } from 'knex';
43
43
  * runs (after `initSchema`, itself after `assertCompatibleVersion` rejects
44
44
  * pre-0.13 archives) the table is always present.
45
45
  * @param instance - The Knex query builder instance connected to the database.
46
+ * @param onLog - Called instead of `console.error` when this migration
47
+ * actually applies (issue #294: a bare `console.error` here can fire
48
+ * while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
49
+ * `Archive.open`, corrupting its cursor tracking). Falls back to
50
+ * `console.error` when omitted (direct/test callers).
46
51
  */
47
- export declare function migrateContentItemsAliasOfId(instance: Knex): Promise<void>;
52
+ export declare function migrateContentItemsAliasOfId(instance: Knex, onLog?: (message: string) => void): Promise<void>;
@@ -42,8 +42,13 @@
42
42
  * runs (after `initSchema`, itself after `assertCompatibleVersion` rejects
43
43
  * pre-0.13 archives) the table is always present.
44
44
  * @param instance - The Knex query builder instance connected to the database.
45
+ * @param onLog - Called instead of `console.error` when this migration
46
+ * actually applies (issue #294: a bare `console.error` here can fire
47
+ * while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
48
+ * `Archive.open`, corrupting its cursor tracking). Falls back to
49
+ * `console.error` when omitted (direct/test callers).
45
50
  */
46
- export async function migrateContentItemsAliasOfId(instance) {
51
+ export async function migrateContentItemsAliasOfId(instance, onLog) {
47
52
  const hasContentItems = await instance.schema.hasTable('content_items');
48
53
  if (!hasContentItems) {
49
54
  return;
@@ -51,8 +56,14 @@ export async function migrateContentItemsAliasOfId(instance) {
51
56
  const hasColumn = await instance.schema.hasColumn('content_items', 'alias_of_id');
52
57
  if (!hasColumn) {
53
58
  await instance.raw('ALTER TABLE content_items ADD COLUMN alias_of_id INTEGER REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED');
54
- // eslint-disable-next-line no-console
55
- console.error('[migrate] content_items.alias_of_id column added');
59
+ const message = '[migrate] content_items.alias_of_id column added';
60
+ if (onLog) {
61
+ onLog(message);
62
+ }
63
+ else {
64
+ // eslint-disable-next-line no-console
65
+ console.error(message);
66
+ }
56
67
  }
57
68
  await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_alias_of_id ON content_items(alias_of_id)');
58
69
  }
@@ -33,9 +33,14 @@ import type { Knex } from 'knex';
33
33
  * (after `initSchema`, itself after `assertCompatibleVersion` rejects
34
34
  * pre-0.13 archives) the table is always present.
35
35
  * @param instance - The Knex query builder instance connected to the database.
36
+ * @param onLog - Called instead of `console.error` when this migration
37
+ * actually applies (issue #294: a bare `console.error` here can fire
38
+ * while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
39
+ * `Archive.open`, corrupting its cursor tracking). Falls back to
40
+ * `console.error` when omitted (direct/test callers).
36
41
  * @example
37
42
  * ```ts
38
43
  * await migrateContentItemsDedupeCapEventId(knex);
39
44
  * ```
40
45
  */
41
- export declare function migrateContentItemsDedupeCapEventId(instance: Knex): Promise<void>;
46
+ export declare function migrateContentItemsDedupeCapEventId(instance: Knex, onLog?: (message: string) => void): Promise<void>;
@@ -32,12 +32,17 @@
32
32
  * (after `initSchema`, itself after `assertCompatibleVersion` rejects
33
33
  * pre-0.13 archives) the table is always present.
34
34
  * @param instance - The Knex query builder instance connected to the database.
35
+ * @param onLog - Called instead of `console.error` when this migration
36
+ * actually applies (issue #294: a bare `console.error` here can fire
37
+ * while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
38
+ * `Archive.open`, corrupting its cursor tracking). Falls back to
39
+ * `console.error` when omitted (direct/test callers).
35
40
  * @example
36
41
  * ```ts
37
42
  * await migrateContentItemsDedupeCapEventId(knex);
38
43
  * ```
39
44
  */
40
- export async function migrateContentItemsDedupeCapEventId(instance) {
45
+ export async function migrateContentItemsDedupeCapEventId(instance, onLog) {
41
46
  const hasContentItems = await instance.schema.hasTable('content_items');
42
47
  if (!hasContentItems) {
43
48
  return;
@@ -45,7 +50,13 @@ export async function migrateContentItemsDedupeCapEventId(instance) {
45
50
  const hasColumn = await instance.schema.hasColumn('content_items', 'dedupe_cap_event_id');
46
51
  if (!hasColumn) {
47
52
  await instance.raw('ALTER TABLE content_items ADD COLUMN dedupe_cap_event_id INTEGER REFERENCES dedupe_cap_events(id) DEFERRABLE INITIALLY DEFERRED');
48
- // eslint-disable-next-line no-console
49
- console.error('[migrate] content_items.dedupe_cap_event_id column added');
53
+ const message = '[migrate] content_items.dedupe_cap_event_id column added';
54
+ if (onLog) {
55
+ onLog(message);
56
+ }
57
+ else {
58
+ // eslint-disable-next-line no-console
59
+ console.error(message);
60
+ }
50
61
  }
51
62
  }
@@ -7,5 +7,10 @@ import type { Knex } from 'knex';
7
7
  * idempotent, and self-healing for archives whose provisioning crashed
8
8
  * partway through.
9
9
  * @param instance - The Knex query builder instance connected to the database.
10
+ * @param onLog - Called instead of `console.error` when this migration
11
+ * actually applies (issue #294: a bare `console.error` here can fire
12
+ * while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
13
+ * `Archive.open`, corrupting its cursor tracking). Falls back to
14
+ * `console.error` when omitted (direct/test callers).
10
15
  */
11
- export declare function migrateInfoMainContentSelector(instance: Knex): Promise<void>;
16
+ export declare function migrateInfoMainContentSelector(instance: Knex, onLog?: (message: string) => void): Promise<void>;
@@ -6,8 +6,13 @@
6
6
  * idempotent, and self-healing for archives whose provisioning crashed
7
7
  * partway through.
8
8
  * @param instance - The Knex query builder instance connected to the database.
9
+ * @param onLog - Called instead of `console.error` when this migration
10
+ * actually applies (issue #294: a bare `console.error` here can fire
11
+ * while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
12
+ * `Archive.open`, corrupting its cursor tracking). Falls back to
13
+ * `console.error` when omitted (direct/test callers).
9
14
  */
10
- export async function migrateInfoMainContentSelector(instance) {
15
+ export async function migrateInfoMainContentSelector(instance, onLog) {
11
16
  const hasInfo = await instance.schema.hasTable('info');
12
17
  if (!hasInfo) {
13
18
  return;
@@ -19,6 +24,12 @@ export async function migrateInfoMainContentSelector(instance) {
19
24
  await instance.schema.table('info', (t) => {
20
25
  t.string('mainContentSelector');
21
26
  });
22
- // eslint-disable-next-line no-console
23
- console.error('[migrate] info.mainContentSelector column added');
27
+ const message = '[migrate] info.mainContentSelector column added';
28
+ if (onLog) {
29
+ onLog(message);
30
+ }
31
+ else {
32
+ // eslint-disable-next-line no-console
33
+ console.error(message);
34
+ }
24
35
  }
@@ -11,5 +11,10 @@ import type { Knex } from 'knex';
11
11
  * performed), a single notice is written to stderr so the user knows the
12
12
  * file was upgraded.
13
13
  * @param instance - The Knex query builder instance connected to the database.
14
+ * @param onLog - Called instead of `console.error` when this migration
15
+ * actually applies (issue #294: a bare `console.error` here can fire
16
+ * while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
17
+ * `Archive.open`, corrupting its cursor tracking). Falls back to
18
+ * `console.error` when omitted (direct/test callers).
14
19
  */
15
- export declare function migrateInfoRoots(instance: Knex): Promise<void>;
20
+ export declare function migrateInfoRoots(instance: Knex, onLog?: (message: string) => void): Promise<void>;
@@ -10,8 +10,13 @@
10
10
  * performed), a single notice is written to stderr so the user knows the
11
11
  * file was upgraded.
12
12
  * @param instance - The Knex query builder instance connected to the database.
13
+ * @param onLog - Called instead of `console.error` when this migration
14
+ * actually applies (issue #294: a bare `console.error` here can fire
15
+ * while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
16
+ * `Archive.open`, corrupting its cursor tracking). Falls back to
17
+ * `console.error` when omitted (direct/test callers).
13
18
  */
14
- export async function migrateInfoRoots(instance) {
19
+ export async function migrateInfoRoots(instance, onLog) {
15
20
  const hasInfo = await instance.schema.hasTable('info');
16
21
  if (!hasInfo) {
17
22
  return;
@@ -36,6 +41,12 @@ export async function migrateInfoRoots(instance) {
36
41
  });
37
42
  changes.push('scope dropped');
38
43
  }
39
- // eslint-disable-next-line no-console
40
- console.error(`[migrate] info table upgraded (${changes.join(', ')})`);
44
+ const message = `[migrate] info table upgraded (${changes.join(', ')})`;
45
+ if (onLog) {
46
+ onLog(message);
47
+ }
48
+ else {
49
+ // eslint-disable-next-line no-console
50
+ console.error(message);
51
+ }
41
52
  }
@@ -16,5 +16,10 @@ import type { Knex } from 'knex';
16
16
  * only by `listInventoryRuns` display surfaces, never consumed by any
17
17
  * runtime decision — matching `scope_skipped` / `invalid_skipped`.
18
18
  * @param instance - The Knex query builder instance connected to the database.
19
+ * @param onLog - Called instead of `console.error` when this migration
20
+ * actually applies (issue #294: a bare `console.error` here can fire
21
+ * while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
22
+ * `Archive.open`, corrupting its cursor tracking). Falls back to
23
+ * `console.error` when omitted (direct/test callers).
19
24
  */
20
- export declare function migrateInventoryRunsExcludeSkipped(instance: Knex): Promise<void>;
25
+ export declare function migrateInventoryRunsExcludeSkipped(instance: Knex, onLog?: (message: string) => void): Promise<void>;
@@ -15,8 +15,13 @@
15
15
  * only by `listInventoryRuns` display surfaces, never consumed by any
16
16
  * runtime decision — matching `scope_skipped` / `invalid_skipped`.
17
17
  * @param instance - The Knex query builder instance connected to the database.
18
+ * @param onLog - Called instead of `console.error` when this migration
19
+ * actually applies (issue #294: a bare `console.error` here can fire
20
+ * while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
21
+ * `Archive.open`, corrupting its cursor tracking). Falls back to
22
+ * `console.error` when omitted (direct/test callers).
18
23
  */
19
- export async function migrateInventoryRunsExcludeSkipped(instance) {
24
+ export async function migrateInventoryRunsExcludeSkipped(instance, onLog) {
20
25
  const hasTable = await instance.schema.hasTable('inventory_runs');
21
26
  if (!hasTable) {
22
27
  return;
@@ -28,6 +33,12 @@ export async function migrateInventoryRunsExcludeSkipped(instance) {
28
33
  await instance.schema.table('inventory_runs', (t) => {
29
34
  t.integer('exclude_skipped');
30
35
  });
31
- // eslint-disable-next-line no-console
32
- console.error('[migrate] inventory_runs.exclude_skipped column added');
36
+ const message = '[migrate] inventory_runs.exclude_skipped column added';
37
+ if (onLog) {
38
+ onLog(message);
39
+ }
40
+ else {
41
+ // eslint-disable-next-line no-console
42
+ console.error(message);
43
+ }
33
44
  }