@nitpicker/crawler 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/append-retry-failed-common-setup-phases.d.ts +15 -0
- package/lib/append-retry-failed-common-setup-phases.js +21 -0
- package/lib/append-setup-phases.d.ts +17 -0
- package/lib/append-setup-phases.js +22 -0
- package/lib/archive/archive-accessor.d.ts +31 -5
- package/lib/archive/archive-accessor.js +46 -24
- package/lib/archive/archive.d.ts +115 -37
- package/lib/archive/archive.js +118 -52
- package/lib/archive/cache/extract-archive-to-cache.d.ts +10 -1
- package/lib/archive/cache/extract-archive-to-cache.js +14 -4
- package/lib/archive/create-adjunct-tables.d.ts +12 -2
- package/lib/archive/create-adjunct-tables.js +91 -29
- package/lib/archive/create-entity-tables.d.ts +5 -1
- package/lib/archive/create-entity-tables.js +6 -1
- package/lib/archive/database.d.ts +39 -9
- package/lib/archive/database.js +63 -18
- package/lib/archive/db-ops/lifecycle/init.d.ts +14 -2
- package/lib/archive/db-ops/lifecycle/init.js +29 -11
- package/lib/archive/db-ops/meta/get-custom-elements-of-page.d.ts +11 -0
- package/lib/archive/db-ops/meta/get-custom-elements-of-page.js +15 -0
- package/lib/archive/db-ops/meta/get-page-technologies-of-page.d.ts +11 -0
- package/lib/archive/db-ops/meta/get-page-technologies-of-page.js +15 -0
- package/lib/archive/db-ops/meta/get-technology-signals-of-page.d.ts +11 -0
- package/lib/archive/db-ops/meta/get-technology-signals-of-page.js +15 -0
- package/lib/archive/db-ops/pages/order/set-url-order.d.ts +6 -1
- package/lib/archive/db-ops/pages/order/set-url-order.js +7 -1
- package/lib/archive/db-ops/pages/read/build-page-query.js +1 -1
- package/lib/archive/db-ops/pages/reset/repromote-external-pages.d.ts +6 -1
- package/lib/archive/db-ops/pages/reset/repromote-external-pages.js +12 -3
- package/lib/archive/db-ops/pages/reset/reset-failed-pages.d.ts +6 -1
- package/lib/archive/db-ops/pages/reset/reset-failed-pages.js +10 -2
- package/lib/archive/db-ops/pages/write/insert-custom-elements.d.ts +27 -0
- package/lib/archive/db-ops/pages/write/insert-custom-elements.js +40 -0
- package/lib/archive/db-ops/pages/write/insert-page.d.ts +24 -4
- package/lib/archive/db-ops/pages/write/insert-page.js +40 -4
- package/lib/archive/db-ops/pages/write/insert-technologies.d.ts +25 -0
- package/lib/archive/db-ops/pages/write/insert-technologies.js +45 -0
- package/lib/archive/db-ops/pages/write/link-redirect-sources.d.ts +7 -1
- package/lib/archive/db-ops/pages/write/link-redirect-sources.js +8 -2
- package/lib/archive/db-ops/pages/write/update-page.d.ts +4 -1
- package/lib/archive/db-ops/pages/write/update-page.js +38 -16
- package/lib/archive/db-ops/resources/get-resource-url-list.d.ts +12 -1
- package/lib/archive/db-ops/resources/get-resource-url-list.js +43 -5
- package/lib/archive/filesystem/copy-file-with-progress.d.ts +20 -0
- package/lib/archive/filesystem/copy-file-with-progress.js +34 -0
- package/lib/archive/filesystem/parse-pax-path.d.ts +18 -0
- package/lib/archive/filesystem/parse-pax-path.js +47 -0
- package/lib/archive/filesystem/parse-tar-size-field.d.ts +13 -0
- package/lib/archive/filesystem/parse-tar-size-field.js +40 -0
- package/lib/archive/filesystem/peek-tar-top-dir.d.ts +10 -3
- package/lib/archive/filesystem/peek-tar-top-dir.js +166 -16
- package/lib/archive/filesystem/tar.d.ts +13 -1
- package/lib/archive/filesystem/tar.js +63 -7
- package/lib/archive/filesystem/untar.d.ts +13 -0
- package/lib/archive/filesystem/untar.js +68 -8
- package/lib/archive/init-schema.d.ts +1 -1
- package/lib/archive/init-schema.js +6 -4
- package/lib/archive/meta/compute-main-contents-denormalized.d.ts +13 -2
- package/lib/archive/meta/compute-main-contents-denormalized.js +15 -2
- package/lib/archive/meta/technologies/combine-technology-confidence.d.ts +46 -0
- package/lib/archive/meta/technologies/combine-technology-confidence.js +93 -0
- package/lib/archive/meta/technologies/convert-legacy-page-tags-to-inserts.d.ts +45 -0
- package/lib/archive/meta/technologies/convert-legacy-page-tags-to-inserts.js +53 -0
- package/lib/archive/meta/technologies/convert-tag-row-to-wappalyzer-signal.d.ts +33 -0
- package/lib/archive/meta/technologies/convert-tag-row-to-wappalyzer-signal.js +46 -0
- package/lib/archive/meta/technologies/extract-technologies-for-archive.d.ts +30 -0
- package/lib/archive/meta/technologies/extract-technologies-for-archive.js +27 -0
- package/lib/archive/meta/technologies/match-structural-technology-signals.d.ts +13 -0
- package/lib/archive/meta/technologies/match-structural-technology-signals.js +44 -0
- package/lib/archive/meta/technologies/normalize-wappalyzer-entries.d.ts +25 -0
- package/lib/archive/meta/technologies/normalize-wappalyzer-entries.js +35 -0
- package/lib/archive/meta/technologies/technology-signal-definitions.d.ts +43 -0
- package/lib/archive/meta/technologies/technology-signal-definitions.js +193 -0
- package/lib/archive/meta/technologies/types.d.ts +68 -0
- package/lib/archive/meta/technologies/types.js +13 -0
- package/lib/archive/meta/types.d.ts +97 -30
- package/lib/archive/migrate-content-items-alias-of-id.d.ts +6 -1
- package/lib/archive/migrate-content-items-alias-of-id.js +14 -3
- package/lib/archive/migrate-content-items-dedupe-cap-event-id.d.ts +6 -1
- package/lib/archive/migrate-content-items-dedupe-cap-event-id.js +14 -3
- package/lib/archive/migrate-info-main-content-selector.d.ts +6 -1
- package/lib/archive/migrate-info-main-content-selector.js +14 -3
- package/lib/archive/migrate-info-roots.d.ts +6 -1
- package/lib/archive/migrate-info-roots.js +14 -3
- package/lib/archive/migrate-inventory-runs-exclude-skipped.d.ts +6 -1
- package/lib/archive/migrate-inventory-runs-exclude-skipped.js +14 -3
- package/lib/archive/migrate-inventory-runs-invalid-skipped.d.ts +6 -1
- package/lib/archive/migrate-inventory-runs-invalid-skipped.js +14 -3
- package/lib/archive/migrate-main-contents-columns.d.ts +6 -1
- package/lib/archive/migrate-main-contents-columns.js +14 -3
- package/lib/archive/migrate-page-meta-body-hash.d.ts +6 -1
- package/lib/archive/migrate-page-meta-body-hash.js +14 -3
- package/lib/archive/migrate-page-meta-console-error-count.d.ts +6 -1
- package/lib/archive/migrate-page-meta-console-error-count.js +14 -3
- package/lib/archive/migrate-page-meta-custom-element-count.d.ts +33 -0
- package/lib/archive/migrate-page-meta-custom-element-count.js +51 -0
- package/lib/archive/migrate-page-tags-to-page-technologies.d.ts +39 -0
- package/lib/archive/migrate-page-tags-to-page-technologies.js +79 -0
- package/lib/archive/page.d.ts +37 -13
- package/lib/archive/page.js +45 -17
- package/lib/archive/retarget-legacy-fk-tables.d.ts +27 -14
- package/lib/archive/retarget-legacy-fk-tables.js +61 -15
- package/lib/archive/types.d.ts +18 -0
- package/lib/crawler/assert-puppeteer-shared-with-beholder.d.ts +24 -0
- package/lib/crawler/assert-puppeteer-shared-with-beholder.js +36 -0
- package/lib/crawler/capture-custom-elements.d.ts +33 -0
- package/lib/crawler/capture-custom-elements.js +39 -0
- package/lib/crawler/close-browser-safely.d.ts +3 -3
- package/lib/crawler/close-browser-safely.js +8 -17
- package/lib/crawler/collect-custom-elements.d.ts +33 -0
- package/lib/crawler/collect-custom-elements.js +110 -0
- package/lib/crawler/crawler.js +101 -46
- package/lib/crawler/fetch-destination.js +6 -13
- package/lib/crawler/find-package-dir.d.ts +24 -0
- package/lib/crawler/find-package-dir.js +39 -0
- package/lib/crawler/scan-js-resource-for-license-comment.d.ts +32 -0
- package/lib/crawler/scan-js-resource-for-license-comment.js +140 -0
- package/lib/crawler/scan-js-resources-for-technology-signals.d.ts +58 -0
- package/lib/crawler/scan-js-resources-for-technology-signals.js +196 -0
- package/lib/crawler/types.d.ts +23 -0
- package/lib/crawler-orchestrator.d.ts +53 -9
- package/lib/crawler-orchestrator.js +223 -46
- package/lib/crawler.d.ts +11 -1
- package/lib/crawler.js +9 -0
- package/lib/inventory-setup-phases.d.ts +32 -0
- package/lib/inventory-setup-phases.js +45 -0
- package/lib/resume-setup-phases.d.ts +16 -0
- package/lib/resume-setup-phases.js +24 -0
- package/lib/retry-failed-setup-phases.d.ts +19 -0
- package/lib/retry-failed-setup-phases.js +24 -0
- package/lib/setup-recovery-phase-labels.d.ts +21 -0
- package/lib/setup-recovery-phase-labels.js +24 -0
- package/lib/types.d.ts +145 -0
- package/package.json +12 -9
- package/lib/archive/db-ops/meta/get-tags-of-page.d.ts +0 -12
- package/lib/archive/db-ops/meta/get-tags-of-page.js +0 -28
- package/lib/archive/db-ops/pages/write/insert-tags.d.ts +0 -16
- package/lib/archive/db-ops/pages/write/insert-tags.js +0 -34
- package/lib/archive/meta/summarize-tags.d.ts +0 -16
- package/lib/archive/meta/summarize-tags.js +0 -33
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fixed fingerprint list for the frontend frameworks this module detects via
|
|
3
|
+
* structural signals (URL path fragments, HTML markers, scoped-attribute
|
|
4
|
+
* conventions). Deliberately a TypeScript array, not an external config
|
|
5
|
+
* file (per the accepted design: precision/coverage is refined by editing
|
|
6
|
+
* this file, not by runtime configuration).
|
|
7
|
+
*
|
|
8
|
+
* Every framework here is ALSO detectable via {@link WAPPALYZER_PROVIDER_TO_TECHNOLOGY}
|
|
9
|
+
* when beholder's Wappalyzer pass independently reports it — the two signal
|
|
10
|
+
* sources compound in `combineTechnologyConfidence` rather than one
|
|
11
|
+
* replacing the other. Wappalyzer detects many more technologies than the
|
|
12
|
+
* frameworks listed here (analytics, CMSes, ...); those pass through with
|
|
13
|
+
* the provider name verbatim as the technology name, uncurated.
|
|
14
|
+
* @module
|
|
15
|
+
*/
|
|
16
|
+
export const TECHNOLOGY_SIGNAL_DEFINITIONS = [
|
|
17
|
+
// Next.js
|
|
18
|
+
{
|
|
19
|
+
technology: 'Next.js',
|
|
20
|
+
signalType: 'html-marker',
|
|
21
|
+
pattern: /<script[^>]*\sid="__NEXT_DATA__"/i,
|
|
22
|
+
weight: 70,
|
|
23
|
+
category: 'JavaScript frameworks',
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
technology: 'Next.js',
|
|
27
|
+
signalType: 'html-marker',
|
|
28
|
+
pattern: /<div[^>]*\sid="__next"/i,
|
|
29
|
+
weight: 60,
|
|
30
|
+
category: 'JavaScript frameworks',
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
technology: 'Next.js',
|
|
34
|
+
signalType: 'url-pattern',
|
|
35
|
+
pattern: /\/_next\//,
|
|
36
|
+
weight: 50,
|
|
37
|
+
category: 'JavaScript frameworks',
|
|
38
|
+
},
|
|
39
|
+
// Astro
|
|
40
|
+
{
|
|
41
|
+
technology: 'Astro',
|
|
42
|
+
signalType: 'html-marker',
|
|
43
|
+
pattern: /<astro-island\b/i,
|
|
44
|
+
weight: 65,
|
|
45
|
+
category: 'Static site generator',
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
technology: 'Astro',
|
|
49
|
+
signalType: 'url-pattern',
|
|
50
|
+
pattern: /\/_astro\//,
|
|
51
|
+
weight: 50,
|
|
52
|
+
category: 'Static site generator',
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
technology: 'Astro',
|
|
56
|
+
signalType: 'scoped-attr',
|
|
57
|
+
pattern: /\sdata-astro-cid-[a-z0-9]+/i,
|
|
58
|
+
weight: 40,
|
|
59
|
+
category: 'Static site generator',
|
|
60
|
+
},
|
|
61
|
+
// Vue
|
|
62
|
+
{
|
|
63
|
+
technology: 'Vue',
|
|
64
|
+
signalType: 'scoped-attr',
|
|
65
|
+
pattern: /\sdata-v-[a-z0-9]{6,10}\b/i,
|
|
66
|
+
weight: 40,
|
|
67
|
+
category: 'JavaScript frameworks',
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
technology: 'Vue',
|
|
71
|
+
signalType: 'weak-marker',
|
|
72
|
+
pattern: /\sid="app"/i,
|
|
73
|
+
weight: 15,
|
|
74
|
+
category: 'JavaScript frameworks',
|
|
75
|
+
},
|
|
76
|
+
{
|
|
77
|
+
technology: 'Vue',
|
|
78
|
+
signalType: 'js-license-comment',
|
|
79
|
+
pattern: /\/\*!\s*\*\s*Vue\.js v\d/i,
|
|
80
|
+
weight: 55,
|
|
81
|
+
category: 'JavaScript frameworks',
|
|
82
|
+
},
|
|
83
|
+
// Nuxt
|
|
84
|
+
{
|
|
85
|
+
technology: 'Nuxt',
|
|
86
|
+
signalType: 'html-marker',
|
|
87
|
+
pattern: /<div[^>]*\sid="__nuxt"/i,
|
|
88
|
+
weight: 60,
|
|
89
|
+
category: 'JavaScript frameworks',
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
technology: 'Nuxt',
|
|
93
|
+
signalType: 'url-pattern',
|
|
94
|
+
pattern: /\/_nuxt\//,
|
|
95
|
+
weight: 50,
|
|
96
|
+
category: 'JavaScript frameworks',
|
|
97
|
+
},
|
|
98
|
+
// Svelte / SvelteKit
|
|
99
|
+
{
|
|
100
|
+
technology: 'Svelte',
|
|
101
|
+
signalType: 'scoped-attr',
|
|
102
|
+
pattern: /class="[^"]*\bsvelte-[a-z0-9]{6,8}\b/i,
|
|
103
|
+
weight: 40,
|
|
104
|
+
category: 'JavaScript frameworks',
|
|
105
|
+
},
|
|
106
|
+
{
|
|
107
|
+
technology: 'SvelteKit',
|
|
108
|
+
signalType: 'url-pattern',
|
|
109
|
+
pattern: /\/_app\/immutable\//,
|
|
110
|
+
weight: 50,
|
|
111
|
+
category: 'JavaScript frameworks',
|
|
112
|
+
},
|
|
113
|
+
// Remix
|
|
114
|
+
{
|
|
115
|
+
technology: 'Remix',
|
|
116
|
+
signalType: 'html-marker',
|
|
117
|
+
pattern: /window\.__remixContext\s*=/i,
|
|
118
|
+
weight: 65,
|
|
119
|
+
category: 'JavaScript frameworks',
|
|
120
|
+
},
|
|
121
|
+
// Gatsby
|
|
122
|
+
{
|
|
123
|
+
technology: 'Gatsby',
|
|
124
|
+
signalType: 'html-marker',
|
|
125
|
+
pattern: /<div[^>]*\sid="___gatsby"/i,
|
|
126
|
+
weight: 65,
|
|
127
|
+
category: 'Static site generator',
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
technology: 'Gatsby',
|
|
131
|
+
signalType: 'url-pattern',
|
|
132
|
+
pattern: /\/page-data\//,
|
|
133
|
+
weight: 50,
|
|
134
|
+
category: 'Static site generator',
|
|
135
|
+
},
|
|
136
|
+
// Angular
|
|
137
|
+
{
|
|
138
|
+
technology: 'Angular',
|
|
139
|
+
signalType: 'html-marker',
|
|
140
|
+
pattern: /\sng-version="[\d.]+"/i,
|
|
141
|
+
weight: 65,
|
|
142
|
+
category: 'JavaScript frameworks',
|
|
143
|
+
},
|
|
144
|
+
{
|
|
145
|
+
technology: 'Angular',
|
|
146
|
+
signalType: 'scoped-attr',
|
|
147
|
+
pattern: /\s_ngcontent-[a-z0-9-]+/i,
|
|
148
|
+
weight: 40,
|
|
149
|
+
category: 'JavaScript frameworks',
|
|
150
|
+
},
|
|
151
|
+
];
|
|
152
|
+
/**
|
|
153
|
+
* `<meta name="generator">` prefix matchers, checked case-insensitively
|
|
154
|
+
* against `page_meta.generator` (already extracted — no new HTML scan
|
|
155
|
+
* needed). Distinct from {@link TECHNOLOGY_SIGNAL_DEFINITIONS} because the
|
|
156
|
+
* match strategy is prefix-on-a-known-field, not regex-over-HTML.
|
|
157
|
+
*/
|
|
158
|
+
export const GENERATOR_TECHNOLOGY_MATCHERS = [
|
|
159
|
+
{ technology: 'Astro', prefix: 'astro', weight: 85, category: 'Static site generator' },
|
|
160
|
+
{ technology: 'Nuxt', prefix: 'nuxt', weight: 85, category: 'JavaScript frameworks' },
|
|
161
|
+
{
|
|
162
|
+
technology: 'Gatsby',
|
|
163
|
+
prefix: 'gatsby',
|
|
164
|
+
weight: 85,
|
|
165
|
+
category: 'Static site generator',
|
|
166
|
+
},
|
|
167
|
+
];
|
|
168
|
+
/**
|
|
169
|
+
* Wappalyzer provider name → normalized technology name, for the curated
|
|
170
|
+
* frameworks that also have {@link TECHNOLOGY_SIGNAL_DEFINITIONS}. Providers
|
|
171
|
+
* NOT in this map still become `page_technologies` rows — the technology
|
|
172
|
+
* name is just the provider name verbatim (see `normalize-wappalyzer-entries.ts`)
|
|
173
|
+
* — this map exists only to fold curated frameworks' Wappalyzer detections
|
|
174
|
+
* onto the same technology name their structural signals use, so the two
|
|
175
|
+
* sources compound instead of appearing as two separate rows.
|
|
176
|
+
*/
|
|
177
|
+
export const WAPPALYZER_PROVIDER_TO_TECHNOLOGY = {
|
|
178
|
+
'Next.js': 'Next.js',
|
|
179
|
+
Astro: 'Astro',
|
|
180
|
+
'Vue.js': 'Vue',
|
|
181
|
+
'Nuxt.js': 'Nuxt',
|
|
182
|
+
Nuxt: 'Nuxt',
|
|
183
|
+
Svelte: 'Svelte',
|
|
184
|
+
SvelteKit: 'SvelteKit',
|
|
185
|
+
Remix: 'Remix',
|
|
186
|
+
Gatsby: 'Gatsby',
|
|
187
|
+
Angular: 'Angular',
|
|
188
|
+
AngularJS: 'Angular',
|
|
189
|
+
};
|
|
190
|
+
/** Confidence a lone `wappalyzer` signal contributes, before compounding with any structural signal. */
|
|
191
|
+
export const WAPPALYZER_SIGNAL_WEIGHT = 60;
|
|
192
|
+
/** Confidence threshold (`combineTechnologyConfidence`'s output) at or above which a `page_technologies` row is written at all. Below this, a single weak/ambiguous signal is treated as noise, not a detection. */
|
|
193
|
+
export const TECHNOLOGY_CONFIDENCE_THRESHOLD = 30;
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Domain types for technology-signal extraction (crawl time) and confidence
|
|
3
|
+
* roll-up. Kept in this dedicated `technologies/types.ts` rather than the
|
|
4
|
+
* parent `meta/types.ts` because these types are internal to the extraction
|
|
5
|
+
* pipeline (`match-structural-technology-signals.ts` →
|
|
6
|
+
* `combine-technology-confidence.ts` → `extract-technologies-for-archive.ts`)
|
|
7
|
+
* and never cross the `archive/meta/types.ts` boundary that the persisted
|
|
8
|
+
* `technology_signals` / `page_technologies` row shapes (`TechnologySignalRow`
|
|
9
|
+
* / `PageTechnologyRow`) live in — see the repo-wide `型は types.ts に集約`
|
|
10
|
+
* rule, applied per-module rather than globally so this module's internal
|
|
11
|
+
* vocabulary does not leak into the db-ops layer's public row types.
|
|
12
|
+
*/
|
|
13
|
+
/**
|
|
14
|
+
* How a technology signal was detected. `wappalyzer` is beholder's
|
|
15
|
+
* `simple-wappalyzer`-based detection (imprecise on its own for frameworks,
|
|
16
|
+
* but authoritative for the long tail of non-framework technologies —
|
|
17
|
+
* analytics, CMSes, etc. — this module does not otherwise attempt to
|
|
18
|
+
* detect). The rest are structural signals this module detects itself.
|
|
19
|
+
*/
|
|
20
|
+
export type TechnologySignalType = 'wappalyzer' | 'meta-generator' | 'html-marker' | 'url-pattern' | 'scoped-attr' | 'weak-marker' | 'js-license-comment';
|
|
21
|
+
/**
|
|
22
|
+
* One un-combined signal for one technology on one page. The unit that
|
|
23
|
+
* `combineTechnologyConfidence` groups (by `technology`) and reduces via
|
|
24
|
+
* noisy-OR into a single {@link PageTechnologyPartial} per technology.
|
|
25
|
+
*/
|
|
26
|
+
export interface TechnologySignalPartial {
|
|
27
|
+
/** Normalized technology name (e.g. `'Next.js'`, `'Google Analytics'`). */
|
|
28
|
+
technology: string;
|
|
29
|
+
signalType: TechnologySignalType;
|
|
30
|
+
/** Matched fragment or raw value, truncated to ~200 chars by the caller. `null` when the signal type carries no useful excerpt. */
|
|
31
|
+
evidence: string | null;
|
|
32
|
+
/** This signal's confidence in isolation, 0-100. */
|
|
33
|
+
weight: number;
|
|
34
|
+
/** Wappalyzer/generator-reported category, when this signal type can supply one. */
|
|
35
|
+
category?: string | null;
|
|
36
|
+
/** Wappalyzer/generator-reported version, when this signal type can supply one. */
|
|
37
|
+
version?: string | null;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* One technology's combined roll-up for one page, before the `pageId` is
|
|
41
|
+
* known (added by the caller at insert time — same "id injected at write
|
|
42
|
+
* time" pattern as {@link import('../types.js').TagRowForInsert}).
|
|
43
|
+
*/
|
|
44
|
+
export interface PageTechnologyPartial {
|
|
45
|
+
technology: string;
|
|
46
|
+
category: string | null;
|
|
47
|
+
version: string | null;
|
|
48
|
+
/** `combineTechnologyConfidence`'s noisy-OR result, 0-100. */
|
|
49
|
+
confidence: number;
|
|
50
|
+
/** Count of distinct `signalType`s that contributed to `confidence`. */
|
|
51
|
+
signalCount: number;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* A single fingerprint definition: one technology matched by one signal
|
|
55
|
+
* type via one regular expression, at one isolated weight. Multiple
|
|
56
|
+
* definitions may share a `technology` (different signal types/patterns)
|
|
57
|
+
* — {@link combineTechnologyConfidence} is what lets those compound.
|
|
58
|
+
*/
|
|
59
|
+
export interface TechnologySignalDefinition {
|
|
60
|
+
technology: string;
|
|
61
|
+
signalType: 'html-marker' | 'url-pattern' | 'scoped-attr' | 'weak-marker' | 'js-license-comment';
|
|
62
|
+
/** Tested against the page's raw HTML string (or, for `js-license-comment`, a JS resource's leading bytes). */
|
|
63
|
+
pattern: RegExp;
|
|
64
|
+
/** This definition's confidence in isolation, 0-100, when `pattern` matches. */
|
|
65
|
+
weight: number;
|
|
66
|
+
/** Category to attach to the technology when only this signal fired. */
|
|
67
|
+
category?: string;
|
|
68
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Domain types for technology-signal extraction (crawl time) and confidence
|
|
3
|
+
* roll-up. Kept in this dedicated `technologies/types.ts` rather than the
|
|
4
|
+
* parent `meta/types.ts` because these types are internal to the extraction
|
|
5
|
+
* pipeline (`match-structural-technology-signals.ts` →
|
|
6
|
+
* `combine-technology-confidence.ts` → `extract-technologies-for-archive.ts`)
|
|
7
|
+
* and never cross the `archive/meta/types.ts` boundary that the persisted
|
|
8
|
+
* `technology_signals` / `page_technologies` row shapes (`TechnologySignalRow`
|
|
9
|
+
* / `PageTechnologyRow`) live in — see the repo-wide `型は types.ts に集約`
|
|
10
|
+
* rule, applied per-module rather than globally so this module's internal
|
|
11
|
+
* vocabulary does not leak into the db-ops layer's public row types.
|
|
12
|
+
*/
|
|
13
|
+
export {};
|
|
@@ -122,6 +122,7 @@ export interface PageDenormalizedColumns {
|
|
|
122
122
|
* main_content_video_count: 0,
|
|
123
123
|
* main_content_audio_count: 0,
|
|
124
124
|
* main_content_canvas_count: 0,
|
|
125
|
+
* main_content_custom_element_count: 0,
|
|
125
126
|
* scroll_height_desktop: 3200,
|
|
126
127
|
* scroll_height_mobile: 5400,
|
|
127
128
|
* };
|
|
@@ -157,6 +158,16 @@ export interface MainContentsDenormalizedColumns {
|
|
|
157
158
|
main_content_audio_count: number | null;
|
|
158
159
|
/** Number of canvases within the main region, or `null`. */
|
|
159
160
|
main_content_canvas_count: number | null;
|
|
161
|
+
/**
|
|
162
|
+
* Number of Web Components (custom elements) within the main region, or
|
|
163
|
+
* `null`. Unlike its siblings above, `null` is not solely "page not
|
|
164
|
+
* rendered" — it also covers "rendered, but nitpicker's own
|
|
165
|
+
* `captureCustomElements` best-effort capture failed" (a distinct state
|
|
166
|
+
* from "captured, zero found" = `0`), since this column is not sourced
|
|
167
|
+
* from beholder's `MainContentsData` at all. See
|
|
168
|
+
* `compute-main-contents-denormalized.ts`.
|
|
169
|
+
*/
|
|
170
|
+
main_content_custom_element_count: number | null;
|
|
160
171
|
/** `document.body.scrollHeight` at the desktop-compact preset, or `null`. */
|
|
161
172
|
scroll_height_desktop: number | null;
|
|
162
173
|
/** `document.body.scrollHeight` at the mobile-small preset, or `null`. */
|
|
@@ -356,6 +367,27 @@ export interface MainContentCanvasRow {
|
|
|
356
367
|
/** IDL bitmap height. */
|
|
357
368
|
height: number;
|
|
358
369
|
}
|
|
370
|
+
/**
|
|
371
|
+
* One row in the `page_main_content_custom_elements` table. Unlike its
|
|
372
|
+
* seven `MainContentXxxRow` siblings, the source data is not beholder's
|
|
373
|
+
* `MainContentsData` but nitpicker's own `capture-custom-elements.ts`.
|
|
374
|
+
* @example
|
|
375
|
+
* const row: MainContentCustomElementRow = { id: 1, pageId: 42, order: 0, nodeName: 'MY-WIDGET', elementId: 'widget-1', classList: '["foo"]' };
|
|
376
|
+
*/
|
|
377
|
+
export interface MainContentCustomElementRow {
|
|
378
|
+
/** Auto-increment primary key. */
|
|
379
|
+
id: number;
|
|
380
|
+
/** FK to `content_items.id`. */
|
|
381
|
+
pageId: number;
|
|
382
|
+
/** 0-based DOM traversal order within the main content region. */
|
|
383
|
+
order: number;
|
|
384
|
+
/** The element's `nodeName` (e.g. `'MY-WIDGET'`). */
|
|
385
|
+
nodeName: string;
|
|
386
|
+
/** The element's `id` attribute, or `null`. */
|
|
387
|
+
elementId: string | null;
|
|
388
|
+
/** JSON-encoded array of the element's CSS classes, or `null`. */
|
|
389
|
+
classList: string | null;
|
|
390
|
+
}
|
|
359
391
|
/**
|
|
360
392
|
* One row in the `page_jsonld` table.
|
|
361
393
|
*
|
|
@@ -474,6 +506,71 @@ export interface TagRow {
|
|
|
474
506
|
* };
|
|
475
507
|
*/
|
|
476
508
|
export type TagRowForInsert = Omit<TagRow, 'id'>;
|
|
509
|
+
/**
|
|
510
|
+
* One row in the `technology_signals` table — one un-combined signal for
|
|
511
|
+
* one technology on one page. See
|
|
512
|
+
* `archive/meta/technologies/types.ts#TechnologySignalPartial` for the
|
|
513
|
+
* pre-insert shape (no `pageId`/`id`) this is built from.
|
|
514
|
+
* @example
|
|
515
|
+
* const row: TechnologySignalRow = {
|
|
516
|
+
* id: 1,
|
|
517
|
+
* pageId: 42,
|
|
518
|
+
* technology: 'Next.js',
|
|
519
|
+
* signalType: 'html-marker',
|
|
520
|
+
* evidence: '<script id="__NEXT_DATA__"',
|
|
521
|
+
* weight: 70,
|
|
522
|
+
* };
|
|
523
|
+
*/
|
|
524
|
+
export interface TechnologySignalRow {
|
|
525
|
+
/** Auto-increment primary key. */
|
|
526
|
+
id: number;
|
|
527
|
+
/** FK to `content_items.id`. */
|
|
528
|
+
pageId: number;
|
|
529
|
+
/** Normalized technology name (e.g. `'Next.js'`, `'Google Analytics'`). */
|
|
530
|
+
technology: string;
|
|
531
|
+
/** How this signal was detected. */
|
|
532
|
+
signalType: 'wappalyzer' | 'meta-generator' | 'html-marker' | 'url-pattern' | 'scoped-attr' | 'weak-marker' | 'js-license-comment';
|
|
533
|
+
/** Matched fragment or raw value, truncated to ~200 chars, or `null`. */
|
|
534
|
+
evidence: string | null;
|
|
535
|
+
/** This signal's confidence in isolation, 0-100. */
|
|
536
|
+
weight: number;
|
|
537
|
+
}
|
|
538
|
+
/** Insert shape for {@link TechnologySignalRow}. */
|
|
539
|
+
export type TechnologySignalRowForInsert = Omit<TechnologySignalRow, 'id'>;
|
|
540
|
+
/**
|
|
541
|
+
* One row in the `page_technologies` table — the confidence-combined
|
|
542
|
+
* roll-up of every {@link TechnologySignalRow} for one technology on one
|
|
543
|
+
* page. Read-optimised counterpart of `technology_signals`, analogous to
|
|
544
|
+
* `page_meta.tag_count` being the roll-up of `page_tags` — except here the
|
|
545
|
+
* roll-up is a full row, not just a count, because `confidence` is a
|
|
546
|
+
* per-technology computed value (`combineTechnologyConfidence`), not a
|
|
547
|
+
* simple count.
|
|
548
|
+
* @example
|
|
549
|
+
* const row: PageTechnologyRow = {
|
|
550
|
+
* id: 1,
|
|
551
|
+
* pageId: 42,
|
|
552
|
+
* technology: 'Next.js',
|
|
553
|
+
* category: 'JavaScript frameworks',
|
|
554
|
+
* version: null,
|
|
555
|
+
* confidence: 80,
|
|
556
|
+
* signalCount: 2,
|
|
557
|
+
* };
|
|
558
|
+
*/
|
|
559
|
+
export interface PageTechnologyRow {
|
|
560
|
+
/** Auto-increment primary key. */
|
|
561
|
+
id: number;
|
|
562
|
+
/** FK to `content_items.id`. */
|
|
563
|
+
pageId: number;
|
|
564
|
+
technology: string;
|
|
565
|
+
category: string | null;
|
|
566
|
+
version: string | null;
|
|
567
|
+
/** `combineTechnologyConfidence`'s noisy-OR result, 0-100. */
|
|
568
|
+
confidence: number;
|
|
569
|
+
/** Count of distinct `signalType`s that contributed to `confidence`. */
|
|
570
|
+
signalCount: number;
|
|
571
|
+
}
|
|
572
|
+
/** Insert shape for {@link PageTechnologyRow}. */
|
|
573
|
+
export type PageTechnologyRowForInsert = Omit<PageTechnologyRow, 'id'>;
|
|
477
574
|
/**
|
|
478
575
|
* Summary of one page's JSON-LD entries returned by `get-page-detail`.
|
|
479
576
|
*
|
|
@@ -495,36 +592,6 @@ export interface JsonLdSummary {
|
|
|
495
592
|
/** Number of entries that failed to parse (i.e. have a non-null `parseError`). */
|
|
496
593
|
parseErrorCount: number;
|
|
497
594
|
}
|
|
498
|
-
/**
|
|
499
|
-
* Summary of one page's Wappalyzer tags returned by `get-page-detail`.
|
|
500
|
-
* @see summarize-tags.ts
|
|
501
|
-
* @example
|
|
502
|
-
* const summary: TagsSummary = {
|
|
503
|
-
* count: 2,
|
|
504
|
-
* providerIds: { 'Google Analytics': ['G-XXXX'], YouTube: [] },
|
|
505
|
-
* };
|
|
506
|
-
*/
|
|
507
|
-
export interface TagsSummary {
|
|
508
|
-
/** Total tag rows for the page. */
|
|
509
|
-
count: number;
|
|
510
|
-
/** Provider → list of external IDs (unique, sorted). Providers with no IDs map to `[]`. */
|
|
511
|
-
providerIds: Readonly<Record<string, readonly string[]>>;
|
|
512
|
-
}
|
|
513
|
-
/**
|
|
514
|
-
* One row of {@link import('@d-zero/beholder').Meta.tags.detected} after
|
|
515
|
-
* archive-side flattening.
|
|
516
|
-
*
|
|
517
|
-
* Used by `get-tag-inventory` to return per-provider page counts across the
|
|
518
|
-
* whole site.
|
|
519
|
-
* @example
|
|
520
|
-
* const entry: TagInventoryEntry = { provider: 'Google Tag Manager', pageCount: 120 };
|
|
521
|
-
*/
|
|
522
|
-
export interface TagInventoryEntry {
|
|
523
|
-
/** Wappalyzer provider name. */
|
|
524
|
-
provider: string;
|
|
525
|
-
/** Number of distinct pages where the provider was detected. */
|
|
526
|
-
pageCount: number;
|
|
527
|
-
}
|
|
528
595
|
/**
|
|
529
596
|
* Error thrown by `assert-compatible-version` when the archive's
|
|
530
597
|
* `info.version` is older than the format version this build accepts.
|
|
@@ -43,5 +43,10 @@ import type { Knex } from 'knex';
|
|
|
43
43
|
* runs (after `initSchema`, itself after `assertCompatibleVersion` rejects
|
|
44
44
|
* pre-0.13 archives) the table is always present.
|
|
45
45
|
* @param instance - The Knex query builder instance connected to the database.
|
|
46
|
+
* @param onLog - Called instead of `console.error` when this migration
|
|
47
|
+
* actually applies (issue #294: a bare `console.error` here can fire
|
|
48
|
+
* while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
|
|
49
|
+
* `Archive.open`, corrupting its cursor tracking). Falls back to
|
|
50
|
+
* `console.error` when omitted (direct/test callers).
|
|
46
51
|
*/
|
|
47
|
-
export declare function migrateContentItemsAliasOfId(instance: Knex): Promise<void>;
|
|
52
|
+
export declare function migrateContentItemsAliasOfId(instance: Knex, onLog?: (message: string) => void): Promise<void>;
|
|
@@ -42,8 +42,13 @@
|
|
|
42
42
|
* runs (after `initSchema`, itself after `assertCompatibleVersion` rejects
|
|
43
43
|
* pre-0.13 archives) the table is always present.
|
|
44
44
|
* @param instance - The Knex query builder instance connected to the database.
|
|
45
|
+
* @param onLog - Called instead of `console.error` when this migration
|
|
46
|
+
* actually applies (issue #294: a bare `console.error` here can fire
|
|
47
|
+
* while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
|
|
48
|
+
* `Archive.open`, corrupting its cursor tracking). Falls back to
|
|
49
|
+
* `console.error` when omitted (direct/test callers).
|
|
45
50
|
*/
|
|
46
|
-
export async function migrateContentItemsAliasOfId(instance) {
|
|
51
|
+
export async function migrateContentItemsAliasOfId(instance, onLog) {
|
|
47
52
|
const hasContentItems = await instance.schema.hasTable('content_items');
|
|
48
53
|
if (!hasContentItems) {
|
|
49
54
|
return;
|
|
@@ -51,8 +56,14 @@ export async function migrateContentItemsAliasOfId(instance) {
|
|
|
51
56
|
const hasColumn = await instance.schema.hasColumn('content_items', 'alias_of_id');
|
|
52
57
|
if (!hasColumn) {
|
|
53
58
|
await instance.raw('ALTER TABLE content_items ADD COLUMN alias_of_id INTEGER REFERENCES content_items(id) DEFERRABLE INITIALLY DEFERRED');
|
|
54
|
-
|
|
55
|
-
|
|
59
|
+
const message = '[migrate] content_items.alias_of_id column added';
|
|
60
|
+
if (onLog) {
|
|
61
|
+
onLog(message);
|
|
62
|
+
}
|
|
63
|
+
else {
|
|
64
|
+
// eslint-disable-next-line no-console
|
|
65
|
+
console.error(message);
|
|
66
|
+
}
|
|
56
67
|
}
|
|
57
68
|
await instance.raw('CREATE INDEX IF NOT EXISTS idx_content_items_alias_of_id ON content_items(alias_of_id)');
|
|
58
69
|
}
|
|
@@ -33,9 +33,14 @@ import type { Knex } from 'knex';
|
|
|
33
33
|
* (after `initSchema`, itself after `assertCompatibleVersion` rejects
|
|
34
34
|
* pre-0.13 archives) the table is always present.
|
|
35
35
|
* @param instance - The Knex query builder instance connected to the database.
|
|
36
|
+
* @param onLog - Called instead of `console.error` when this migration
|
|
37
|
+
* actually applies (issue #294: a bare `console.error` here can fire
|
|
38
|
+
* while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
|
|
39
|
+
* `Archive.open`, corrupting its cursor tracking). Falls back to
|
|
40
|
+
* `console.error` when omitted (direct/test callers).
|
|
36
41
|
* @example
|
|
37
42
|
* ```ts
|
|
38
43
|
* await migrateContentItemsDedupeCapEventId(knex);
|
|
39
44
|
* ```
|
|
40
45
|
*/
|
|
41
|
-
export declare function migrateContentItemsDedupeCapEventId(instance: Knex): Promise<void>;
|
|
46
|
+
export declare function migrateContentItemsDedupeCapEventId(instance: Knex, onLog?: (message: string) => void): Promise<void>;
|
|
@@ -32,12 +32,17 @@
|
|
|
32
32
|
* (after `initSchema`, itself after `assertCompatibleVersion` rejects
|
|
33
33
|
* pre-0.13 archives) the table is always present.
|
|
34
34
|
* @param instance - The Knex query builder instance connected to the database.
|
|
35
|
+
* @param onLog - Called instead of `console.error` when this migration
|
|
36
|
+
* actually applies (issue #294: a bare `console.error` here can fire
|
|
37
|
+
* while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
|
|
38
|
+
* `Archive.open`, corrupting its cursor tracking). Falls back to
|
|
39
|
+
* `console.error` when omitted (direct/test callers).
|
|
35
40
|
* @example
|
|
36
41
|
* ```ts
|
|
37
42
|
* await migrateContentItemsDedupeCapEventId(knex);
|
|
38
43
|
* ```
|
|
39
44
|
*/
|
|
40
|
-
export async function migrateContentItemsDedupeCapEventId(instance) {
|
|
45
|
+
export async function migrateContentItemsDedupeCapEventId(instance, onLog) {
|
|
41
46
|
const hasContentItems = await instance.schema.hasTable('content_items');
|
|
42
47
|
if (!hasContentItems) {
|
|
43
48
|
return;
|
|
@@ -45,7 +50,13 @@ export async function migrateContentItemsDedupeCapEventId(instance) {
|
|
|
45
50
|
const hasColumn = await instance.schema.hasColumn('content_items', 'dedupe_cap_event_id');
|
|
46
51
|
if (!hasColumn) {
|
|
47
52
|
await instance.raw('ALTER TABLE content_items ADD COLUMN dedupe_cap_event_id INTEGER REFERENCES dedupe_cap_events(id) DEFERRABLE INITIALLY DEFERRED');
|
|
48
|
-
|
|
49
|
-
|
|
53
|
+
const message = '[migrate] content_items.dedupe_cap_event_id column added';
|
|
54
|
+
if (onLog) {
|
|
55
|
+
onLog(message);
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
// eslint-disable-next-line no-console
|
|
59
|
+
console.error(message);
|
|
60
|
+
}
|
|
50
61
|
}
|
|
51
62
|
}
|
|
@@ -7,5 +7,10 @@ import type { Knex } from 'knex';
|
|
|
7
7
|
* idempotent, and self-healing for archives whose provisioning crashed
|
|
8
8
|
* partway through.
|
|
9
9
|
* @param instance - The Knex query builder instance connected to the database.
|
|
10
|
+
* @param onLog - Called instead of `console.error` when this migration
|
|
11
|
+
* actually applies (issue #294: a bare `console.error` here can fire
|
|
12
|
+
* while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
|
|
13
|
+
* `Archive.open`, corrupting its cursor tracking). Falls back to
|
|
14
|
+
* `console.error` when omitted (direct/test callers).
|
|
10
15
|
*/
|
|
11
|
-
export declare function migrateInfoMainContentSelector(instance: Knex): Promise<void>;
|
|
16
|
+
export declare function migrateInfoMainContentSelector(instance: Knex, onLog?: (message: string) => void): Promise<void>;
|
|
@@ -6,8 +6,13 @@
|
|
|
6
6
|
* idempotent, and self-healing for archives whose provisioning crashed
|
|
7
7
|
* partway through.
|
|
8
8
|
* @param instance - The Knex query builder instance connected to the database.
|
|
9
|
+
* @param onLog - Called instead of `console.error` when this migration
|
|
10
|
+
* actually applies (issue #294: a bare `console.error` here can fire
|
|
11
|
+
* while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
|
|
12
|
+
* `Archive.open`, corrupting its cursor tracking). Falls back to
|
|
13
|
+
* `console.error` when omitted (direct/test callers).
|
|
9
14
|
*/
|
|
10
|
-
export async function migrateInfoMainContentSelector(instance) {
|
|
15
|
+
export async function migrateInfoMainContentSelector(instance, onLog) {
|
|
11
16
|
const hasInfo = await instance.schema.hasTable('info');
|
|
12
17
|
if (!hasInfo) {
|
|
13
18
|
return;
|
|
@@ -19,6 +24,12 @@ export async function migrateInfoMainContentSelector(instance) {
|
|
|
19
24
|
await instance.schema.table('info', (t) => {
|
|
20
25
|
t.string('mainContentSelector');
|
|
21
26
|
});
|
|
22
|
-
|
|
23
|
-
|
|
27
|
+
const message = '[migrate] info.mainContentSelector column added';
|
|
28
|
+
if (onLog) {
|
|
29
|
+
onLog(message);
|
|
30
|
+
}
|
|
31
|
+
else {
|
|
32
|
+
// eslint-disable-next-line no-console
|
|
33
|
+
console.error(message);
|
|
34
|
+
}
|
|
24
35
|
}
|
|
@@ -11,5 +11,10 @@ import type { Knex } from 'knex';
|
|
|
11
11
|
* performed), a single notice is written to stderr so the user knows the
|
|
12
12
|
* file was upgraded.
|
|
13
13
|
* @param instance - The Knex query builder instance connected to the database.
|
|
14
|
+
* @param onLog - Called instead of `console.error` when this migration
|
|
15
|
+
* actually applies (issue #294: a bare `console.error` here can fire
|
|
16
|
+
* while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
|
|
17
|
+
* `Archive.open`, corrupting its cursor tracking). Falls back to
|
|
18
|
+
* `console.error` when omitted (direct/test callers).
|
|
14
19
|
*/
|
|
15
|
-
export declare function migrateInfoRoots(instance: Knex): Promise<void>;
|
|
20
|
+
export declare function migrateInfoRoots(instance: Knex, onLog?: (message: string) => void): Promise<void>;
|
|
@@ -10,8 +10,13 @@
|
|
|
10
10
|
* performed), a single notice is written to stderr so the user knows the
|
|
11
11
|
* file was upgraded.
|
|
12
12
|
* @param instance - The Knex query builder instance connected to the database.
|
|
13
|
+
* @param onLog - Called instead of `console.error` when this migration
|
|
14
|
+
* actually applies (issue #294: a bare `console.error` here can fire
|
|
15
|
+
* while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
|
|
16
|
+
* `Archive.open`, corrupting its cursor tracking). Falls back to
|
|
17
|
+
* `console.error` when omitted (direct/test callers).
|
|
13
18
|
*/
|
|
14
|
-
export async function migrateInfoRoots(instance) {
|
|
19
|
+
export async function migrateInfoRoots(instance, onLog) {
|
|
15
20
|
const hasInfo = await instance.schema.hasTable('info');
|
|
16
21
|
if (!hasInfo) {
|
|
17
22
|
return;
|
|
@@ -36,6 +41,12 @@ export async function migrateInfoRoots(instance) {
|
|
|
36
41
|
});
|
|
37
42
|
changes.push('scope dropped');
|
|
38
43
|
}
|
|
39
|
-
|
|
40
|
-
|
|
44
|
+
const message = `[migrate] info table upgraded (${changes.join(', ')})`;
|
|
45
|
+
if (onLog) {
|
|
46
|
+
onLog(message);
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
// eslint-disable-next-line no-console
|
|
50
|
+
console.error(message);
|
|
51
|
+
}
|
|
41
52
|
}
|
|
@@ -16,5 +16,10 @@ import type { Knex } from 'knex';
|
|
|
16
16
|
* only by `listInventoryRuns` display surfaces, never consumed by any
|
|
17
17
|
* runtime decision — matching `scope_skipped` / `invalid_skipped`.
|
|
18
18
|
* @param instance - The Knex query builder instance connected to the database.
|
|
19
|
+
* @param onLog - Called instead of `console.error` when this migration
|
|
20
|
+
* actually applies (issue #294: a bare `console.error` here can fire
|
|
21
|
+
* while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
|
|
22
|
+
* `Archive.open`, corrupting its cursor tracking). Falls back to
|
|
23
|
+
* `console.error` when omitted (direct/test callers).
|
|
19
24
|
*/
|
|
20
|
-
export declare function migrateInventoryRunsExcludeSkipped(instance: Knex): Promise<void>;
|
|
25
|
+
export declare function migrateInventoryRunsExcludeSkipped(instance: Knex, onLog?: (message: string) => void): Promise<void>;
|
|
@@ -15,8 +15,13 @@
|
|
|
15
15
|
* only by `listInventoryRuns` display surfaces, never consumed by any
|
|
16
16
|
* runtime decision — matching `scope_skipped` / `invalid_skipped`.
|
|
17
17
|
* @param instance - The Knex query builder instance connected to the database.
|
|
18
|
+
* @param onLog - Called instead of `console.error` when this migration
|
|
19
|
+
* actually applies (issue #294: a bare `console.error` here can fire
|
|
20
|
+
* while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
|
|
21
|
+
* `Archive.open`, corrupting its cursor tracking). Falls back to
|
|
22
|
+
* `console.error` when omitted (direct/test callers).
|
|
18
23
|
*/
|
|
19
|
-
export async function migrateInventoryRunsExcludeSkipped(instance) {
|
|
24
|
+
export async function migrateInventoryRunsExcludeSkipped(instance, onLog) {
|
|
20
25
|
const hasTable = await instance.schema.hasTable('inventory_runs');
|
|
21
26
|
if (!hasTable) {
|
|
22
27
|
return;
|
|
@@ -28,6 +33,12 @@ export async function migrateInventoryRunsExcludeSkipped(instance) {
|
|
|
28
33
|
await instance.schema.table('inventory_runs', (t) => {
|
|
29
34
|
t.integer('exclude_skipped');
|
|
30
35
|
});
|
|
31
|
-
|
|
32
|
-
|
|
36
|
+
const message = '[migrate] inventory_runs.exclude_skipped column added';
|
|
37
|
+
if (onLog) {
|
|
38
|
+
onLog(message);
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
// eslint-disable-next-line no-console
|
|
42
|
+
console.error(message);
|
|
43
|
+
}
|
|
33
44
|
}
|