@nitpicker/crawler 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/append-retry-failed-common-setup-phases.d.ts +1 -1
- package/lib/append-retry-failed-common-setup-phases.js +5 -1
- package/lib/append-setup-phases.d.ts +1 -1
- package/lib/archive/archive.d.ts +57 -8
- package/lib/archive/archive.js +83 -12
- package/lib/archive/create-adjunct-tables.d.ts +3 -2
- package/lib/archive/create-adjunct-tables.js +12 -9
- package/lib/archive/create-entity-tables.js +3 -0
- package/lib/archive/database.d.ts +39 -7
- package/lib/archive/database.js +49 -9
- package/lib/archive/db-ops/_shared/keyset-paginate-by-id.d.ts +49 -0
- package/lib/archive/db-ops/_shared/keyset-paginate-by-id.js +69 -0
- package/lib/archive/db-ops/_shared/resolve-content-item-id.d.ts +13 -12
- package/lib/archive/db-ops/_shared/resolve-content-item-id.js +63 -31
- package/lib/archive/db-ops/_shared/types.d.ts +39 -0
- package/lib/archive/db-ops/analysis/replace-analysis-violations.js +7 -4
- package/lib/archive/db-ops/config/get-created-cwd.d.ts +11 -0
- package/lib/archive/db-ops/config/get-created-cwd.js +13 -0
- package/lib/archive/db-ops/config/info-column-allowlist.js +1 -0
- package/lib/archive/db-ops/dedupe-cap/list-dedupe-cap-observations.d.ts +57 -0
- package/lib/archive/db-ops/dedupe-cap/list-dedupe-cap-observations.js +87 -0
- package/lib/archive/db-ops/errors/insert-page-error.js +3 -1
- package/lib/archive/db-ops/lifecycle/init.d.ts +13 -6
- package/lib/archive/db-ops/lifecycle/init.js +30 -10
- package/lib/archive/db-ops/list-reconcile/record-list-reconcile-run.d.ts +22 -0
- package/lib/archive/db-ops/{inventory/record-inventory-run.js → list-reconcile/record-list-reconcile-run.js} +11 -10
- package/lib/archive/db-ops/pages/read/build-page-query.js +1 -1
- package/lib/archive/db-ops/pages/read/get-crawling-state.d.ts +9 -2
- package/lib/archive/db-ops/pages/read/get-crawling-state.js +11 -3
- package/lib/archive/db-ops/pages/reset/clear-page-derived-rows.d.ts +20 -0
- package/lib/archive/db-ops/pages/reset/clear-page-derived-rows.js +37 -0
- package/lib/archive/db-ops/pages/reset/repromote-external-pages.d.ts +3 -2
- package/lib/archive/db-ops/pages/reset/repromote-external-pages.js +17 -37
- package/lib/archive/db-ops/pages/reset/reset-failed-pages.d.ts +22 -1
- package/lib/archive/db-ops/pages/reset/reset-failed-pages.js +79 -33
- package/lib/archive/db-ops/pages/reset/reset-pages-by-urls.d.ts +78 -0
- package/lib/archive/db-ops/pages/reset/reset-pages-by-urls.js +146 -0
- package/lib/archive/db-ops/pages/reset/retryable-image-scan-codes.d.ts +24 -0
- package/lib/archive/db-ops/pages/reset/retryable-image-scan-codes.js +24 -0
- package/lib/archive/db-ops/pages/write/insert-inventory-content-items.js +2 -1
- package/lib/archive/db-ops/pages/write/insert-page.js +4 -2
- package/lib/archive/db-ops/pages/write/link-redirect-sources.js +3 -1
- package/lib/archive/db-ops/pages/write/record-redirect.js +1 -1
- package/lib/archive/db-ops/pages/write/set-skipped-page.js +3 -1
- package/lib/archive/db-ops/pages/write/update-page.d.ts +9 -1
- package/lib/archive/db-ops/pages/write/update-page.js +28 -6
- package/lib/archive/db-ops/resources/get-resource-url-list.d.ts +3 -6
- package/lib/archive/db-ops/resources/get-resource-url-list.js +10 -35
- package/lib/archive/init-schema.d.ts +1 -1
- package/lib/archive/init-schema.js +3 -2
- package/lib/archive/meta/compute-main-contents-denormalized.d.ts +11 -3
- package/lib/archive/meta/compute-main-contents-denormalized.js +14 -2
- package/lib/archive/meta/types.d.ts +24 -0
- package/lib/archive/migrate-content-items-is-metadata-only.d.ts +44 -0
- package/lib/archive/migrate-content-items-is-metadata-only.js +60 -0
- package/lib/archive/migrate-info-created-cwd.d.ts +17 -0
- package/lib/archive/migrate-info-created-cwd.js +36 -0
- package/lib/archive/migrate-inventory-runs-to-list-reconcile-runs.d.ts +34 -0
- package/lib/archive/migrate-inventory-runs-to-list-reconcile-runs.js +51 -0
- package/lib/archive/{migrate-inventory-runs-exclude-skipped.d.ts → migrate-list-reconcile-runs-exclude-skipped.d.ts} +8 -8
- package/lib/archive/{migrate-inventory-runs-exclude-skipped.js → migrate-list-reconcile-runs-exclude-skipped.js} +12 -12
- package/lib/archive/migrate-list-reconcile-runs-invalid-skipped.d.ts +16 -0
- package/lib/archive/migrate-list-reconcile-runs-invalid-skipped.js +35 -0
- package/lib/archive/migrate-page-meta-image-scan.d.ts +27 -0
- package/lib/archive/migrate-page-meta-image-scan.js +52 -0
- package/lib/archive/page.d.ts +12 -0
- package/lib/archive/page.js +16 -0
- package/lib/archive/types.d.ts +97 -10
- package/lib/compute-auto-retry-backoff-delay.d.ts +30 -0
- package/lib/compute-auto-retry-backoff-delay.js +36 -0
- package/lib/crawler/apply-crawl-runtime-options-patch.d.ts +32 -0
- package/lib/crawler/apply-crawl-runtime-options-patch.js +129 -0
- package/lib/crawler/crawler.d.ts +35 -3
- package/lib/crawler/crawler.js +486 -338
- package/lib/crawler/dedupe/build-dedupe-cap-observation.d.ts +74 -0
- package/lib/crawler/dedupe/build-dedupe-cap-observation.js +97 -0
- package/lib/crawler/dedupe/compute-meta-signature.d.ts +7 -5
- package/lib/crawler/dedupe/compute-meta-signature.js +0 -0
- package/lib/crawler/dedupe/compute-shape-key.d.ts +20 -0
- package/lib/crawler/dedupe/compute-shape-key.js +73 -1
- package/lib/crawler/dedupe/dedupe-cap-tracker.d.ts +6 -1
- package/lib/crawler/dedupe/dedupe-cap-tracker.js +6 -1
- package/lib/crawler/dedupe/resolve-og-url-mismatch.d.ts +7 -5
- package/lib/crawler/dedupe/resolve-og-url-mismatch.js +5 -3
- package/lib/crawler/dedupe/types.d.ts +18 -0
- package/lib/crawler/fetch-destination.js +1 -0
- package/lib/crawler/link-list.d.ts +9 -1
- package/lib/crawler/link-list.js +12 -2
- package/lib/crawler/link-to-page-data.js +1 -0
- package/lib/crawler/resource-to-page-data.js +1 -0
- package/lib/crawler/types.d.ts +75 -0
- package/lib/crawler-orchestrator.d.ts +171 -8
- package/lib/crawler-orchestrator.js +1177 -234
- package/lib/crawler.d.ts +3 -0
- package/lib/crawler.js +3 -0
- package/lib/delay-or-abort.d.ts +20 -0
- package/lib/delay-or-abort.js +35 -0
- package/lib/pending-urls-remain-error.d.ts +35 -0
- package/lib/pending-urls-remain-error.js +43 -0
- package/lib/recrawl-setup-phases.d.ts +38 -0
- package/lib/recrawl-setup-phases.js +52 -0
- package/lib/resume-setup-phases.d.ts +1 -1
- package/lib/resume-setup-phases.js +5 -1
- package/lib/retry-failed-setup-phases.d.ts +1 -1
- package/lib/setup-recovery-phase-labels.d.ts +17 -8
- package/lib/setup-recovery-phase-labels.js +17 -8
- package/lib/types.d.ts +62 -12
- package/lib/utils/compute-file-sha256.d.ts +1 -1
- package/lib/utils/compute-file-sha256.js +1 -1
- package/package.json +4 -4
- package/lib/archive/db-ops/inventory/record-inventory-run.d.ts +0 -21
- package/lib/archive/migrate-inventory-runs-invalid-skipped.d.ts +0 -16
- package/lib/archive/migrate-inventory-runs-invalid-skipped.js +0 -35
|
@@ -125,6 +125,8 @@ export interface PageDenormalizedColumns {
|
|
|
125
125
|
* main_content_custom_element_count: 0,
|
|
126
126
|
* scroll_height_desktop: 3200,
|
|
127
127
|
* scroll_height_mobile: 5400,
|
|
128
|
+
* image_scan_desktop: 0,
|
|
129
|
+
* image_scan_mobile: 1,
|
|
128
130
|
* };
|
|
129
131
|
*/
|
|
130
132
|
export interface MainContentsDenormalizedColumns {
|
|
@@ -172,6 +174,28 @@ export interface MainContentsDenormalizedColumns {
|
|
|
172
174
|
scroll_height_desktop: number | null;
|
|
173
175
|
/** `document.body.scrollHeight` at the mobile-small preset, or `null`. */
|
|
174
176
|
scroll_height_mobile: number | null;
|
|
177
|
+
/**
|
|
178
|
+
* `@d-zero/beholder`'s `IMAGE_SCAN_CODE` outcome for the desktop-compact
|
|
179
|
+
* `<img>` element scan (0=ok, 1=degraded, 2=nav-unsettled, 3=frame-lost,
|
|
180
|
+
* 4=scroll-height-exceeded, 255=unknown), or `null` when not attempted
|
|
181
|
+
* (page not fully rendered, or the archive predates this column).
|
|
182
|
+
*/
|
|
183
|
+
image_scan_desktop: number | null;
|
|
184
|
+
/** Same as {@link MainContentsDenormalizedColumns.image_scan_desktop}, for the mobile-small preset. */
|
|
185
|
+
image_scan_mobile: number | null;
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Per-viewport `@d-zero/beholder` `IMAGE_SCAN_CODE` outcome, as passed into
|
|
189
|
+
* `computeMainContentsDenormalized`. Deliberately plain `number | null`
|
|
190
|
+
* rather than beholder's `ImageScanCode` type — this shape is written by
|
|
191
|
+
* `@nitpicker/crawler` independently of which `@d-zero/beholder` version is
|
|
192
|
+
* currently installed (see `compute-main-contents-denormalized.ts`).
|
|
193
|
+
*/
|
|
194
|
+
export interface ImageScanColumns {
|
|
195
|
+
/** Outcome for the desktop-compact viewport, or `null` when not attempted. */
|
|
196
|
+
desktop: number | null;
|
|
197
|
+
/** Outcome for the mobile-small viewport, or `null` when not attempted. */
|
|
198
|
+
mobile: number | null;
|
|
175
199
|
}
|
|
176
200
|
/**
|
|
177
201
|
* One row in the `page_main_content_headings` table.
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { Knex } from 'knex';
|
|
2
|
+
/**
|
|
3
|
+
* Adds the `content_items.is_metadata_only` column to archives created
|
|
4
|
+
* before this feature.
|
|
5
|
+
*
|
|
6
|
+
* `content_items` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
|
|
7
|
+
* {@link import('./create-entity-tables.js').createEntityTables}, which
|
|
8
|
+
* self-heals a *missing table* on every `initSchema` call but is a no-op
|
|
9
|
+
* against an *existing* table — adding a column to the DDL string never
|
|
10
|
+
* reaches an archive whose `content_items` predates this change. This
|
|
11
|
+
* mirrors {@link import('./migrate-content-items-dedupe-cap-event-id.js').migrateContentItemsDedupeCapEventId}'s
|
|
12
|
+
* catch-up: a `hasColumn`-guarded `ALTER TABLE` for the one column
|
|
13
|
+
* `CREATE TABLE IF NOT EXISTS` cannot retrofit.
|
|
14
|
+
*
|
|
15
|
+
* Unlike that migration, this column is `NOT NULL DEFAULT 0` (not nullable):
|
|
16
|
+
* SQLite requires a `DEFAULT` on `ALTER TABLE ... ADD COLUMN ... NOT NULL`,
|
|
17
|
+
* and the fresh-archive DDL carries the same `DEFAULT 0` so both paths stay
|
|
18
|
+
* bit-for-bit identical (see `createEntityTables`'s DDL comment). Existing
|
|
19
|
+
* rows backfill to `0` (not metadata-only) — a strict-pending row from
|
|
20
|
+
* before this migration was always resumed as a full-scrape target anyway
|
|
21
|
+
* (the bug this column fixes), so `0` preserves that prior behaviour rather
|
|
22
|
+
* than silently reclassifying already-crawled archives.
|
|
23
|
+
*
|
|
24
|
+
* No index: the only reader (`getCrawlingState`) already filters to
|
|
25
|
+
* `scraped = 0 AND is_external = 0` first, a small set. See
|
|
26
|
+
* `createEntityTables`'s DDL comment for the same "no speculative index"
|
|
27
|
+
* reasoning applied to `dedupe_cap_event_id`.
|
|
28
|
+
*
|
|
29
|
+
* Idempotent: adding the column is a no-op once it exists. Guards on
|
|
30
|
+
* `content_items`'s existence defensively, though by the time this runs
|
|
31
|
+
* (after `initSchema`, itself after `assertCompatibleVersion` rejects
|
|
32
|
+
* pre-0.13 archives) the table is always present.
|
|
33
|
+
* @param instance - The Knex query builder instance connected to the database.
|
|
34
|
+
* @param onLog - Called instead of `console.error` when this migration
|
|
35
|
+
* actually applies (issue #294: a bare `console.error` here can fire
|
|
36
|
+
* while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
|
|
37
|
+
* `Archive.open`, corrupting its cursor tracking). Falls back to
|
|
38
|
+
* `console.error` when omitted (direct/test callers).
|
|
39
|
+
* @example
|
|
40
|
+
* ```ts
|
|
41
|
+
* await migrateContentItemsIsMetadataOnly(knex);
|
|
42
|
+
* ```
|
|
43
|
+
*/
|
|
44
|
+
export declare function migrateContentItemsIsMetadataOnly(instance: Knex, onLog?: (message: string) => void): Promise<void>;
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Adds the `content_items.is_metadata_only` column to archives created
|
|
3
|
+
* before this feature.
|
|
4
|
+
*
|
|
5
|
+
* `content_items` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
|
|
6
|
+
* {@link import('./create-entity-tables.js').createEntityTables}, which
|
|
7
|
+
* self-heals a *missing table* on every `initSchema` call but is a no-op
|
|
8
|
+
* against an *existing* table — adding a column to the DDL string never
|
|
9
|
+
* reaches an archive whose `content_items` predates this change. This
|
|
10
|
+
* mirrors {@link import('./migrate-content-items-dedupe-cap-event-id.js').migrateContentItemsDedupeCapEventId}'s
|
|
11
|
+
* catch-up: a `hasColumn`-guarded `ALTER TABLE` for the one column
|
|
12
|
+
* `CREATE TABLE IF NOT EXISTS` cannot retrofit.
|
|
13
|
+
*
|
|
14
|
+
* Unlike that migration, this column is `NOT NULL DEFAULT 0` (not nullable):
|
|
15
|
+
* SQLite requires a `DEFAULT` on `ALTER TABLE ... ADD COLUMN ... NOT NULL`,
|
|
16
|
+
* and the fresh-archive DDL carries the same `DEFAULT 0` so both paths stay
|
|
17
|
+
* bit-for-bit identical (see `createEntityTables`'s DDL comment). Existing
|
|
18
|
+
* rows backfill to `0` (not metadata-only) — a strict-pending row from
|
|
19
|
+
* before this migration was always resumed as a full-scrape target anyway
|
|
20
|
+
* (the bug this column fixes), so `0` preserves that prior behaviour rather
|
|
21
|
+
* than silently reclassifying already-crawled archives.
|
|
22
|
+
*
|
|
23
|
+
* No index: the only reader (`getCrawlingState`) already filters to
|
|
24
|
+
* `scraped = 0 AND is_external = 0` first, a small set. See
|
|
25
|
+
* `createEntityTables`'s DDL comment for the same "no speculative index"
|
|
26
|
+
* reasoning applied to `dedupe_cap_event_id`.
|
|
27
|
+
*
|
|
28
|
+
* Idempotent: adding the column is a no-op once it exists. Guards on
|
|
29
|
+
* `content_items`'s existence defensively, though by the time this runs
|
|
30
|
+
* (after `initSchema`, itself after `assertCompatibleVersion` rejects
|
|
31
|
+
* pre-0.13 archives) the table is always present.
|
|
32
|
+
* @param instance - The Knex query builder instance connected to the database.
|
|
33
|
+
* @param onLog - Called instead of `console.error` when this migration
|
|
34
|
+
* actually applies (issue #294: a bare `console.error` here can fire
|
|
35
|
+
* while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
|
|
36
|
+
* `Archive.open`, corrupting its cursor tracking). Falls back to
|
|
37
|
+
* `console.error` when omitted (direct/test callers).
|
|
38
|
+
* @example
|
|
39
|
+
* ```ts
|
|
40
|
+
* await migrateContentItemsIsMetadataOnly(knex);
|
|
41
|
+
* ```
|
|
42
|
+
*/
|
|
43
|
+
export async function migrateContentItemsIsMetadataOnly(instance, onLog) {
|
|
44
|
+
const hasContentItems = await instance.schema.hasTable('content_items');
|
|
45
|
+
if (!hasContentItems) {
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
const hasColumn = await instance.schema.hasColumn('content_items', 'is_metadata_only');
|
|
49
|
+
if (!hasColumn) {
|
|
50
|
+
await instance.raw('ALTER TABLE content_items ADD COLUMN is_metadata_only INTEGER NOT NULL DEFAULT 0');
|
|
51
|
+
const message = '[migrate] content_items.is_metadata_only column added';
|
|
52
|
+
if (onLog) {
|
|
53
|
+
onLog(message);
|
|
54
|
+
}
|
|
55
|
+
else {
|
|
56
|
+
// eslint-disable-next-line no-console
|
|
57
|
+
console.error(message);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { Knex } from 'knex';
|
|
2
|
+
/**
|
|
3
|
+
* Adds the `info.createdCwd` column to archives created before it existed.
|
|
4
|
+
* `CREATE TABLE IF NOT EXISTS` (used for `info` itself) cannot retrofit a new
|
|
5
|
+
* column onto an already-existing table, so this lightweight,
|
|
6
|
+
* `hasColumn`-guarded `ALTER TABLE` runs on every `initSchema` call —
|
|
7
|
+
* idempotent, and self-healing for archives whose provisioning crashed
|
|
8
|
+
* partway through. A stub missing this column simply falls back to
|
|
9
|
+
* `Archive.resume`'s own `process.cwd()`, same as before this column existed.
|
|
10
|
+
* @param instance - The Knex query builder instance connected to the database.
|
|
11
|
+
* @param onLog - Called instead of `console.error` when this migration
|
|
12
|
+
* actually applies (issue #294: a bare `console.error` here can fire
|
|
13
|
+
* while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
|
|
14
|
+
* `Archive.open`, corrupting its cursor tracking). Falls back to
|
|
15
|
+
* `console.error` when omitted (direct/test callers).
|
|
16
|
+
*/
|
|
17
|
+
export declare function migrateInfoCreatedCwd(instance: Knex, onLog?: (message: string) => void): Promise<void>;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Adds the `info.createdCwd` column to archives created before it existed.
|
|
3
|
+
* `CREATE TABLE IF NOT EXISTS` (used for `info` itself) cannot retrofit a new
|
|
4
|
+
* column onto an already-existing table, so this lightweight,
|
|
5
|
+
* `hasColumn`-guarded `ALTER TABLE` runs on every `initSchema` call —
|
|
6
|
+
* idempotent, and self-healing for archives whose provisioning crashed
|
|
7
|
+
* partway through. A stub missing this column simply falls back to
|
|
8
|
+
* `Archive.resume`'s own `process.cwd()`, same as before this column existed.
|
|
9
|
+
* @param instance - The Knex query builder instance connected to the database.
|
|
10
|
+
* @param onLog - Called instead of `console.error` when this migration
|
|
11
|
+
* actually applies (issue #294: a bare `console.error` here can fire
|
|
12
|
+
* while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
|
|
13
|
+
* `Archive.open`, corrupting its cursor tracking). Falls back to
|
|
14
|
+
* `console.error` when omitted (direct/test callers).
|
|
15
|
+
*/
|
|
16
|
+
export async function migrateInfoCreatedCwd(instance, onLog) {
|
|
17
|
+
const hasInfo = await instance.schema.hasTable('info');
|
|
18
|
+
if (!hasInfo) {
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
const hasColumn = await instance.schema.hasColumn('info', 'createdCwd');
|
|
22
|
+
if (hasColumn) {
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
await instance.schema.table('info', (t) => {
|
|
26
|
+
t.string('createdCwd');
|
|
27
|
+
});
|
|
28
|
+
const message = '[migrate] info.createdCwd column added';
|
|
29
|
+
if (onLog) {
|
|
30
|
+
onLog(message);
|
|
31
|
+
}
|
|
32
|
+
else {
|
|
33
|
+
// eslint-disable-next-line no-console
|
|
34
|
+
console.error(message);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { Knex } from 'knex';
|
|
2
|
+
/**
|
|
3
|
+
* Renames the legacy `inventory_runs` table to `list_reconcile_runs` for
|
|
4
|
+
* archives created before the rename (issue #354: the old name implied
|
|
5
|
+
* `--inventory`-only despite `--recrawl` writing to the same table via the
|
|
6
|
+
* same {@link import('./db-ops/list-reconcile/record-list-reconcile-run.js').recordListReconcileRun}
|
|
7
|
+
* path).
|
|
8
|
+
*
|
|
9
|
+
* Must run BEFORE {@link import('./init-schema.js').initSchema} in the boot
|
|
10
|
+
* sequence, not alongside the other column-adding `migrate*` functions
|
|
11
|
+
* that run after it. `initSchema` unconditionally re-runs
|
|
12
|
+
* `createAdjunctTables`, which creates `list_reconcile_runs` fresh
|
|
13
|
+
* (`CREATE TABLE IF NOT EXISTS`) if it doesn't see the new name yet — so if
|
|
14
|
+
* this rename ran after `initSchema`, it would find `list_reconcile_runs`
|
|
15
|
+
* already present (empty) and skip, silently orphaning the old table's
|
|
16
|
+
* rows. Running first guarantees `initSchema` sees the already-renamed
|
|
17
|
+
* table and its `IF NOT EXISTS` guard is a no-op.
|
|
18
|
+
*
|
|
19
|
+
* A straight `renameTable` (not the convert-then-drop pattern used for
|
|
20
|
+
* `page_tags` → `technology_signals`/`page_technologies`) because this is a
|
|
21
|
+
* pure name change with no data-shape change — the row shape is identical
|
|
22
|
+
* before and after.
|
|
23
|
+
*
|
|
24
|
+
* Idempotent: a no-op once `inventory_runs` is gone (either because this
|
|
25
|
+
* function already ran, or the archive was created after the rename
|
|
26
|
+
* shipped and never had the old name).
|
|
27
|
+
* @param instance - The Knex query builder instance connected to the database.
|
|
28
|
+
* @param onLog - Called instead of `console.error` when this migration
|
|
29
|
+
* actually applies (issue #294: a bare `console.error` here can fire
|
|
30
|
+
* while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
|
|
31
|
+
* `Archive.open`, corrupting its cursor tracking). Falls back to
|
|
32
|
+
* `console.error` when omitted (direct/test callers).
|
|
33
|
+
*/
|
|
34
|
+
export declare function migrateInventoryRunsToListReconcileRuns(instance: Knex, onLog?: (message: string) => void): Promise<void>;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Renames the legacy `inventory_runs` table to `list_reconcile_runs` for
|
|
3
|
+
* archives created before the rename (issue #354: the old name implied
|
|
4
|
+
* `--inventory`-only despite `--recrawl` writing to the same table via the
|
|
5
|
+
* same {@link import('./db-ops/list-reconcile/record-list-reconcile-run.js').recordListReconcileRun}
|
|
6
|
+
* path).
|
|
7
|
+
*
|
|
8
|
+
* Must run BEFORE {@link import('./init-schema.js').initSchema} in the boot
|
|
9
|
+
* sequence, not alongside the other column-adding `migrate*` functions
|
|
10
|
+
* that run after it. `initSchema` unconditionally re-runs
|
|
11
|
+
* `createAdjunctTables`, which creates `list_reconcile_runs` fresh
|
|
12
|
+
* (`CREATE TABLE IF NOT EXISTS`) if it doesn't see the new name yet — so if
|
|
13
|
+
* this rename ran after `initSchema`, it would find `list_reconcile_runs`
|
|
14
|
+
* already present (empty) and skip, silently orphaning the old table's
|
|
15
|
+
* rows. Running first guarantees `initSchema` sees the already-renamed
|
|
16
|
+
* table and its `IF NOT EXISTS` guard is a no-op.
|
|
17
|
+
*
|
|
18
|
+
* A straight `renameTable` (not the convert-then-drop pattern used for
|
|
19
|
+
* `page_tags` → `technology_signals`/`page_technologies`) because this is a
|
|
20
|
+
* pure name change with no data-shape change — the row shape is identical
|
|
21
|
+
* before and after.
|
|
22
|
+
*
|
|
23
|
+
* Idempotent: a no-op once `inventory_runs` is gone (either because this
|
|
24
|
+
* function already ran, or the archive was created after the rename
|
|
25
|
+
* shipped and never had the old name).
|
|
26
|
+
* @param instance - The Knex query builder instance connected to the database.
|
|
27
|
+
* @param onLog - Called instead of `console.error` when this migration
|
|
28
|
+
* actually applies (issue #294: a bare `console.error` here can fire
|
|
29
|
+
* while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
|
|
30
|
+
* `Archive.open`, corrupting its cursor tracking). Falls back to
|
|
31
|
+
* `console.error` when omitted (direct/test callers).
|
|
32
|
+
*/
|
|
33
|
+
export async function migrateInventoryRunsToListReconcileRuns(instance, onLog) {
|
|
34
|
+
const hasOldTable = await instance.schema.hasTable('inventory_runs');
|
|
35
|
+
if (!hasOldTable) {
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
const hasNewTable = await instance.schema.hasTable('list_reconcile_runs');
|
|
39
|
+
if (hasNewTable) {
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
await instance.schema.renameTable('inventory_runs', 'list_reconcile_runs');
|
|
43
|
+
const message = '[migrate] inventory_runs renamed to list_reconcile_runs';
|
|
44
|
+
if (onLog) {
|
|
45
|
+
onLog(message);
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
// eslint-disable-next-line no-console
|
|
49
|
+
console.error(message);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import type { Knex } from 'knex';
|
|
2
2
|
/**
|
|
3
|
-
* Adds the `
|
|
4
|
-
* before it existed. `CREATE TABLE IF NOT EXISTS` (used for
|
|
5
|
-
* itself) cannot retrofit a new column onto an
|
|
6
|
-
* this lightweight, `hasColumn`-guarded
|
|
7
|
-
* `initSchema` call — idempotent, and
|
|
8
|
-
* provisioning crashed partway through.
|
|
3
|
+
* Adds the `list_reconcile_runs.exclude_skipped` column to archives created
|
|
4
|
+
* before it existed. `CREATE TABLE IF NOT EXISTS` (used for
|
|
5
|
+
* `list_reconcile_runs` itself) cannot retrofit a new column onto an
|
|
6
|
+
* already-existing table, so this lightweight, `hasColumn`-guarded
|
|
7
|
+
* `ALTER TABLE` runs on every `initSchema` call — idempotent, and
|
|
8
|
+
* self-healing for archives whose provisioning crashed partway through.
|
|
9
9
|
*
|
|
10
10
|
* Pre-migration rows stay `NULL`: those runs predate ingestion-side
|
|
11
11
|
* exclusion (issue #260), so their excluded URLs were imported as real
|
|
@@ -13,7 +13,7 @@ import type { Knex } from 'knex';
|
|
|
13
13
|
* measured", not `0`.
|
|
14
14
|
*
|
|
15
15
|
* The column is pure audit output: written once per run and read back
|
|
16
|
-
* only by `
|
|
16
|
+
* only by `listReconcileRuns` display surfaces, never consumed by any
|
|
17
17
|
* runtime decision — matching `scope_skipped` / `invalid_skipped`.
|
|
18
18
|
* @param instance - The Knex query builder instance connected to the database.
|
|
19
19
|
* @param onLog - Called instead of `console.error` when this migration
|
|
@@ -22,4 +22,4 @@ import type { Knex } from 'knex';
|
|
|
22
22
|
* `Archive.open`, corrupting its cursor tracking). Falls back to
|
|
23
23
|
* `console.error` when omitted (direct/test callers).
|
|
24
24
|
*/
|
|
25
|
-
export declare function
|
|
25
|
+
export declare function migrateListReconcileRunsExcludeSkipped(instance: Knex, onLog?: (message: string) => void): Promise<void>;
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Adds the `
|
|
3
|
-
* before it existed. `CREATE TABLE IF NOT EXISTS` (used for
|
|
4
|
-
* itself) cannot retrofit a new column onto an
|
|
5
|
-
* this lightweight, `hasColumn`-guarded
|
|
6
|
-
* `initSchema` call — idempotent, and
|
|
7
|
-
* provisioning crashed partway through.
|
|
2
|
+
* Adds the `list_reconcile_runs.exclude_skipped` column to archives created
|
|
3
|
+
* before it existed. `CREATE TABLE IF NOT EXISTS` (used for
|
|
4
|
+
* `list_reconcile_runs` itself) cannot retrofit a new column onto an
|
|
5
|
+
* already-existing table, so this lightweight, `hasColumn`-guarded
|
|
6
|
+
* `ALTER TABLE` runs on every `initSchema` call — idempotent, and
|
|
7
|
+
* self-healing for archives whose provisioning crashed partway through.
|
|
8
8
|
*
|
|
9
9
|
* Pre-migration rows stay `NULL`: those runs predate ingestion-side
|
|
10
10
|
* exclusion (issue #260), so their excluded URLs were imported as real
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* measured", not `0`.
|
|
13
13
|
*
|
|
14
14
|
* The column is pure audit output: written once per run and read back
|
|
15
|
-
* only by `
|
|
15
|
+
* only by `listReconcileRuns` display surfaces, never consumed by any
|
|
16
16
|
* runtime decision — matching `scope_skipped` / `invalid_skipped`.
|
|
17
17
|
* @param instance - The Knex query builder instance connected to the database.
|
|
18
18
|
* @param onLog - Called instead of `console.error` when this migration
|
|
@@ -21,19 +21,19 @@
|
|
|
21
21
|
* `Archive.open`, corrupting its cursor tracking). Falls back to
|
|
22
22
|
* `console.error` when omitted (direct/test callers).
|
|
23
23
|
*/
|
|
24
|
-
export async function
|
|
25
|
-
const hasTable = await instance.schema.hasTable('
|
|
24
|
+
export async function migrateListReconcileRunsExcludeSkipped(instance, onLog) {
|
|
25
|
+
const hasTable = await instance.schema.hasTable('list_reconcile_runs');
|
|
26
26
|
if (!hasTable) {
|
|
27
27
|
return;
|
|
28
28
|
}
|
|
29
|
-
const hasColumn = await instance.schema.hasColumn('
|
|
29
|
+
const hasColumn = await instance.schema.hasColumn('list_reconcile_runs', 'exclude_skipped');
|
|
30
30
|
if (hasColumn) {
|
|
31
31
|
return;
|
|
32
32
|
}
|
|
33
|
-
await instance.schema.table('
|
|
33
|
+
await instance.schema.table('list_reconcile_runs', (t) => {
|
|
34
34
|
t.integer('exclude_skipped');
|
|
35
35
|
});
|
|
36
|
-
const message = '[migrate]
|
|
36
|
+
const message = '[migrate] list_reconcile_runs.exclude_skipped column added';
|
|
37
37
|
if (onLog) {
|
|
38
38
|
onLog(message);
|
|
39
39
|
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { Knex } from 'knex';
|
|
2
|
+
/**
|
|
3
|
+
* Adds the `list_reconcile_runs.invalid_skipped` column to archives created
|
|
4
|
+
* before it existed. `CREATE TABLE IF NOT EXISTS` (used for
|
|
5
|
+
* `list_reconcile_runs` itself) cannot retrofit a new column onto an
|
|
6
|
+
* already-existing table, so this lightweight, `hasColumn`-guarded
|
|
7
|
+
* `ALTER TABLE` runs on every `initSchema` call — idempotent, and
|
|
8
|
+
* self-healing for archives whose provisioning crashed partway through.
|
|
9
|
+
* @param instance - The Knex query builder instance connected to the database.
|
|
10
|
+
* @param onLog - Called instead of `console.error` when this migration
|
|
11
|
+
* actually applies (issue #294: a bare `console.error` here can fire
|
|
12
|
+
* while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
|
|
13
|
+
* `Archive.open`, corrupting its cursor tracking). Falls back to
|
|
14
|
+
* `console.error` when omitted (direct/test callers).
|
|
15
|
+
*/
|
|
16
|
+
export declare function migrateListReconcileRunsInvalidSkipped(instance: Knex, onLog?: (message: string) => void): Promise<void>;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Adds the `list_reconcile_runs.invalid_skipped` column to archives created
|
|
3
|
+
* before it existed. `CREATE TABLE IF NOT EXISTS` (used for
|
|
4
|
+
* `list_reconcile_runs` itself) cannot retrofit a new column onto an
|
|
5
|
+
* already-existing table, so this lightweight, `hasColumn`-guarded
|
|
6
|
+
* `ALTER TABLE` runs on every `initSchema` call — idempotent, and
|
|
7
|
+
* self-healing for archives whose provisioning crashed partway through.
|
|
8
|
+
* @param instance - The Knex query builder instance connected to the database.
|
|
9
|
+
* @param onLog - Called instead of `console.error` when this migration
|
|
10
|
+
* actually applies (issue #294: a bare `console.error` here can fire
|
|
11
|
+
* while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
|
|
12
|
+
* `Archive.open`, corrupting its cursor tracking). Falls back to
|
|
13
|
+
* `console.error` when omitted (direct/test callers).
|
|
14
|
+
*/
|
|
15
|
+
export async function migrateListReconcileRunsInvalidSkipped(instance, onLog) {
|
|
16
|
+
const hasTable = await instance.schema.hasTable('list_reconcile_runs');
|
|
17
|
+
if (!hasTable) {
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
const hasColumn = await instance.schema.hasColumn('list_reconcile_runs', 'invalid_skipped');
|
|
21
|
+
if (hasColumn) {
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
await instance.schema.table('list_reconcile_runs', (t) => {
|
|
25
|
+
t.integer('invalid_skipped');
|
|
26
|
+
});
|
|
27
|
+
const message = '[migrate] list_reconcile_runs.invalid_skipped column added';
|
|
28
|
+
if (onLog) {
|
|
29
|
+
onLog(message);
|
|
30
|
+
}
|
|
31
|
+
else {
|
|
32
|
+
// eslint-disable-next-line no-console
|
|
33
|
+
console.error(message);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { Knex } from 'knex';
|
|
2
|
+
/**
|
|
3
|
+
* Adds the `page_meta.image_scan_desktop` / `page_meta.image_scan_mobile`
|
|
4
|
+
* columns to archives created before this feature.
|
|
5
|
+
*
|
|
6
|
+
* `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
|
|
7
|
+
* {@link import('./create-entity-tables.js').createEntityTables}, which
|
|
8
|
+
* self-heals a *missing table* on every `initSchema` call but is a no-op
|
|
9
|
+
* against an *existing* table — adding a column to the DDL string never
|
|
10
|
+
* reaches an archive whose `page_meta` predates this change. Same catch-up
|
|
11
|
+
* shape as {@link import('./migrate-page-meta-console-error-count.js').migratePageMetaConsoleErrorCount}.
|
|
12
|
+
*
|
|
13
|
+
* Both columns are added in the same migration since they are always
|
|
14
|
+
* written together (one `@d-zero/beholder` `imageScan` result per page).
|
|
15
|
+
* There is no backfill: a page that has never been re-scraped since this
|
|
16
|
+
* feature shipped has no recorded outcome, so `NULL` (not attempted) is
|
|
17
|
+
* already the correct value.
|
|
18
|
+
*
|
|
19
|
+
* Idempotent: adding a column that already exists is a no-op.
|
|
20
|
+
* @param instance - The Knex query builder instance connected to the database.
|
|
21
|
+
* @param onLog - Called instead of `console.error` when this migration
|
|
22
|
+
* actually applies (issue #294: a bare `console.error` here can fire
|
|
23
|
+
* while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
|
|
24
|
+
* `Archive.open`, corrupting its cursor tracking). Falls back to
|
|
25
|
+
* `console.error` when omitted (direct/test callers).
|
|
26
|
+
*/
|
|
27
|
+
export declare function migratePageMetaImageScan(instance: Knex, onLog?: (message: string) => void): Promise<void>;
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Adds the `page_meta.image_scan_desktop` / `page_meta.image_scan_mobile`
|
|
3
|
+
* columns to archives created before this feature.
|
|
4
|
+
*
|
|
5
|
+
* `page_meta` is provisioned via a bare `CREATE TABLE IF NOT EXISTS` in
|
|
6
|
+
* {@link import('./create-entity-tables.js').createEntityTables}, which
|
|
7
|
+
* self-heals a *missing table* on every `initSchema` call but is a no-op
|
|
8
|
+
* against an *existing* table — adding a column to the DDL string never
|
|
9
|
+
* reaches an archive whose `page_meta` predates this change. Same catch-up
|
|
10
|
+
* shape as {@link import('./migrate-page-meta-console-error-count.js').migratePageMetaConsoleErrorCount}.
|
|
11
|
+
*
|
|
12
|
+
* Both columns are added in the same migration since they are always
|
|
13
|
+
* written together (one `@d-zero/beholder` `imageScan` result per page).
|
|
14
|
+
* There is no backfill: a page that has never been re-scraped since this
|
|
15
|
+
* feature shipped has no recorded outcome, so `NULL` (not attempted) is
|
|
16
|
+
* already the correct value.
|
|
17
|
+
*
|
|
18
|
+
* Idempotent: adding a column that already exists is a no-op.
|
|
19
|
+
* @param instance - The Knex query builder instance connected to the database.
|
|
20
|
+
* @param onLog - Called instead of `console.error` when this migration
|
|
21
|
+
* actually applies (issue #294: a bare `console.error` here can fire
|
|
22
|
+
* while a `@d-zero/dealer` `Lanes`/`TaskList` display is mid-redraw during
|
|
23
|
+
* `Archive.open`, corrupting its cursor tracking). Falls back to
|
|
24
|
+
* `console.error` when omitted (direct/test callers).
|
|
25
|
+
*/
|
|
26
|
+
export async function migratePageMetaImageScan(instance, onLog) {
|
|
27
|
+
const hasPageMeta = await instance.schema.hasTable('page_meta');
|
|
28
|
+
if (!hasPageMeta) {
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
const hasDesktopColumn = await instance.schema.hasColumn('page_meta', 'image_scan_desktop');
|
|
32
|
+
const hasMobileColumn = await instance.schema.hasColumn('page_meta', 'image_scan_mobile');
|
|
33
|
+
if (hasDesktopColumn && hasMobileColumn) {
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
await instance.schema.table('page_meta', (t) => {
|
|
37
|
+
if (!hasDesktopColumn) {
|
|
38
|
+
t.integer('image_scan_desktop');
|
|
39
|
+
}
|
|
40
|
+
if (!hasMobileColumn) {
|
|
41
|
+
t.integer('image_scan_mobile');
|
|
42
|
+
}
|
|
43
|
+
});
|
|
44
|
+
const message = '[migrate] page_meta.image_scan_desktop/image_scan_mobile columns added';
|
|
45
|
+
if (onLog) {
|
|
46
|
+
onLog(message);
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
// eslint-disable-next-line no-console
|
|
50
|
+
console.error(message);
|
|
51
|
+
}
|
|
52
|
+
}
|
package/lib/archive/page.d.ts
CHANGED
|
@@ -239,6 +239,18 @@ export default class Page {
|
|
|
239
239
|
* aggregate written at scrape time), or null.
|
|
240
240
|
*/
|
|
241
241
|
get scrollHeightMobile(): number | null;
|
|
242
|
+
/**
|
|
243
|
+
* `@d-zero/beholder`'s `IMAGE_SCAN_CODE` outcome for the desktop-compact
|
|
244
|
+
* `<img>` element scan (denormalised aggregate written at scrape time), or
|
|
245
|
+
* null when not attempted.
|
|
246
|
+
*/
|
|
247
|
+
get imageScanDesktop(): number | null;
|
|
248
|
+
/**
|
|
249
|
+
* `@d-zero/beholder`'s `IMAGE_SCAN_CODE` outcome for the mobile-small
|
|
250
|
+
* `<img>` element scan (denormalised aggregate written at scrape time), or
|
|
251
|
+
* null when not attempted.
|
|
252
|
+
*/
|
|
253
|
+
get imageScanMobile(): number | null;
|
|
242
254
|
/**
|
|
243
255
|
* The reason this page was skipped during crawling, or null if it was not skipped.
|
|
244
256
|
*/
|
package/lib/archive/page.js
CHANGED
|
@@ -417,6 +417,22 @@ export default class Page {
|
|
|
417
417
|
get scrollHeightMobile() {
|
|
418
418
|
return this.#raw.scroll_height_mobile;
|
|
419
419
|
}
|
|
420
|
+
/**
|
|
421
|
+
* `@d-zero/beholder`'s `IMAGE_SCAN_CODE` outcome for the desktop-compact
|
|
422
|
+
* `<img>` element scan (denormalised aggregate written at scrape time), or
|
|
423
|
+
* null when not attempted.
|
|
424
|
+
*/
|
|
425
|
+
get imageScanDesktop() {
|
|
426
|
+
return this.#raw.image_scan_desktop;
|
|
427
|
+
}
|
|
428
|
+
/**
|
|
429
|
+
* `@d-zero/beholder`'s `IMAGE_SCAN_CODE` outcome for the mobile-small
|
|
430
|
+
* `<img>` element scan (denormalised aggregate written at scrape time), or
|
|
431
|
+
* null when not attempted.
|
|
432
|
+
*/
|
|
433
|
+
get imageScanMobile() {
|
|
434
|
+
return this.#raw.image_scan_mobile;
|
|
435
|
+
}
|
|
420
436
|
/**
|
|
421
437
|
* The reason this page was skipped during crawling, or null if it was not skipped.
|
|
422
438
|
*/
|