@nitpicker/crawler 0.12.0 → 0.13.0

package/lib/utils/compute-file-sha256.js

@@ -5,10 +5,11 @@ import { crawlerLog } from '../debug.js';
  * Compute the SHA-256 digest of a file's bytes, streaming so memory stays
  * O(1) regardless of file size.
  *
- * Used by {@link CrawlerOrchestrator.inventory} to fingerprint the source
- * URL list `.txt` and store the digest on the `inventory_runs` row.
- * Phase 3 (`--refresh`) will key dedupe on this column; Phase 1 just
- * records it for operator audit.
+ * Used by `CrawlerOrchestrator.inventory` to fingerprint the source
+ * URL list `.txt` and store the digest on the `inventory_runs` row. The
+ * digest is the content-identity key for "was this exact list applied
+ * before" — recorded for operator audit; nothing dedupes on it at write
+ * time.
  *
  * Returns `null` instead of throwing when the file cannot be read (e.g.
  * vanished mid-run, permissions issue) so the inventory run's audit row

package/lib/utils/error/emit-error-with-retry.d.ts

@@ -21,7 +21,7 @@ interface Emitter {
  * derived automatically by the `@retry` decorator via
  * `ClassMethodDecoratorContext.name`).
  *
- * Inlined try/catch rather than composing {@link emitError} with a
+ * Inlined try/catch rather than composing `emitError` with a
  * `() => retryCall(...)` closure to keep the per-invocation closure count
  * at 1 — this method is called on the DB write hot path (`updatePage`,
  * `insertResource`), where wrapping decorators previously ran with zero

package/lib/utils/error/emit-error-with-retry.js

@@ -15,7 +15,7 @@ const errorLog = log.extend('ErrorEmitter');
  * derived automatically by the `@retry` decorator via
  * `ClassMethodDecoratorContext.name`).
  *
- * Inlined try/catch rather than composing {@link emitError} with a
+ * Inlined try/catch rather than composing `emitError` with a
  * `() => retryCall(...)` closure to keep the per-invocation closure count
  * at 1 — this method is called on the DB write hot path (`updatePage`,
  * `insertResource`), where wrapping decorators previously ran with zero

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nitpicker/crawler",
-	"version": "0.12.0",
+	"version": "0.13.0",
 	"description": "Web crawler engine with headless browser rendering and archive storage",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
@@ -27,19 +27,19 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/beholder": "3.1.3",
-		"@d-zero/dealer": "1.9.3",
-		"@d-zero/fs": "0.2.5",
-		"@d-zero/shared": "0.22.1",
+		"@d-zero/beholder": "3.1.5",
+		"@d-zero/dealer": "1.9.4",
+		"@d-zero/fs": "0.2.6",
+		"@d-zero/shared": "0.22.2",
 		"ansi-colors": "4.1.3",
 		"debug": "4.4.3",
 		"follow-redirects": "1.16.0",
-		"fs-extra": "11.3.5",
-		"knex": "3.2.10",
+		"fs-extra": "11.3.6",
+		"knex": "3.3.0",
 		"libsql": "0.5.29",
-		"puppeteer": "24.37.5",
+		"puppeteer": "25.3.0",
 		"robots-parser": "3.0.1",
-		"tar": "7.5.16"
+		"tar": "7.5.20"
 	},
 	"devDependencies": {
 		"@types/debug": "4.1.13",
@@ -48,5 +48,5 @@
 		"@types/tar": "7.0.87",
 		"@types/unzipper": "0.10.11"
 	},
-	"gitHead": "58b68416e0b6e3dcdd92772bee754f186a35505f"
+	"gitHead": "2cd462ea6b183f4462b7caba27b4db04aaba7c70"
 }

package/lib/archive/migrate-crawl-errors.d.ts

@@ -1,20 +0,0 @@
-import type { Knex } from 'knex';
-/**
- * Adds the `crawl_errors` table to archives created before structured capture
- * of the crawler-level `error` channel landed.
- *
- * `crawl_errors` records errors that previously only reached `error.log` (DNS
- * failures, connection resets, TLS problems, process-level errors), in a
- * queryable form so the `error-kinds` analysis can classify them without
- * parsing the text log. Older `.nitpicker` files predate this table; this
- * migration creates it idempotently. Existing archives stay empty until they
- * are crawled again (resume / append / retry), at which point new errors are
- * written here — analysis of pre-existing data falls back to parsing
- * `error.log`.
- *
- * Idempotent: when the table already exists, the function exits without
- * touching the schema or writing to stderr. It also skips empty archives (no
- * `pages` table), which the regular initSchema path will fully provision.
- * @param instance - The Knex query builder instance connected to the database.
- */
-export declare function migrateCrawlErrors(instance: Knex): Promise<void>;

package/lib/archive/migrate-crawl-errors.js

@@ -1,38 +0,0 @@
-/**
- * Adds the `crawl_errors` table to archives created before structured capture
- * of the crawler-level `error` channel landed.
- *
- * `crawl_errors` records errors that previously only reached `error.log` (DNS
- * failures, connection resets, TLS problems, process-level errors), in a
- * queryable form so the `error-kinds` analysis can classify them without
- * parsing the text log. Older `.nitpicker` files predate this table; this
- * migration creates it idempotently. Existing archives stay empty until they
- * are crawled again (resume / append / retry), at which point new errors are
- * written here — analysis of pre-existing data falls back to parsing
- * `error.log`.
- *
- * Idempotent: when the table already exists, the function exits without
- * touching the schema or writing to stderr. It also skips empty archives (no
- * `pages` table), which the regular initSchema path will fully provision.
- * @param instance - The Knex query builder instance connected to the database.
- */
-export async function migrateCrawlErrors(instance) {
-    const hasTable = await instance.schema.hasTable('crawl_errors');
-    if (hasTable) {
-        return;
-    }
-    const hasPages = await instance.schema.hasTable('pages');
-    if (!hasPages) {
-        // Empty archive; the regular initSchema path will create the table.
-        return;
-    }
-    await instance.schema.createTable('crawl_errors', (t) => {
-        t.increments('id');
-        t.string('url', 8190).nullable();
-        t.boolean('isExternal');
-        t.text('message').notNullable();
-        t.integer('createdAt').notNullable();
-    });
-    // eslint-disable-next-line no-console
-    console.error('[migrate] crawl_errors table created');
-}

package/lib/archive/migrate-html-blob-tables.d.ts

@@ -1,24 +0,0 @@
-import type { Knex } from 'knex';
-/**
- * Adds the BLOB-backed HTML snapshot tables (`page_html_blobs` +
- * `page_html_ref`) to archives created before #75.
- *
- * A pre-#75 `.nitpicker` already has the `info` table, so `initSchema`'s
- * early-return path skips the freshly-introduced `CREATE TABLE`s. This
- * migration brings the schema forward idempotently so the read API stops
- * raising raw "no such table" errors against legacy archives. It does NOT
- * touch the `pages.html` column or backfill the new tables from the
- * archive's `snapshot-html.zip`: that data migration belongs to the
- * `scripts/migrate-to-0.10.mjs` one-shot, which can run whenever the
- * user is ready to commit the (multi-hour) CPU cost.
- *
- * Outcome on a legacy archive that has NOT yet been data-migrated:
- *   - new tables exist but are empty,
- *   - `getHtmlOfPageById` returns `null` for every page (no row in `page_html_ref`),
- *   - viewer / MCP / analyze plugins see "snapshot unavailable" instead of crashing.
- *
- * The migration prints a one-line notice when it runs so the user knows
- * the archive was upgraded in place.
- * @param instance - The Knex query builder instance connected to the database.
- */
-export declare function migrateHtmlBlobTables(instance: Knex): Promise<void>;

package/lib/archive/migrate-html-blob-tables.js

@@ -1,53 +0,0 @@
-/**
- * Adds the BLOB-backed HTML snapshot tables (`page_html_blobs` +
- * `page_html_ref`) to archives created before #75.
- *
- * A pre-#75 `.nitpicker` already has the `info` table, so `initSchema`'s
- * early-return path skips the freshly-introduced `CREATE TABLE`s. This
- * migration brings the schema forward idempotently so the read API stops
- * raising raw "no such table" errors against legacy archives. It does NOT
- * touch the `pages.html` column or backfill the new tables from the
- * archive's `snapshot-html.zip`: that data migration belongs to the
- * `scripts/migrate-to-0.10.mjs` one-shot, which can run whenever the
- * user is ready to commit the (multi-hour) CPU cost.
- *
- * Outcome on a legacy archive that has NOT yet been data-migrated:
- *   - new tables exist but are empty,
- *   - `getHtmlOfPageById` returns `null` for every page (no row in `page_html_ref`),
- *   - viewer / MCP / analyze plugins see "snapshot unavailable" instead of crashing.
- *
- * The migration prints a one-line notice when it runs so the user knows
- * the archive was upgraded in place.
- * @param instance - The Knex query builder instance connected to the database.
- */
-export async function migrateHtmlBlobTables(instance) {
-    const hasBlobs = await instance.schema.hasTable('page_html_blobs');
-    if (hasBlobs) {
-        return;
-    }
-    const hasPages = await instance.schema.hasTable('pages');
-    if (!hasPages) {
-        // Empty archive; initSchema will create both tables on the
-        // fresh-DB path. Nothing to migrate.
-        return;
-    }
-    await instance.raw(`
-		CREATE TABLE page_html_blobs (
-			hash         BLOB PRIMARY KEY,
-			body         BLOB NOT NULL,
-			codec        TEXT NOT NULL CHECK(codec IN ('zstd', 'none')),
-			size_raw     INTEGER NOT NULL,
-			size_stored  INTEGER NOT NULL
-		) WITHOUT ROWID
-	`);
-    await instance.raw(`
-		CREATE TABLE page_html_ref (
-			page_id  INTEGER PRIMARY KEY REFERENCES pages(id) ON DELETE CASCADE,
-			hash     BLOB NOT NULL REFERENCES page_html_blobs(hash)
-		) WITHOUT ROWID
-	`);
-    await instance.raw('CREATE INDEX idx_page_html_ref_hash ON page_html_ref(hash)');
-    // eslint-disable-next-line no-console
-    console.error('[migrate] page_html_blobs / page_html_ref tables created. ' +
-        'HTML snapshots are empty until `node scripts/migrate-to-0.10.mjs` is run on the original archive.');
-}

package/lib/archive/migrate-inventory-runs.d.ts

@@ -1,29 +0,0 @@
-import type { Knex } from 'knex';
-/**
- * Adds the `inventory_runs` audit-log table to archives created before
- * Phase 1 of inventory run tracking shipped.
- *
- * `inventory_runs` records one row per successful `--inventory <list>`
- * invocation, capturing which deploy list was applied when and what
- * scale it operated at. The motivation is operational: client /
- * director conversations repeatedly ask "did you apply last month's
- * list" / "we didn't double-apply, right" — the archive itself had no
- * trace of inventory passes (`.bak` is unlinked on success), so this
- * table is the durable provenance record.
- *
- * Schema details (column semantics, NULL policy, index) live in
- * {@link initSchema} — this migration only re-creates the table shape
- * on legacy archives so the rest of the codebase can treat the table
- * as always-present once a writer connection has opened the file.
- *
- * Idempotent: when the table already exists, the function exits
- * silently — the `[migrate] inventory_runs table created` stderr line
- * fires **only** on the first run against a legacy archive, matching
- * the established pattern of `migrate-page-errors.ts` /
- * `migrate-crawl-errors.ts`. Operators can rely on the log line as a
- * stable "first time this archive saw Phase 1 schema" event marker.
- * Empty archives (no `pages` table) are skipped entirely; the regular
- * `initSchema` path provisions them at first crawl.
- * @param instance - The Knex query builder instance connected to the database.
- */
-export declare function migrateInventoryRuns(instance: Knex): Promise<void>;

package/lib/archive/migrate-inventory-runs.js

@@ -1,52 +0,0 @@
-/**
- * Adds the `inventory_runs` audit-log table to archives created before
- * Phase 1 of inventory run tracking shipped.
- *
- * `inventory_runs` records one row per successful `--inventory <list>`
- * invocation, capturing which deploy list was applied when and what
- * scale it operated at. The motivation is operational: client /
- * director conversations repeatedly ask "did you apply last month's
- * list" / "we didn't double-apply, right" — the archive itself had no
- * trace of inventory passes (`.bak` is unlinked on success), so this
- * table is the durable provenance record.
- *
- * Schema details (column semantics, NULL policy, index) live in
- * {@link initSchema} — this migration only re-creates the table shape
- * on legacy archives so the rest of the codebase can treat the table
- * as always-present once a writer connection has opened the file.
- *
- * Idempotent: when the table already exists, the function exits
- * silently — the `[migrate] inventory_runs table created` stderr line
- * fires **only** on the first run against a legacy archive, matching
- * the established pattern of `migrate-page-errors.ts` /
- * `migrate-crawl-errors.ts`. Operators can rely on the log line as a
- * stable "first time this archive saw Phase 1 schema" event marker.
- * Empty archives (no `pages` table) are skipped entirely; the regular
- * `initSchema` path provisions them at first crawl.
- * @param instance - The Knex query builder instance connected to the database.
- */
-export async function migrateInventoryRuns(instance) {
-    const hasTable = await instance.schema.hasTable('inventory_runs');
-    if (hasTable) {
-        return;
-    }
-    const hasPages = await instance.schema.hasTable('pages');
-    if (!hasPages) {
-        // Empty archive; the regular initSchema path will create the table.
-        return;
-    }
-    await instance.schema.createTable('inventory_runs', (t) => {
-        t.increments('id');
-        t.string('ran_at').notNullable();
-        t.string('list_label').nullable();
-        t.string('source_file_sha256', 64).nullable();
-        t.integer('total_lines').nullable();
-        t.integer('new_pages').nullable();
-        t.integer('new_resources').nullable();
-        t.integer('scope_skipped').nullable();
-        t.text('notes').nullable();
-        t.index('ran_at');
-    });
-    // eslint-disable-next-line no-console
-    console.error('[migrate] inventory_runs table created');
-}

package/lib/archive/migrate-page-errors.d.ts

@@ -1,16 +0,0 @@
-import type { Knex } from 'knex';
-/**
- * Adds the `page_errors` table to archives created before partial-failure
- * recording landed.
- *
- * `page_errors` captures secondary scrape failures (e.g. a viewport switch
- * that detaches the frame and surfaces `retryExhausted`) so they show up in
- * the archive instead of being lost to stdout logs. Older `.nitpicker` files
- * predate this table; this migration creates it idempotently.
- *
- * Idempotent: when the table already exists, the function exits without
- * touching the schema or writing to stderr. When it has to run, a single
- * notice is written so the user knows the file was upgraded.
- * @param instance - The Knex query builder instance connected to the database.
- */
-export declare function migratePageErrors(instance: Knex): Promise<void>;

package/lib/archive/migrate-page-errors.js

@@ -1,35 +0,0 @@
-/**
- * Adds the `page_errors` table to archives created before partial-failure
- * recording landed.
- *
- * `page_errors` captures secondary scrape failures (e.g. a viewport switch
- * that detaches the frame and surfaces `retryExhausted`) so they show up in
- * the archive instead of being lost to stdout logs. Older `.nitpicker` files
- * predate this table; this migration creates it idempotently.
- *
- * Idempotent: when the table already exists, the function exits without
- * touching the schema or writing to stderr. When it has to run, a single
- * notice is written so the user knows the file was upgraded.
- * @param instance - The Knex query builder instance connected to the database.
- */
-export async function migratePageErrors(instance) {
-    const hasTable = await instance.schema.hasTable('page_errors');
-    if (hasTable) {
-        return;
-    }
-    const hasPages = await instance.schema.hasTable('pages');
-    if (!hasPages) {
-        // Empty archive; the regular initSchema path will create both tables.
-        return;
-    }
-    await instance.schema.createTable('page_errors', (t) => {
-        t.increments('id');
-        t.integer('pageId').notNullable().unsigned().references('pages.id');
-        t.string('phase').notNullable();
-        t.text('message').notNullable();
-        t.integer('createdAt').notNullable();
-        t.index('pageId');
-    });
-    // eslint-disable-next-line no-console
-    console.error('[migrate] page_errors table created');
-}

package/lib/archive/migrate-pages-resources-source.d.ts

@@ -1,16 +0,0 @@
-import type { Knex } from 'knex';
-/**
- * Add `pages.source` and `resources.source` columns (provenance taxonomy:
- * `crawled` / `inventory-seed` / `inventory-discovered`) and their indexes
- * to archives that pre-date the `crawl --inventory` feature.
- *
- * Idempotent: a no-op when both columns already exist. SQLite's
- * `ALTER TABLE ADD COLUMN` with a NOT NULL DEFAULT applies the default to
- * every existing row at column-add time, so no explicit `UPDATE` is needed
- * to backfill — pre-existing rows become `'crawled'` automatically.
- *
- * Runs only on writer-side {@link Database.connect}; read-only viewer
- * attaches skip this so the user's tmpDir is never rewritten.
- * @param instance - The Knex query builder instance connected to the database.
- */
-export declare function migratePagesResourcesSource(instance: Knex): Promise<void>;

package/lib/archive/migrate-pages-resources-source.js

@@ -1,46 +0,0 @@
-/**
- * Add `pages.source` and `resources.source` columns (provenance taxonomy:
- * `crawled` / `inventory-seed` / `inventory-discovered`) and their indexes
- * to archives that pre-date the `crawl --inventory` feature.
- *
- * Idempotent: a no-op when both columns already exist. SQLite's
- * `ALTER TABLE ADD COLUMN` with a NOT NULL DEFAULT applies the default to
- * every existing row at column-add time, so no explicit `UPDATE` is needed
- * to backfill — pre-existing rows become `'crawled'` automatically.
- *
- * Runs only on writer-side {@link Database.connect}; read-only viewer
- * attaches skip this so the user's tmpDir is never rewritten.
- * @param instance - The Knex query builder instance connected to the database.
- */
-export async function migratePagesResourcesSource(instance) {
-    const hasPages = await instance.schema.hasTable('pages');
-    const hasResources = await instance.schema.hasTable('resources');
-    if (!hasPages && !hasResources) {
-        return;
-    }
-    const hasPagesSource = hasPages && (await instance.schema.hasColumn('pages', 'source'));
-    const hasResourcesSource = hasResources && (await instance.schema.hasColumn('resources', 'source'));
-    if (hasPagesSource && hasResourcesSource) {
-        return;
-    }
-    const changes = [];
-    if (hasPages && !hasPagesSource) {
-        await instance.schema.table('pages', (t) => {
-            t.string('source').notNullable().defaultTo('crawled');
-            t.index('source');
-        });
-        changes.push('pages.source added');
-    }
-    if (hasResources && !hasResourcesSource) {
-        await instance.schema.table('resources', (t) => {
-            t.string('source').notNullable().defaultTo('crawled');
-            t.index('source');
-        });
-        changes.push('resources.source added');
-    }
-    if (changes.length === 0) {
-        return;
-    }
-    // eslint-disable-next-line no-console
-    console.error(`[migrate] ${changes.join(', ')}`);
-}