@nitpicker/crawler 0.11.0 → 0.13.0

package/lib/archive/meta/types.d.ts

@@ -15,6 +15,14 @@
  * downstream consumers (e.g. `find-mismatches`) can compare against the
  * absolute page URL directly.
  * @see derive-flat-from-meta.ts
+ * @example
+ * // A `pages` row projected onto (a subset of) its flat meta columns:
+ * const meta: Partial<FlatPageMetaColumns> = {
+ *   title: 'Home',
+ *   canonical: 'https://example.com/',
+ *   robots_noindex: 0,
+ *   og_type: 'website',
+ * };
  */
 export interface FlatPageMetaColumns {
     lang: string | null;
@@ -70,6 +78,12 @@ export interface FlatPageMetaColumns {
  * `page_tags` / `page_jsonld` for the common "how many?" and "which
  * providers?" questions.
  * @see compute-page-denormalized.ts
+ * @example
+ * const denorm: PageDenormalizedColumns = {
+ *   tag_count: 3,
+ *   jsonld_count: 1,
+ *   tags_providers_csv: 'Google Analytics,Google Tag Manager',
+ * };
  */
 export interface PageDenormalizedColumns {
     /** Total Wappalyzer tag entries for the page. */
@@ -86,6 +100,16 @@ export interface PageDenormalizedColumns {
  * `<script type="speculationrules">` (`kind = 'speculationrules'`) entries.
  * @see extract-tags-for-archive.ts (sibling for tags) and the table definition
  * in `archive/init-schema.ts`.
+ * @example
+ * const row: JsonLdRow = {
+ *   id: 1,
+ *   pageId: 42,
+ *   kind: 'ld+json',
+ *   type: 'Article',
+ *   raw: '{"@type":"Article","headline":"Hello"}',
+ *   parsed: { '@type': 'Article', headline: 'Hello' },
+ *   parseError: null,
+ * };
  */
 export interface JsonLdRow {
     /** Auto-increment primary key. */
@@ -108,6 +132,15 @@ export interface JsonLdRow {
  *
  * Mirrors the row shape minus the auto-increment `id`. `parsed` is the raw
  * JSON value (the database layer JSON-stringifies it before write).
+ * @example
+ * const insert: JsonLdRowForInsert = {
+ *   pageId: 42,
+ *   kind: 'speculationrules',
+ *   type: null,
+ *   raw: '{"prerender":[]}',
+ *   parsed: { prerender: [] },
+ *   parseError: null,
+ * };
  */
 export type JsonLdRowForInsert = Omit<JsonLdRow, 'id'>;
 /**
@@ -115,6 +148,24 @@ export type JsonLdRowForInsert = Omit<JsonLdRow, 'id'>;
  *
  * Each row represents one detected Wappalyzer provider × external-id tuple for
  * one page. A page typically has 1–10 rows.
+ * @example
+ * const row: TagRow = {
+ *   id: 1,
+ *   pageId: 42,
+ *   provider: 'Google Tag Manager',
+ *   category: 'Tag managers',
+ *   externalId: 'GTM-XXXX',
+ *   version: null,
+ *   confidence: 100,
+ *   categories: ['Tag managers'],
+ *   sources: [
+ *     {
+ *       type: 'script-src',
+ *       src: 'https://www.googletagmanager.com/gtm.js',
+ *       location: 'head',
+ *     },
+ *   ],
+ * };
  */
 export interface TagRow {
     /** Auto-increment primary key. */
@@ -147,6 +198,17 @@ export interface TagRow {
  * Mirrors the row shape minus the auto-increment `id`. `categories` and
  * `sources` are passed as plain JS arrays (the database layer JSON-stringifies
  * them before write).
+ * @example
+ * const insert: TagRowForInsert = {
+ *   pageId: 42,
+ *   provider: 'Google Analytics',
+ *   category: 'Analytics',
+ *   externalId: 'G-XXXX',
+ *   version: null,
+ *   confidence: 100,
+ *   categories: ['Analytics'],
+ *   sources: [{ type: 'window-global', globalName: 'gtag' }],
+ * };
  */
 export type TagRowForInsert = Omit<TagRow, 'id'>;
 /**
@@ -155,6 +217,12 @@ export type TagRowForInsert = Omit<TagRow, 'id'>;
  * Keeps the response token-bounded for MCP / LLM consumers; the full `raw`
  * payload is fetched separately via `get-page-jsonld(url)`.
  * @see summarize-jsonld.ts
+ * @example
+ * const summary: JsonLdSummary = {
+ *   count: 2,
+ *   types: ['Article', 'BreadcrumbList'],
+ *   parseErrorCount: 0,
+ * };
  */
 export interface JsonLdSummary {
     /** Total entries across `ld+json` and `speculationrules`. */
@@ -167,6 +235,11 @@ export interface JsonLdSummary {
 /**
  * Summary of one page's Wappalyzer tags returned by `get-page-detail`.
  * @see summarize-tags.ts
+ * @example
+ * const summary: TagsSummary = {
+ *   count: 2,
+ *   providerIds: { 'Google Analytics': ['G-XXXX'], YouTube: [] },
+ * };
  */
 export interface TagsSummary {
     /** Total tag rows for the page. */
@@ -180,6 +253,8 @@ export interface TagsSummary {
  *
  * Used by `get-tag-inventory` to return per-provider page counts across the
  * whole site.
+ * @example
+ * const entry: TagInventoryEntry = { provider: 'Google Tag Manager', pageCount: 120 };
  */
 export interface TagInventoryEntry {
     /** Wappalyzer provider name. */
@@ -193,6 +268,17 @@ export interface TagInventoryEntry {
  *
  * Catch this at CLI / viewer boundaries to print a friendly message; do not
  * confuse with generic `Error` thrown by `Database.connect` (lockfile / I/O).
+ * @example
+ * try {
+ *   const accessor = await Archive.openCached(filePath);
+ * } catch (error) {
+ *   if (error instanceof IncompatibleArchiveError) {
+ *     // Message names the migration script(s) to run.
+ *     console.error(error.message);
+ *   } else {
+ *     throw error;
+ *   }
+ * }
  */
 export declare class IncompatibleArchiveError extends Error {
     readonly archiveVersion: string;
@@ -201,7 +287,7 @@ export declare class IncompatibleArchiveError extends Error {
      * @param archiveVersion - The `info.version` value read from the archive
      *   (or `'unknown'` when the column is missing / null).
      * @param requiredVersion - The minimum format version this build accepts
-     *   (semver string, e.g. `'0.10.0'`).
+     *   (semver string, e.g. `'0.13.0'`).
      */
     constructor(archiveVersion: string, requiredVersion: string);
 }

package/lib/archive/meta/types.js

@@ -7,12 +7,24 @@
  * objects returned by `get-page-detail` consumers.
  * @module
  */
+import { compareSemver } from './compare-semver.js';
 /**
  * Error thrown by `assert-compatible-version` when the archive's
  * `info.version` is older than the format version this build accepts.
  *
  * Catch this at CLI / viewer boundaries to print a friendly message; do not
  * confuse with generic `Error` thrown by `Database.connect` (lockfile / I/O).
+ * @example
+ * try {
+ *   const accessor = await Archive.openCached(filePath);
+ * } catch (error) {
+ *   if (error instanceof IncompatibleArchiveError) {
+ *     // Message names the migration script(s) to run.
+ *     console.error(error.message);
+ *   } else {
+ *     throw error;
+ *   }
+ * }
  */
 export class IncompatibleArchiveError extends Error {
     archiveVersion;
@@ -21,13 +33,33 @@ export class IncompatibleArchiveError extends Error {
      * @param archiveVersion - The `info.version` value read from the archive
      *   (or `'unknown'` when the column is missing / null).
      * @param requiredVersion - The minimum format version this build accepts
-     *   (semver string, e.g. `'0.10.0'`).
+     *   (semver string, e.g. `'0.13.0'`).
      */
     constructor(archiveVersion, requiredVersion) {
         super(`Archive uses Nitpicker ${archiveVersion}; this build requires ${requiredVersion} or newer. ` +
-            `Run \`node scripts/migrate-to-0.10.mjs <path>\` to produce an upgraded copy next to it.`);
+            `Run ${suggestMigrationScript(archiveVersion)} to produce an upgraded copy next to it.`);
         this.archiveVersion = archiveVersion;
         this.requiredVersion = requiredVersion;
         this.name = 'IncompatibleArchiveError';
     }
 }
+/**
+ * Selects the migration script an operator should run to bring
+ * `archiveVersion` up to the current {@link IncompatibleArchiveError.requiredVersion}.
+ * Chained: pre-0.10 archives run migrate-to-0.10 first, then
+ * migrate-to-0.13 — so the message points at BOTH steps in order.
+ * 0.10.0-through-0.12.x archives only need migrate-to-0.13.
+ *
+ * Uses {@link compareSemver} instead of `<` string comparison: `'0.9.0'`
+ * lexicographically compares GREATER than `'0.10.0'` (because `'9' > '1'`),
+ * which would misroute pre-0.10 archives into the single-step hint and
+ * make the resulting migrator invocation fail with a confusing error.
+ * @param archiveVersion - Semver read from `info.version`, or `'unknown'`.
+ * @returns Bracketed command string for embedding into the error message.
+ */
+function suggestMigrationScript(archiveVersion) {
+    if (archiveVersion === 'unknown' || compareSemver(archiveVersion, '0.10.0') < 0) {
+        return '`node scripts/migrate-to-0.10.mjs <path>` (then `node scripts/migrate-to-0.13.mjs <path>`)';
+    }
+    return '`node scripts/migrate-to-0.13.mjs <path>`';
+}

package/lib/archive/migrate-entity-tables.d.ts

@@ -0,0 +1,45 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the 0.13 entity / edge tables (issue #192) to archives created
+ * before the 0.13 format.
+ *
+ * These are the 6 core normalised tables (`content_items`, `page_meta`,
+ * `resource_items`, `anchor_edges`, `resource_ref_edges`, `image_items`) that
+ * replace the legacy write model in the 0.13 format. Fresh archives get
+ * them via `initSchema`; this migration is called only by
+ * `scripts/migrate-to-0.13.mjs` as its schema catch-up step for pre-0.13
+ * input archives. The archive-open path (`db-ops/lifecycle/init.ts`) does
+ * not run it — `assertCompatibleVersion` guarantees every openable
+ * archive already has the full table set.
+ *
+ * The migration is intentionally table-only. It does not back-fill any data
+ * from `pages` / `resources` / `anchors` / `images` — that is the job of the
+ * population steps (`populateRefTables` + `populateEntityTables`), which
+ * read from the legacy tables and write
+ * the entity rows in a single WAL transaction with `.bak` protection. This
+ * migration only guarantees the empty tables exist so the populate step can
+ * INSERT into them without hitting `SQLITE_ERROR: no such table:
+ * content_items`.
+ *
+ * **Blocked-by contract**: `createEntityTables` references the
+ * 0.13 ref tables (`url_refs`, `content_type_refs`, `text_refs`,
+ * `json_refs`, `blob_refs`, `header_sets`) via `REFERENCES`. On an
+ * existing archive, `migrateRefTables` MUST have run first so the
+ * ref tables exist by the time this migration is applied.
+ * `scripts/migrate-to-0.13.mjs` orders the two calls statically so this
+ * ordering is enforced there, not here.
+ *
+ * **Idempotency**: `createEntityTables` itself uses
+ * `CREATE TABLE IF NOT EXISTS` / `CREATE INDEX IF NOT EXISTS` for every
+ * statement, so calling it multiple times against any DB state is safe.
+ * That means this wrapper does not need a defensive sentinel — the
+ * "content_items exists but page_meta was dropped externally" partial-
+ * corruption case is fully repaired on next open because every CREATE
+ * statement individually checks its own existence. The one thing the
+ * wrapper adds beyond the primitive is a log line, which we still want
+ * to emit only when creation actually happened, so we probe
+ * `content_items` first and short-circuit when the schema is already
+ * fully installed.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migrateEntityTables(instance: Knex): Promise<void>;

package/lib/archive/migrate-entity-tables.js

@@ -0,0 +1,56 @@
+import { createEntityTables } from './create-entity-tables.js';
+/**
+ * Adds the 0.13 entity / edge tables (issue #192) to archives created
+ * before the 0.13 format.
+ *
+ * These are the 6 core normalised tables (`content_items`, `page_meta`,
+ * `resource_items`, `anchor_edges`, `resource_ref_edges`, `image_items`) that
+ * replace the legacy write model in the 0.13 format. Fresh archives get
+ * them via `initSchema`; this migration is called only by
+ * `scripts/migrate-to-0.13.mjs` as its schema catch-up step for pre-0.13
+ * input archives. The archive-open path (`db-ops/lifecycle/init.ts`) does
+ * not run it — `assertCompatibleVersion` guarantees every openable
+ * archive already has the full table set.
+ *
+ * The migration is intentionally table-only. It does not back-fill any data
+ * from `pages` / `resources` / `anchors` / `images` — that is the job of the
+ * population steps (`populateRefTables` + `populateEntityTables`), which
+ * read from the legacy tables and write
+ * the entity rows in a single WAL transaction with `.bak` protection. This
+ * migration only guarantees the empty tables exist so the populate step can
+ * INSERT into them without hitting `SQLITE_ERROR: no such table:
+ * content_items`.
+ *
+ * **Blocked-by contract**: `createEntityTables` references the
+ * 0.13 ref tables (`url_refs`, `content_type_refs`, `text_refs`,
+ * `json_refs`, `blob_refs`, `header_sets`) via `REFERENCES`. On an
+ * existing archive, `migrateRefTables` MUST have run first so the
+ * ref tables exist by the time this migration is applied.
+ * `scripts/migrate-to-0.13.mjs` orders the two calls statically so this
+ * ordering is enforced there, not here.
+ *
+ * **Idempotency**: `createEntityTables` itself uses
+ * `CREATE TABLE IF NOT EXISTS` / `CREATE INDEX IF NOT EXISTS` for every
+ * statement, so calling it multiple times against any DB state is safe.
+ * That means this wrapper does not need a defensive sentinel — the
+ * "content_items exists but page_meta was dropped externally" partial-
+ * corruption case is fully repaired on next open because every CREATE
+ * statement individually checks its own existence. The one thing the
+ * wrapper adds beyond the primitive is a log line, which we still want
+ * to emit only when creation actually happened, so we probe
+ * `content_items` first and short-circuit when the schema is already
+ * fully installed.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migrateEntityTables(instance) {
+    const hasContentItems = await instance.schema.hasTable('content_items');
+    if (hasContentItems) {
+        // Fast path: fully installed. `createEntityTables` would
+        // still be a safe no-op, but re-running it just to be defensive
+        // wastes 15 IF NOT EXISTS round-trips on every archive open.
+        return;
+    }
+    await createEntityTables(instance);
+    // eslint-disable-next-line no-console
+    console.error('[migrate] 0.13 entity/edge tables created');
+}

package/lib/archive/migrate-ref-tables.d.ts

@@ -0,0 +1,25 @@
+import type { Knex } from 'knex';
+/**
+ * Adds the 0.13 dictionary tables (issue #190) to archives created before
+ * the 0.13 format.
+ *
+ * These are the 10 ref / header dictionary tables that form the
+ * durable write-model of the 0.13 format. Fresh archives get them via
+ * `initSchema`; this migration is called only by
+ * `scripts/migrate-to-0.13.mjs` as its schema catch-up step for pre-0.13
+ * input archives. The archive-open path (`db-ops/lifecycle/init.ts`) does
+ * not run it — `assertCompatibleVersion` guarantees every openable
+ * archive already has the full table set.
+ *
+ * The migration is intentionally table-only. It does not back-fill any data
+ * from `pages` / `resources` / `anchors` / `images` into the ref tables — that
+ * is the job of the 0.13 population step, which reads from the legacy
+ * tables and writes the ref rows in a single WAL transaction with `.bak`
+ * protection. This migration only guarantees the empty tables exist so
+ * consumers don't crash with `SQLITE_ERROR: no such table: url_refs`.
+ *
+ * Idempotent: presence of `url_refs` is used as the sentinel (all 10
+ * tables are created together, so any one of them can serve as the sentinel).
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migrateRefTables(instance: Knex): Promise<void>;

package/lib/archive/migrate-ref-tables.js

@@ -0,0 +1,38 @@
+import { createRefTables } from './create-ref-tables.js';
+/**
+ * Adds the 0.13 dictionary tables (issue #190) to archives created before
+ * the 0.13 format.
+ *
+ * These are the 10 ref / header dictionary tables that form the
+ * durable write-model of the 0.13 format. Fresh archives get them via
+ * `initSchema`; this migration is called only by
+ * `scripts/migrate-to-0.13.mjs` as its schema catch-up step for pre-0.13
+ * input archives. The archive-open path (`db-ops/lifecycle/init.ts`) does
+ * not run it — `assertCompatibleVersion` guarantees every openable
+ * archive already has the full table set.
+ *
+ * The migration is intentionally table-only. It does not back-fill any data
+ * from `pages` / `resources` / `anchors` / `images` into the ref tables — that
+ * is the job of the 0.13 population step, which reads from the legacy
+ * tables and writes the ref rows in a single WAL transaction with `.bak`
+ * protection. This migration only guarantees the empty tables exist so
+ * consumers don't crash with `SQLITE_ERROR: no such table: url_refs`.
+ *
+ * Idempotent: presence of `url_refs` is used as the sentinel (all 10
+ * tables are created together, so any one of them can serve as the sentinel).
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migrateRefTables(instance) {
+    const hasUrlRefs = await instance.schema.hasTable('url_refs');
+    if (hasUrlRefs) {
+        return;
+    }
+    const hasPages = await instance.schema.hasTable('pages');
+    if (!hasPages) {
+        // Empty archive; the regular initSchema path will create the tables.
+        return;
+    }
+    await createRefTables(instance);
+    // eslint-disable-next-line no-console
+    console.error('[migrate] 0.13 ref/header tables created');
+}

package/lib/archive/page-meta-column-maps.d.ts

@@ -0,0 +1,32 @@
+import type { FlatPageMetaColumns } from './meta/types.js';
+/**
+ * Flat-meta-field → `page_meta` FK-column mapping, shared by the two
+ * writers that materialise `page_meta` rows: the live-crawl upsert
+ * (`db-ops/pages/write/insert-page.ts`) and the archive-migration
+ * populate (`populate-entity-tables/populate-page-meta.ts`).
+ *
+ * The two writers MUST agree on this mapping — a fork would make
+ * live-crawled archives and migrated archives disagree on which flat
+ * meta field feeds which `page_meta` column for structurally identical
+ * inputs. Keeping the single definition here is what enforces that;
+ * adding, renaming, or removing a flat meta column is a one-place edit.
+ *
+ * - `text`: text-shaped columns mapped 1:1 to `<name>_text_id` FKs into
+ *   `text_refs`.
+ * - `url`: URL-shaped columns mapped 1:1 to `<name>_url_id` FKs into
+ *   `url_refs`.
+ * @example
+ * for (const { source, target } of PAGE_META_COLUMN_MAPS.text) {
+ *   row[target] = textIds.get(flat[source]) ?? null;
+ * }
+ */
+export declare const PAGE_META_COLUMN_MAPS: {
+    readonly text: readonly {
+        source: keyof FlatPageMetaColumns;
+        target: string;
+    }[];
+    readonly url: readonly {
+        source: keyof FlatPageMetaColumns;
+        target: string;
+    }[];
+};

package/lib/archive/page-meta-column-maps.js

@@ -0,0 +1,43 @@
+/**
+ * Flat-meta-field → `page_meta` FK-column mapping, shared by the two
+ * writers that materialise `page_meta` rows: the live-crawl upsert
+ * (`db-ops/pages/write/insert-page.ts`) and the archive-migration
+ * populate (`populate-entity-tables/populate-page-meta.ts`).
+ *
+ * The two writers MUST agree on this mapping — a fork would make
+ * live-crawled archives and migrated archives disagree on which flat
+ * meta field feeds which `page_meta` column for structurally identical
+ * inputs. Keeping the single definition here is what enforces that;
+ * adding, renaming, or removing a flat meta column is a one-place edit.
+ *
+ * - `text`: text-shaped columns mapped 1:1 to `<name>_text_id` FKs into
+ *   `text_refs`.
+ * - `url`: URL-shaped columns mapped 1:1 to `<name>_url_id` FKs into
+ *   `url_refs`.
+ * @example
+ * for (const { source, target } of PAGE_META_COLUMN_MAPS.text) {
+ *   row[target] = textIds.get(flat[source]) ?? null;
+ * }
+ */
+export const PAGE_META_COLUMN_MAPS = {
+    text: [
+        { source: 'title', target: 'title_text_id' },
+        { source: 'description', target: 'description_text_id' },
+        { source: 'keywords', target: 'keywords_text_id' },
+        { source: 'robots_raw', target: 'robots_raw_text_id' },
+        { source: 'og_title', target: 'og_title_text_id' },
+        { source: 'og_description', target: 'og_description_text_id' },
+        { source: 'twitter_title', target: 'twitter_title_text_id' },
+        { source: 'twitter_description', target: 'twitter_description_text_id' },
+    ],
+    url: [
+        { source: 'canonical', target: 'canonical_url_id' },
+        { source: 'amphtml', target: 'amphtml_url_id' },
+        { source: 'manifest', target: 'manifest_url_id' },
+        { source: 'icon_href', target: 'icon_url_id' },
+        { source: 'appleTouchIcon_href', target: 'apple_touch_icon_url_id' },
+        { source: 'og_url', target: 'og_url_id' },
+        { source: 'og_image', target: 'og_image_url_id' },
+        { source: 'twitter_image', target: 'twitter_image_url_id' },
+    ],
+};

package/lib/archive/page.d.ts

@@ -14,13 +14,13 @@ declare const FLAT_META_COLUMNS: readonly ["lang", "dir", "charset", "baseHref",
  * Represents a crawled page stored in the archive.
  *
  * Provides typed getters for the most-used meta columns (title, canonical,
- * og:*, twitter_card, robots flags, lang), plus {@link metaFlat} as an
- * iterable view over all ~47 flat meta columns and {@link metaExtras} for
+ * og:*, twitter_card, robots flags, lang), plus {@link Page.metaFlat} as an
+ * iterable view over all ~47 flat meta columns and {@link Page.metaExtras} for
  * the JSON catch-all of nested sub-objects not flattened to columns.
  *
  * JSON-LD entries and Wappalyzer tag rows live in dedicated tables and are
- * fetched on demand via {@link jsonLd} / {@link tags} (lazy reads, same
- * pattern as {@link getAnchors}).
+ * fetched on demand via {@link Page.getJsonLd} / {@link Page.getTags} (lazy reads, same
+ * pattern as {@link Page.getAnchors}).
  *
  * Instances are created by {@link ArchiveAccessor.getPages} or
  * {@link ArchiveAccessor.getPagesWithRefs}.
@@ -177,7 +177,7 @@ export default class Page {
     /**
      * Sorted unique Wappalyzer provider names, comma-separated, empty string
      * when no tags. Denormalised aggregate; for the structured form fetch
-     * {@link tags} (lazy).
+     * {@link Page.getTags} (lazy).
      */
     get tagsProvidersCsv(): string;
     /**
@@ -281,8 +281,8 @@ export default class Page {
         isInternalPage: boolean;
         getAnchors: Anchor[];
         getReferrers: Referrer[];
-        dir: string | number | null;
         lang: string | number | null;
+        dir: string | number | null;
         charset: string | number | null;
         baseHref: string | number | null;
         viewport_raw: string | number | null;

package/lib/archive/page.js

@@ -65,13 +65,13 @@ const FLAT_META_COLUMNS = [
  * Represents a crawled page stored in the archive.
  *
  * Provides typed getters for the most-used meta columns (title, canonical,
- * og:*, twitter_card, robots flags, lang), plus {@link metaFlat} as an
- * iterable view over all ~47 flat meta columns and {@link metaExtras} for
+ * og:*, twitter_card, robots flags, lang), plus {@link Page.metaFlat} as an
+ * iterable view over all ~47 flat meta columns and {@link Page.metaExtras} for
  * the JSON catch-all of nested sub-objects not flattened to columns.
  *
  * JSON-LD entries and Wappalyzer tag rows live in dedicated tables and are
- * fetched on demand via {@link jsonLd} / {@link tags} (lazy reads, same
- * pattern as {@link getAnchors}).
+ * fetched on demand via {@link Page.getJsonLd} / {@link Page.getTags} (lazy reads, same
+ * pattern as {@link Page.getAnchors}).
  *
  * Instances are created by {@link ArchiveAccessor.getPages} or
  * {@link ArchiveAccessor.getPagesWithRefs}.
@@ -308,7 +308,7 @@ export default class Page {
     /**
      * Sorted unique Wappalyzer provider names, comma-separated, empty string
      * when no tags. Denormalised aggregate; for the structured form fetch
-     * {@link tags} (lazy).
+     * {@link Page.getTags} (lazy).
      */
     get tagsProvidersCsv() {
         return this.#raw.tags_providers_csv ?? '';

package/lib/archive/peek-archive-lock.d.ts

@@ -23,13 +23,13 @@ export interface ArchiveLockHolder {
 /**
  * Probe `<tmpDir>.lock/pid.txt` without acquiring the lock.
  *
- * Mirror of the alive-check inside {@link acquireArchiveLock}, exposed for
+ * Mirror of the alive-check inside `acquireArchiveLock`, exposed for
  * read-only consumers so they can detect (and surface) a concurrent crawler
  * without competing for the lock. Returns `null` when no lock directory is
  * present or the pid file is missing/malformed — callers should treat that
  * as "no detectable crawler" rather than as an error.
  *
- * Co-located with {@link acquireArchiveLock} so the writer side and the
+ * Co-located with `acquireArchiveLock` so the writer side and the
  * read-only probe stay in lockstep when the lock format evolves (e.g.
  * adding a hostname/timestamp field).
  * @param tmpDir - The archive's temporary working directory whose

package/lib/archive/peek-archive-lock.js

@@ -3,13 +3,13 @@ import path from 'node:path';
 /**
  * Probe `<tmpDir>.lock/pid.txt` without acquiring the lock.
  *
- * Mirror of the alive-check inside {@link acquireArchiveLock}, exposed for
+ * Mirror of the alive-check inside `acquireArchiveLock`, exposed for
  * read-only consumers so they can detect (and surface) a concurrent crawler
  * without competing for the lock. Returns `null` when no lock directory is
  * present or the pid file is missing/malformed — callers should treat that
  * as "no detectable crawler" rather than as an error.
  *
- * Co-located with {@link acquireArchiveLock} so the writer side and the
+ * Co-located with `acquireArchiveLock` so the writer side and the
  * read-only probe stay in lockstep when the lock format evolves (e.g.
  * adding a hostname/timestamp field).
  * @param tmpDir - The archive's temporary working directory whose

package/lib/archive/populate-entity-tables/collapse-anchor-rows.d.ts

@@ -0,0 +1,41 @@
+import type { AnchorEdgeRowInProgress, AnchorInputRow } from './types.js';
+/**
+ * Collapses consecutive `anchors`-shaped rows into `anchor_edges` rows by
+ * `(pageId, hrefId)` (issue #193).
+ *
+ * **Input contract**: `rows` MUST be sorted by `(pageId, hrefId, id)`
+ * ascending. The caller (`populate-anchor-edges.ts`) achieves this via a
+ * single keyset scan `ORDER BY pageId, hrefId, id` over `anchors`.
+ *
+ * The `id`-then-key sort matters because the dedup rule is
+ * "first instance wins": the smallest `anchors.id` for a given
+ * `(pageId, hrefId)` pair contributes its `hash` and `textContent` to
+ * the edge; every subsequent duplicate is counted but its body is
+ * discarded. A naive `min(hash) GROUP BY pageId, hrefId` would pick the
+ * lexicographically smallest hex string (uniformly distributed) rather
+ * than the earliest occurrence, so the pass has to happen in JS.
+ *
+ * The generator yields one {@link AnchorEdgeRowInProgress} per distinct
+ * pair as soon as the next pair boundary is observed. Emitting eagerly
+ * keeps peak memory bounded to O(1) — the collapser holds at most one
+ * open pair state at a time.
+ *
+ * `first_text_id` is intentionally left `null` here; the caller resolves
+ * it in a second pass after all edges are known (see
+ * `populate-anchor-edges.ts`). The `first_hash` field is set to the
+ * first row's `hash` verbatim; a first row with `hash === null` results
+ * in `first_hash = null` on the edge (rare but possible on legacy
+ * archives that failed to compute a hash).
+ * @param rows - Anchor input rows sorted by `(pageId, hrefId, id)`.
+ * @yields {AnchorEdgeRowInProgress} One entry per distinct
+ *   `(pageId, hrefId)` pair in the input.
+ * @example
+ * const edges = [...collapseAnchorRows([
+ *   { id: 1, pageId: 10, hrefId: 20, hash: 'a', textContent: 'first' },
+ *   { id: 2, pageId: 10, hrefId: 20, hash: 'b', textContent: 'second' },
+ *   { id: 3, pageId: 10, hrefId: 30, hash: 'c', textContent: 'x' },
+ * ])];
+ * // edges[0] = { page_id: 10, href_page_id: 20, count: 2, first_hash: 'a', ... }
+ * // edges[1] = { page_id: 10, href_page_id: 30, count: 1, first_hash: 'c', ... }
+ */
+export declare function collapseAnchorRows(rows: Iterable<AnchorInputRow>): Generator<AnchorEdgeRowInProgress>;