@nitpicker/archive 0.0.0 → 0.25.0

package/lib/populate-entity-tables/resolve-url-refs.d.ts

@@ -0,0 +1,33 @@
+import type { UrlRefIdMap } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Batch-resolves `url_refs.id` for a set of URL strings (issue #193).
+ *
+ * Called by every entity populate helper that needs to translate a
+ * legacy URL string column into a `url_id` FK. Rather than open a
+ * separate SELECT per row (which multiplies migration wall-clock by
+ * chunk size × URL column count) the resolver collects the distinct
+ * strings the caller cares about, splits them into chunks of
+ * {@link LOOKUP_CHUNK_SIZE}, issues one `WHERE url IN (?, ...)` per
+ * chunk, and returns a single map.
+ *
+ * Missing entries fall out of the map — callers distinguish "URL is a
+ * large data URI routed to blob_refs" from "URL is absent from the
+ * dictionary" via a separate {@link ./resolve-blob-refs.ts} probe.
+ *
+ * Duplicates in `urls` are deduped internally so an image whose `src`
+ * and `currentSrc` are the same value only counts once against
+ * `LOOKUP_CHUNK_SIZE`.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param urls - Iterable of URL strings to resolve. Empty and null-like
+ *   values are ignored.
+ * @returns Map keyed by the raw URL string; missing entries indicate
+ *   the URL is not present in `url_refs`.
+ * @example
+ * const idMap = await resolveUrlRefs(trx, [
+ *   'https://example.com/a',
+ *   'https://example.com/b',
+ * ]);
+ * const id = idMap.get('https://example.com/a'); // number | undefined
+ */
+export declare function resolveUrlRefs(trx: Knex, urls: Iterable<string>): Promise<UrlRefIdMap>;

package/lib/populate-entity-tables/resolve-url-refs.js

@@ -0,0 +1,60 @@
+/**
+ * Rows sent per `SELECT ... WHERE url IN (?, ...)` chunk. SQLite's default
+ * `SQLITE_MAX_VARIABLE_NUMBER` is 32766 on modern builds — 800 URLs per
+ * chunk stays well under that while amortising the round-trip cost across
+ * many rows.
+ */
+const LOOKUP_CHUNK_SIZE = 800;
+/**
+ * Batch-resolves `url_refs.id` for a set of URL strings (issue #193).
+ *
+ * Called by every entity populate helper that needs to translate a
+ * legacy URL string column into a `url_id` FK. Rather than open a
+ * separate SELECT per row (which multiplies migration wall-clock by
+ * chunk size × URL column count) the resolver collects the distinct
+ * strings the caller cares about, splits them into chunks of
+ * {@link LOOKUP_CHUNK_SIZE}, issues one `WHERE url IN (?, ...)` per
+ * chunk, and returns a single map.
+ *
+ * Missing entries fall out of the map — callers distinguish "URL is a
+ * large data URI routed to blob_refs" from "URL is absent from the
+ * dictionary" via a separate {@link ./resolve-blob-refs.ts} probe.
+ *
+ * Duplicates in `urls` are deduped internally so an image whose `src`
+ * and `currentSrc` are the same value only counts once against
+ * `LOOKUP_CHUNK_SIZE`.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param urls - Iterable of URL strings to resolve. Empty and null-like
+ *   values are ignored.
+ * @returns Map keyed by the raw URL string; missing entries indicate
+ *   the URL is not present in `url_refs`.
+ * @example
+ * const idMap = await resolveUrlRefs(trx, [
+ *   'https://example.com/a',
+ *   'https://example.com/b',
+ * ]);
+ * const id = idMap.get('https://example.com/a'); // number | undefined
+ */
+export async function resolveUrlRefs(trx, urls) {
+    const distinct = new Set();
+    for (const url of urls) {
+        if (typeof url === 'string' && url !== '') {
+            distinct.add(url);
+        }
+    }
+    if (distinct.size === 0) {
+        return new Map();
+    }
+    const values = [...distinct];
+    const result = new Map();
+    for (let index = 0; index < values.length; index += LOOKUP_CHUNK_SIZE) {
+        const chunk = values.slice(index, index + LOOKUP_CHUNK_SIZE);
+        const rows = await trx('url_refs')
+            .select('id', 'url')
+            .whereIn('url', chunk);
+        for (const row of rows) {
+            result.set(row.url, row.id);
+        }
+    }
+    return result;
+}

package/lib/populate-entity-tables/test-utils/count-rows.d.ts

@@ -0,0 +1,17 @@
+import type { Knex } from 'knex';
+/**
+ * Runs `SELECT count(<countColumn>) FROM <table>` and returns the count
+ * as a plain JS number. Kept in this package instead of importing from
+ * `populate-ref-tables/test-utils/count-rows.ts` because a spec in populate-entities importing
+ * from populate-refs test-utils would establish a cross-directory test-utils
+ * dependency; the populate-refs copy has the same shape but its default
+ * countColumn is scoped to that phase's tables. Duplication is minimal
+ * and keeps each phase's test scaffolding self-contained.
+ * @param db - Knex instance (typically a spec-local in-memory DB).
+ * @param table - Table name.
+ * @param countColumn - Column to count; defaults to `'*'` so this helper
+ *   works uniformly across `content_items` / `page_meta` /
+ *   `resource_ref_edges` (some of which do not have an `id` column).
+ * @returns Row count.
+ */
+export declare function countRows(db: Knex, table: string, countColumn?: string): Promise<number>;

package/lib/populate-entity-tables/test-utils/count-rows.js

@@ -0,0 +1,20 @@
+/**
+ * Runs `SELECT count(<countColumn>) FROM <table>` and returns the count
+ * as a plain JS number. Kept in this package instead of importing from
+ * `populate-ref-tables/test-utils/count-rows.ts` because a spec in populate-entities importing
+ * from populate-refs test-utils would establish a cross-directory test-utils
+ * dependency; the populate-refs copy has the same shape but its default
+ * countColumn is scoped to that phase's tables. Duplication is minimal
+ * and keeps each phase's test scaffolding self-contained.
+ * @param db - Knex instance (typically a spec-local in-memory DB).
+ * @param table - Table name.
+ * @param countColumn - Column to count; defaults to `'*'` so this helper
+ *   works uniformly across `content_items` / `page_meta` /
+ *   `resource_ref_edges` (some of which do not have an `id` column).
+ * @returns Row count.
+ */
+export async function countRows(db, table, countColumn = '*') {
+    const rows = await db(table).count({ n: countColumn });
+    const first = rows[0];
+    return Number(first.n);
+}

package/lib/populate-entity-tables/test-utils/seed-content-items.d.ts

@@ -0,0 +1,25 @@
+import type knex from 'knex';
+/**
+ * Inserts one row into each of `pages`, `url_refs`, and `content_items` per
+ * requested id so downstream inserts against tables that FK to
+ * `content_items(id)` (`anchor_edges`, `image_items`, `page_meta`, …) satisfy
+ * their foreign key without every spec re-authoring the same 20 lines of
+ * seed logic.
+ *
+ * Shared by 0.13 check specs (`check-anchor-edges-count.spec.ts`,
+ * `check-anchor-edges-sum.spec.ts`, `check-image-items-count.spec.ts`,
+ * `verify-migration.spec.ts`) so a schema change to
+ * pages/url_refs/content_items only requires editing one file. Kept in
+ * `populate-entity-tables/test-utils/` alongside {@link setup-entities-db.ts} because that
+ * module already provisions the tables this helper writes to.
+ *
+ * Rows are inserted with defaults suitable for count / structural
+ * invariants: `scraped=1`, `isTarget=1`, `isExternal=0`, `source='crawled'`,
+ * URL `https://example.com/<id>` and matching `url_refs` row on the same id
+ * so `url_id` binds 1:1 with `pages.id` (helpful for URL round-trip specs).
+ * @param db - Knex handle from {@link setupMigrationDb}.
+ * @param ids - Content-item ids to create. Ordering does not matter but the
+ *   helper does not deduplicate the input list — callers should pass
+ *   distinct ids.
+ */
+export declare function seedContentItems(db: ReturnType<typeof knex>, ids: readonly number[]): Promise<void>;

package/lib/populate-entity-tables/test-utils/seed-content-items.js

@@ -0,0 +1,42 @@
+/**
+ * Inserts one row into each of `pages`, `url_refs`, and `content_items` per
+ * requested id so downstream inserts against tables that FK to
+ * `content_items(id)` (`anchor_edges`, `image_items`, `page_meta`, …) satisfy
+ * their foreign key without every spec re-authoring the same 20 lines of
+ * seed logic.
+ *
+ * Shared by 0.13 check specs (`check-anchor-edges-count.spec.ts`,
+ * `check-anchor-edges-sum.spec.ts`, `check-image-items-count.spec.ts`,
+ * `verify-migration.spec.ts`) so a schema change to
+ * pages/url_refs/content_items only requires editing one file. Kept in
+ * `populate-entity-tables/test-utils/` alongside {@link setup-entities-db.ts} because that
+ * module already provisions the tables this helper writes to.
+ *
+ * Rows are inserted with defaults suitable for count / structural
+ * invariants: `scraped=1`, `isTarget=1`, `isExternal=0`, `source='crawled'`,
+ * URL `https://example.com/<id>` and matching `url_refs` row on the same id
+ * so `url_id` binds 1:1 with `pages.id` (helpful for URL round-trip specs).
+ * @param db - Knex handle from {@link setupMigrationDb}.
+ * @param ids - Content-item ids to create. Ordering does not matter but the
+ *   helper does not deduplicate the input list — callers should pass
+ *   distinct ids.
+ */
+export async function seedContentItems(db, ids) {
+    for (const id of ids) {
+        await db('pages').insert({
+            id,
+            url: `https://example.com/${id}`,
+            scraped: 1,
+            isTarget: 1,
+        });
+        await db('url_refs').insert({ id, url: `https://example.com/${id}` });
+        await db('content_items').insert({
+            id,
+            url_id: id,
+            is_external: 0,
+            scraped: 1,
+            is_target: 1,
+            source: 'crawled',
+        });
+    }
+}

package/lib/populate-entity-tables/test-utils/setup-entities-db.d.ts

@@ -0,0 +1,30 @@
+import knex from 'knex';
+/**
+ * Provisions an in-memory SQLite database with:
+ *
+ * - A minimal subset of the legacy write model (`pages`, `resources`,
+ *   `anchors`, `images`, `resources-referrers`, `page_html_ref`,
+ *   `page_html_blobs`) — only the columns the 0.13 populates
+ *   actually read.
+ * - The 0.13 ref / header tables (via {@link createRefTables}).
+ * - The 0.13 entity tables (via {@link createEntityTables}).
+ * - The 0.13 adjunct tables (via {@link createAdjunctTables}) — required
+ *   because `content_items.dedupe_cap_event_id REFERENCES
+ *   dedupe_cap_events(id)`; under `PRAGMA foreign_keys = ON` (enabled
+ *   below), inserting into `content_items` fails with `no such table:
+ *   dedupe_cap_events` if the adjunct tables were skipped. `initSchema`
+ *   always calls both create functions together, so this mirrors a real
+ *   archive's actual schema rather than an artificially incomplete one.
+ *
+ * Every 0.13 populate spec calls this to obtain a fresh DB. The
+ * caller is responsible for `db.destroy()` (spec `afterEach`).
+ *
+ * `PRAGMA foreign_keys = ON` is enabled at setup time so specs mirror
+ * the migration script's execution mode. `content_items.redirect_dest_id`'s
+ * DEFERRABLE FK is enforced at COMMIT, so specs that exercise the
+ * self-reference must run their inserts inside a `db.transaction`
+ * boundary (see `populate-content-items.spec.ts`'s redirect test) —
+ * bare inserts would trip the FK immediately.
+ * @returns Connected Knex instance.
+ */
+export declare function setupMigrationDb(): Promise<ReturnType<typeof knex>>;

package/lib/populate-entity-tables/test-utils/setup-entities-db.js

@@ -0,0 +1,175 @@
+import knex from 'knex';
+import { createAdjunctTables } from '../../create-adjunct-tables.js';
+import { createEntityTables } from '../../create-entity-tables.js';
+import { createRefTables } from '../../create-ref-tables.js';
+import { LibsqlDialect } from '../../libsql-dialect.js';
+/**
+ * Provisions an in-memory SQLite database with:
+ *
+ * - A minimal subset of the legacy write model (`pages`, `resources`,
+ *   `anchors`, `images`, `resources-referrers`, `page_html_ref`,
+ *   `page_html_blobs`) — only the columns the 0.13 populates
+ *   actually read.
+ * - The 0.13 ref / header tables (via {@link createRefTables}).
+ * - The 0.13 entity tables (via {@link createEntityTables}).
+ * - The 0.13 adjunct tables (via {@link createAdjunctTables}) — required
+ *   because `content_items.dedupe_cap_event_id REFERENCES
+ *   dedupe_cap_events(id)`; under `PRAGMA foreign_keys = ON` (enabled
+ *   below), inserting into `content_items` fails with `no such table:
+ *   dedupe_cap_events` if the adjunct tables were skipped. `initSchema`
+ *   always calls both create functions together, so this mirrors a real
+ *   archive's actual schema rather than an artificially incomplete one.
+ *
+ * Every 0.13 populate spec calls this to obtain a fresh DB. The
+ * caller is responsible for `db.destroy()` (spec `afterEach`).
+ *
+ * `PRAGMA foreign_keys = ON` is enabled at setup time so specs mirror
+ * the migration script's execution mode. `content_items.redirect_dest_id`'s
+ * DEFERRABLE FK is enforced at COMMIT, so specs that exercise the
+ * self-reference must run their inserts inside a `db.transaction`
+ * boundary (see `populate-content-items.spec.ts`'s redirect test) —
+ * bare inserts would trip the FK immediately.
+ * @returns Connected Knex instance.
+ */
+export async function setupMigrationDb() {
+    const db = knex({
+        client: LibsqlDialect,
+        connection: { filename: ':memory:' },
+        useNullAsDefault: true,
+    });
+    await db.raw('PRAGMA foreign_keys = ON');
+    await db.schema.createTable('pages', (t) => {
+        t.increments('id');
+        t.string('url').notNullable();
+        t.integer('redirectDestId').nullable();
+        t.boolean('scraped').notNullable();
+        t.boolean('isTarget').notNullable();
+        t.boolean('isExternal');
+        t.integer('status');
+        t.string('statusText');
+        t.string('contentType').nullable();
+        t.integer('contentLength').nullable();
+        t.json('responseHeaders').nullable();
+        t.string('lang');
+        t.string('dir');
+        t.string('charset');
+        t.string('baseHref');
+        t.text('viewport_raw');
+        t.string('themeColor');
+        t.string('applicationName');
+        t.string('author');
+        t.string('generator');
+        t.string('publisher');
+        t.string('title');
+        t.text('description');
+        t.text('keywords');
+        t.text('robots_raw');
+        t.integer('robots_noindex');
+        t.integer('robots_nofollow');
+        t.integer('robots_noarchive');
+        t.integer('robots_noimageindex');
+        t.string('googlebot');
+        t.string('canonical');
+        t.string('amphtml');
+        t.string('manifest');
+        t.string('icon_href');
+        t.string('appleTouchIcon_href');
+        t.string('og_type');
+        t.string('og_title');
+        t.string('og_url');
+        t.string('og_site_name');
+        t.text('og_description');
+        t.string('og_image');
+        t.string('og_image_alt');
+        t.string('og_image_width');
+        t.string('og_image_height');
+        t.string('og_locale');
+        t.string('og_article_published_time');
+        t.string('og_article_modified_time');
+        t.string('twitter_card');
+        t.string('twitter_site');
+        t.string('twitter_creator');
+        t.string('twitter_title');
+        t.text('twitter_description');
+        t.string('twitter_image');
+        t.string('fb_app_id');
+        t.string('verification_google');
+        t.integer('formatDetection_telephone');
+        t.integer('firstCrawledAt');
+        t.integer('lastCrawledAt');
+        t.integer('tag_count');
+        t.integer('jsonld_count');
+        t.text('tags_providers_csv');
+        t.json('meta_extras');
+        t.boolean('isSkipped');
+        t.string('skipReason');
+        t.integer('order');
+        t.string('source').notNullable().defaultTo('crawled');
+    });
+    await db.schema.createTable('resources', (t) => {
+        t.increments('id');
+        t.string('url').notNullable();
+        t.boolean('isExternal');
+        t.integer('status');
+        t.string('statusText');
+        t.string('contentType').nullable();
+        t.integer('contentLength').nullable();
+        t.string('compress').nullable();
+        t.string('cdn').nullable();
+        t.json('responseHeaders').nullable();
+        t.string('source').notNullable().defaultTo('crawled');
+    });
+    await db.schema.createTable('anchors', (t) => {
+        t.increments('id');
+        t.integer('pageId').notNullable();
+        t.integer('hrefId').notNullable();
+        t.string('hash');
+        t.string('textContent').nullable();
+    });
+    await db.schema.createTable('images', (t) => {
+        t.increments('id');
+        t.integer('pageId').notNullable();
+        t.string('src').nullable();
+        t.string('currentSrc').nullable();
+        t.string('alt');
+        t.float('width');
+        t.float('height');
+        t.integer('naturalWidth');
+        t.integer('naturalHeight');
+        t.boolean('isLazy');
+        t.integer('viewportWidth');
+        t.string('sourceCode');
+    });
+    await db.schema.createTable('resources-referrers', (t) => {
+        t.increments('id');
+        t.integer('resourceId').notNullable();
+        t.integer('pageId').notNullable();
+        t.unique(['resourceId', 'pageId']);
+    });
+    // `populate-image-items.ts` reads HTML BLOBs via the same trx that
+    // owns the outer populate transaction (see that file's JSDoc for the
+    // deadlock the inline read avoids). Provide the two tables so specs
+    // can seed rows or leave them empty (the reader returns `null` on a
+    // missing row, matching the `getHtmlOfPageById` contract). Raw SQL
+    // so the schema mirrors `init-schema.ts` exactly (`WITHOUT ROWID`,
+    // `size_raw` + `size_stored`, `codec` CHECK).
+    await db.raw(`
+		CREATE TABLE page_html_blobs (
+			hash         BLOB PRIMARY KEY,
+			body         BLOB NOT NULL,
+			codec        TEXT NOT NULL CHECK(codec IN ('zstd', 'none')),
+			size_raw     INTEGER NOT NULL,
+			size_stored  INTEGER NOT NULL
+		) WITHOUT ROWID
+	`);
+    await db.raw(`
+		CREATE TABLE page_html_ref (
+			page_id  INTEGER PRIMARY KEY,
+			hash     BLOB NOT NULL REFERENCES page_html_blobs(hash)
+		) WITHOUT ROWID
+	`);
+    await createRefTables(db);
+    await createEntityTables(db);
+    await createAdjunctTables(db);
+    return db;
+}

package/lib/populate-entity-tables/types.d.ts

@@ -0,0 +1,157 @@
+/**
+ * Domain types for the 0.13 entity/edge population step (issue #193).
+ *
+ * The 0.13 populate helpers translate rows from the legacy write
+ * model (`pages`, `resources`, `anchors`, `images`, `resources-referrers`)
+ * into the normalised entity + edge tables (`content_items`, `page_meta`,
+ * `resource_items`, `anchor_edges`, `resource_ref_edges`, `image_items`)
+ * that 0.13 created. Every ref-id lookup goes through a small set of
+ * shared resolvers whose interfaces live here per the repo-wide
+ * `型は types.ts に集約` rule.
+ */
+/**
+ * Mapping from `url_refs.url` (the natural key) to `url_refs.id`. Populated
+ * on demand from the DB by {@link ./resolve-url-refs.ts}. `null` on values
+ * that are stored in {@link ./resolve-blob-refs.ts} instead (large data
+ * URIs — see the routing rule in
+ * {@link ../populate-ref-tables/populate-url-refs.ts}) or on lookup miss.
+ */
+export interface UrlRefResolution {
+    /** `url_refs.id` when the URL string is present in the dictionary. */
+    readonly urlId: number | null;
+    /** `blob_refs.id` when the value is a large data URI routed to blob_refs. */
+    readonly blobId: number | null;
+}
+/**
+ * Batch-resolved `text_refs.id` for a set of raw text strings. Keyed by
+ * the raw text verbatim so the caller does not need to re-hash for
+ * cache probes.
+ */
+export type TextRefIdMap = ReadonlyMap<string, number>;
+/**
+ * Batch-resolved `url_refs.id` for a set of URL strings. Keyed by the
+ * raw URL verbatim; missing entries mean the URL is either a large data
+ * URI (routed to `blob_refs`) or absent from the dictionary — the caller
+ * distinguishes via a separate `blob_refs` probe.
+ */
+export type UrlRefIdMap = ReadonlyMap<string, number>;
+/**
+ * Batch-resolved `blob_refs.id` for a set of large data URI values.
+ * Keyed by the raw value verbatim (the data URI string), consistent with
+ * how `populateBlobRefs` inserts one row per distinct data URI payload
+ * hash — the resolver looks up by SHA-256 of the decoded payload but
+ * returns the map keyed by the raw URI so callers do not have to keep
+ * the decoded bytes around.
+ */
+export type BlobRefIdMap = ReadonlyMap<string, number>;
+/**
+ * Batch-resolved `content_type_refs.id` for a set of raw content-type
+ * header values. Keyed by the raw value exactly as stored in
+ * `pages.contentType` / `resources.contentType`; missing entries mean
+ * the content type is null or absent from the dictionary (which should
+ * not happen after `populateContentTypeRefs` populates every distinct
+ * value).
+ */
+export type ContentTypeRefIdMap = ReadonlyMap<string, number>;
+/**
+ * Batch-resolved `header_sets.id` for a set of raw `responseHeaders`
+ * JSON strings. Keyed by the raw JSON string; the resolver hashes each
+ * value in JS and looks up `header_sets.raw_json_hash` because SQLite
+ * has no built-in BLAKE3 / SHA-256 (see
+ * {@link ../populate-ref-tables/compute-content-hash.ts} for the algorithm choice).
+ */
+export type HeaderSetIdMap = ReadonlyMap<string, number>;
+/**
+ * One collapsed anchor edge produced by the single-pass keyset scan in
+ * {@link ./populate-anchor-edges.ts}. Rows are emitted in `(page_id,
+ * href_page_id)` order; `first_text_id` is resolved in a second pass
+ * after `text_refs` lookups.
+ */
+export interface AnchorEdgeRow {
+    /** `content_items.id` of the page containing the anchor. */
+    page_id: number;
+    /** `content_items.id` of the anchor target. */
+    href_page_id: number;
+    /** Number of `anchors` rows collapsed into this edge. */
+    count: number;
+    /** `first_hash` — the `anchors.hash` of the first (lowest-id) instance. */
+    first_hash: string | null;
+    /**
+     * `first_text_id` — resolved from `first_textContent` against
+     * `text_refs`. `null` when the first instance had no textContent (an
+     * empty `<a>` tag) or when the text failed to resolve.
+     */
+    first_text_id: number | null;
+}
+/**
+ * One input row for {@link ./collapse-anchor-rows.ts}. Every field
+ * mirrors the legacy `anchors` schema; the collapser does not depend on
+ * knex or the DB so it can be unit-tested with plain arrays.
+ */
+export interface AnchorInputRow {
+    /** Legacy `anchors.id` — used only to sort input; not stored on the edge. */
+    id: number;
+    /** Legacy `anchors.pageId` — `content_items.id` of the source page. */
+    pageId: number;
+    /** Legacy `anchors.hrefId` — `content_items.id` of the anchor target. */
+    hrefId: number;
+    /** Legacy `anchors.hash` — verbatim SHA-256 hex from beholder. */
+    hash: string | null;
+    /** Legacy `anchors.textContent` — anchor visible text (empty for `<a>` with no content). */
+    textContent: string | null;
+}
+/**
+ * Intermediate shape yielded by {@link ./collapse-anchor-rows.ts}. Differs
+ * from the final {@link AnchorEdgeRow} in that `first_text_id` has not
+ * been resolved yet — the raw text is carried through as
+ * `first_textContent` so the second pass can batch-look up every
+ * distinct text against `text_refs`.
+ */
+export interface AnchorEdgeRowInProgress {
+    /** `content_items.id` of the page. */
+    page_id: number;
+    /** `content_items.id` of the anchor target. */
+    href_page_id: number;
+    /** Number of collapsed `anchors` rows. */
+    count: number;
+    /** Verbatim `anchors.hash` of the first (lowest-id) instance. */
+    first_hash: string | null;
+    /**
+     * Verbatim `anchors.textContent` of the first instance — carried
+     * unresolved so the caller can batch-look up ids across every edge.
+     */
+    first_textContent: string | null;
+}
+/**
+ * Case discriminator for {@link ./match-images-to-dom-paths.ts}: which of
+ * the three resolution paths produced the returned `dom_path` string. The
+ * migration script's jsdom resolver emits a diagnostic warning log for
+ * `unknown` fallbacks so operators can audit reconstruction fidelity.
+ */
+export type DomPathDerivationCase = 'single-match' | 'ordinal-match' | 'unknown';
+/**
+ * Result of the DOM-path derivation for one legacy `images` row.
+ * `path` is the string to insert into `text_refs.text`; `case` records
+ * which branch produced it.
+ */
+export interface DomPathResult {
+    /** The `dom_path` string, e.g. `html/body[1]/main[1]/img[3]`. */
+    path: string;
+    /** Which branch produced the string. */
+    case: DomPathDerivationCase;
+}
+/**
+ * One `<img>` observed in a rendered / parsed document, in document
+ * order: its `outerHTML` (the matching key against the crawler's stored
+ * `sourceCode`) and its pre-derived `dom_path` string. Produced either
+ * by a jsdom walk over an archived HTML snapshot (migration script) or
+ * by an in-browser walk over the live page (crawler write path) — both
+ * feed {@link ./match-images-to-dom-paths.ts}, which is DOM-free and
+ * only compares these strings.
+ */
+export interface DomPathCandidate {
+    /** The element's `outerHTML`, verbatim. */
+    outerHTML: string;
+    /** The element's derived `dom_path` (see {@link ./derive-dom-path.ts}). */
+    path: string;
+}

package/lib/populate-entity-tables/types.js

@@ -0,0 +1,12 @@
+/**
+ * Domain types for the 0.13 entity/edge population step (issue #193).
+ *
+ * The 0.13 populate helpers translate rows from the legacy write
+ * model (`pages`, `resources`, `anchors`, `images`, `resources-referrers`)
+ * into the normalised entity + edge tables (`content_items`, `page_meta`,
+ * `resource_items`, `anchor_edges`, `resource_ref_edges`, `image_items`)
+ * that 0.13 created. Every ref-id lookup goes through a small set of
+ * shared resolvers whose interfaces live here per the repo-wide
+ * `型は types.ts に集約` rule.
+ */
+export {};

package/lib/populate-entity-tables/upsert-text-refs.d.ts

@@ -0,0 +1,38 @@
+import type { TextRefIdMap } from './types.js';
+import type { Knex } from 'knex';
+/**
+ * Upserts a set of text strings into `text_refs` and returns their ids
+ * (issue #193).
+ *
+ * Unlike {@link ./resolve-text-refs.ts}, which only reads, this helper
+ * inserts every missing string first. It exists specifically for
+ * `dom_path` strings, which are synthesised by the image-items populate
+ * ({@link ./populate-image-items.ts}) and are not among the columns the
+ * `text_refs` populate scans
+ * ({@link ../populate-ref-tables/populate-text-refs.ts}) — the caller
+ * cannot rely on the dictionary already containing them.
+ *
+ * Strategy:
+ *
+ * 1. **Hash and INSERT OR IGNORE** every input string in bulk. New rows
+ *    materialise; conflicts are silently skipped.
+ * 2. **Read back** by hash + text (the `(hash, text)` UNIQUE composite)
+ *    to pick up both freshly-inserted and pre-existing ids.
+ *
+ * The read-back narrows on both hash and text (same defensive shape as
+ * {@link ../populate-ref-tables/populate-header-tables.ts}'s `resolveValueIds`) so
+ * an astronomically improbable hash collision cannot leak the wrong
+ * id back to the caller.
+ *
+ * Empty / null strings are ignored; duplicates in `texts` are deduped
+ * internally.
+ * @param trx - Knex instance or transaction connected to the archive DB.
+ * @param texts - Iterable of text strings to upsert.
+ * @returns Map keyed by the raw text string.
+ * @example
+ * const idMap = await upsertTextRefs(trx, [
+ *   'html/body[1]/img[1]',
+ *   'unknown/42',
+ * ]);
+ */
+export declare function upsertTextRefs(trx: Knex, texts: Iterable<string>): Promise<TextRefIdMap>;