@nitpicker/archive 0.0.0 → 0.25.1

package/lib/template-classification/page-cluster/resolve-page-cluster-keys.js

@@ -0,0 +1,236 @@
+import { createTemplateClassificationRun } from '@nitpicker/core/create-template-classification-run';
+/** Pages whose signals are handed to the run per call. */
+const SIGNALS_PER_BATCH = 1000;
+/**
+ * Pages a read uses handed to the native run per call (the pages it does
+ * not use ride along without counting): enough to keep its thread pool busy
+ * on one call, few enough that the next batch is read while it runs. Only
+ * throughput depends on it, never the keys.
+ */
+const PAGES_PER_BATCH = 64;
+/**
+ * HTML (in UTF-16 code units) handed to the native run per call, at most, so
+ * a batch of unusually large pages does not hold their copies all at once.
+ * Only memory and throughput depend on it, never the keys.
+ */
+const HTML_PER_BATCH = 32 * 1024 * 1024;
+/**
+ * Pages whose HTML is being read at once, ahead of the one the batch takes
+ * next. Each read is a database query and a decompression, so one at a time
+ * leaves the native run waiting for the next batch (on a ~178,000-page
+ * archive, reading ahead took classification from 139 s to 110 s); a few at
+ * a time keeps it busy while bounding the HTML held in flight. Only
+ * throughput and memory depend on it, never the keys (pages still enter
+ * batches in order).
+ */
+const HTML_READ_AHEAD = 16;
+/** What a page a read does not use is handed over as. */
+const UNUSED_PAGE = Object.freeze({ html: '' });
+/**
+ * The error for a factory that yields a different corpus on a later read.
+ * @param produced How many pages the read produced: its exact count, or
+ *   `{ moreThan }` the first read's count when it was cut short at the
+ *   first page beyond it (the rest is not read).
+ * @param expected
+ */
+function factoryMismatch(produced, expected) {
+    const producedText = typeof produced === 'number'
+        ? `${produced} pages`
+        : `more than ${produced.moreThan} pages`;
+    return new Error(`resolvePageClusterKeys: a read of the page factory produced ${producedText} but the first read saw ${expected} — the factory must produce the same pages in the same order on every call`);
+}
+/**
+ * The page as one read hands it to the run: its HTML and content-root hint
+ * when the read uses it, the shared empty page otherwise.
+ * @param page
+ * @param used
+ */
+async function toPageHtml(page, used) {
+    if (!used) {
+        // A page the read does not use costs the run nothing, so it only holds
+        // its place in the corpus order.
+        return UNUSED_PAGE;
+    }
+    const html = await page.getHtml();
+    return {
+        html,
+        ...(page.contentRoot ? { contentRoot: page.contentRoot } : {}),
+    };
+}
+/**
+ * Reads `pages` once and hands them to `push` in batches, reading the HTML
+ * only of the pages `mask` marks (the others go with empty HTML, which the
+ * run does not read). The HTML of up to {@link HTML_READ_AHEAD} pages is read
+ * at once, and a batch is read while the previous one is processed (the
+ * native run works off Node's main thread); each push is awaited before the
+ * next one starts, as the run requires.
+ *
+ * Only the page count is checked against the first read: a factory yielding
+ * as many pages but different ones, or in another order, goes unnoticed.
+ * @param read
+ * @param read.pages
+ * @param read.mask The run's `pageMask()` for this read; its length is the
+ *   page count of the first read.
+ * @param read.push
+ */
+async function streamPages(read) {
+    const { pages, mask, push } = read;
+    let inFlight = Promise.resolve();
+    let batch = [];
+    let usedInBatch = 0;
+    let batchLength = 0;
+    let pagesRead = 0;
+    const reading = [];
+    let pushFailure;
+    const flush = async () => {
+        await inFlight;
+        inFlight = push(batch);
+        // Read on while it runs; a rejection is noted here (so it does not
+        // count as unhandled) and stops the reading at the next page.
+        inFlight.catch((error) => {
+            pushFailure ??= { error };
+        });
+        batch = [];
+        usedInBatch = 0;
+        batchLength = 0;
+    };
+    const takeNext = async () => {
+        const entry = await reading.shift();
+        batch.push(entry);
+        if (entry === UNUSED_PAGE)
+            return;
+        usedInBatch++;
+        batchLength += entry.html.length;
+        if (usedInBatch >= PAGES_PER_BATCH || batchLength >= HTML_PER_BATCH) {
+            await flush();
+        }
+    };
+    try {
+        for await (const page of pages()) {
+            if (pushFailure)
+                throw pushFailure.error;
+            if (pagesRead >= mask.length) {
+                throw factoryMismatch({ moreThan: mask.length }, mask.length);
+            }
+            const entry = toPageHtml(page, mask[pagesRead] === 1);
+            // Awaited in order by `takeNext`; until then a rejection must not
+            // count as unhandled.
+            entry.catch(() => { });
+            reading.push(entry);
+            pagesRead++;
+            if (reading.length > HTML_READ_AHEAD) {
+                await takeNext();
+            }
+        }
+        while (reading.length > 0) {
+            await takeNext();
+        }
+    }
+    catch (error) {
+        // Let the running call and the HTML reads already started settle
+        // before the caller cancels the run (and perhaps closes the archive),
+        // so nothing is left working on a run that is being given up.
+        await inFlight.catch(() => { });
+        await Promise.allSettled(reading);
+        throw error;
+    }
+    if (batch.length > 0) {
+        await flush();
+    }
+    await inFlight;
+    if (pagesRead !== mask.length) {
+        throw factoryMismatch(pagesRead, mask.length);
+    }
+}
+/**
+ * Reads every page's blocking signals into `run`, reporting every 1,000
+ * pages.
+ * @param read
+ * @param read.pages
+ * @param read.run
+ * @param read.onProgress
+ */
+async function pushSignals(read) {
+    const { pages, run, onProgress } = read;
+    let pagesSeen = 0;
+    let batch = [];
+    for await (const page of pages()) {
+        if (onProgress && pagesSeen > 0 && pagesSeen % SIGNALS_PER_BATCH === 0) {
+            onProgress({ phase: 'pass0-signals', pagesSeen });
+        }
+        pagesSeen++;
+        batch.push(page);
+        if (batch.length >= SIGNALS_PER_BATCH) {
+            run.pushSignals(batch);
+            batch = [];
+        }
+    }
+    run.pushSignals(batch);
+}
+/**
+ * Clusters pages into templates by DOM structure, in the native addon
+ * (`@nitpicker/core`'s `createTemplateClassificationRun`):
+ *
+ * - **Pass 0** splits the corpus into blocks by URL section and first-party
+ *   stylesheets, from the signals alone — no HTML.
+ * - **Stage A** clusters each block: landmark chrome is cut out, the content
+ *   depth is capped at the block's knee, and complete-linkage clustering
+ *   with a max-gap cut splits the block.
+ * - **Stage B** merges clusters across blocks and explains each final one.
+ *
+ * A block is clustered whole unless its Stage A similarity table
+ * (`n * (n - 1) / 2 * 8` bytes for `n` pages) exceeds `options.memoryBudget`,
+ * or the blocks clustered whole would hold more than
+ * `TEMPLATE_EXACT_PAGE_LIMIT` pages together (the largest are then sampled
+ * until they fit); a sampled block is clustered on a sample and a further
+ * read assigns its other pages. The factory is read twice (signals, HTML)
+ * or, with a sampled block that has pages outside its sample, three times; each page's HTML is read
+ * once, and JavaScript never holds the whole corpus's HTML.
+ * @param pages
+ * @param options
+ * @example
+ * ```ts
+ * const { keys, reasons } = await resolvePageClusterKeys(() => [
+ * 	{ paths: ['news', '1'], stylesheetHrefs: [], getHtml: () => '<body><article>one</article></body>' },
+ * 	{ paths: ['news', '2'], stylesheetHrefs: [], getHtml: () => '<body><article>two</article></body>' },
+ * 	{ paths: ['about'], stylesheetHrefs: [], getHtml: () => '<body><section>about</section></body>' },
+ * ]);
+ * // keys: ['["path:news","cluster:0"]', '["path:news","cluster:0"]', '["path:about","cluster:0"]']
+ * // reasons: one per cluster key, e.g. reasons.get('["path:about","cluster:0"]')?.memberCount === 1
+ * ```
+ */
+export async function resolvePageClusterKeys(pages, options) {
+    const onProgress = options?.onProgress;
+    const memoryBudget = options?.memoryBudget;
+    const run = createTemplateClassificationRun({
+        ...(memoryBudget === undefined ? {} : { memoryBudget }),
+        ...(onProgress ? { onProgress } : {}),
+    });
+    try {
+        await pushSignals({ pages, run, onProgress });
+        await run.block();
+        await streamPages({
+            pages,
+            mask: run.pageMask(),
+            push: (batch) => run.pushPages(batch),
+        });
+        await run.finishPages();
+        if (run.needsAssignment()) {
+            await streamPages({
+                pages,
+                mask: run.pageMask(),
+                push: (batch) => run.pushAssignmentPages(batch),
+            });
+        }
+        const output = await run.takeOutput();
+        return {
+            keys: Array.from(output.keyOfPage, (key) => output.keys[key]),
+            reasons: output.reasons,
+            sampledBlocks: output.sampledBlocks,
+        };
+    }
+    catch (error) {
+        run.cancel();
+        throw error;
+    }
+}

package/lib/template-classification/page-cluster/types.d.ts

@@ -0,0 +1,123 @@
+import type { ContentRootHint, TemplateClassificationRun, TemplateClusterReason, TemplatePageHtml, TemplateSampledBlock } from '@nitpicker/core/types';
+/**
+ * One page as `resolvePageClusterKeys` reads it: the blocking signals, read
+ * on every pass, and the HTML, read only on the passes that use the page.
+ */
+export interface PageClusterPage {
+    /** The page URL's path segments (`ExURL.paths`). */
+    readonly paths: readonly string[];
+    /** Absolute hrefs of the page's stylesheets. */
+    readonly stylesheetHrefs: readonly string[];
+    /**
+     * The page URL's host (hostname, optionally `:port` — `new URL(pageUrl).host`).
+     * Stylesheets on other hosts take no part in blocking.
+     */
+    readonly host?: string;
+    /**
+     * Identity of the page's content-root element, as a crawler detected it.
+     * Anchors the per-block content depth cap on sites without
+     * `<main>`/`role="main"`; without it the cap falls back to
+     * `<main>`/`role="main"` and then the built-in `#main`, `#content`, … list.
+     */
+    readonly contentRoot?: ContentRootHint;
+    /**
+     * The page's HTML, non-empty. Called only on the reads that use the page
+     * (every page of a block clustered whole, and the sampled pages of a
+     * sampled block, on the first read; the other pages of the sampled blocks
+     * on the second), so a page's HTML is decompressed once however many
+     * times the corpus is read.
+     */
+    getHtml(): string | Promise<string>;
+}
+/**
+ * Factory returning an iterator over the corpus. Called once per read — the
+ * signals, then the HTML, and, when a block was sampled, the HTML again for
+ * the pages outside the samples. Every call must yield the same pages in the
+ * same order; a corpus that changes size between reads is rejected.
+ *
+ * A factory rather than an `AsyncIterable`: one returned once cannot be
+ * traversed a second time, while a factory lets the caller re-issue its
+ * archive query per read, so no read holds the corpus's HTML at once.
+ */
+export type PageFactory = () => Iterable<PageClusterPage> | AsyncIterable<PageClusterPage>;
+/**
+ * Progress of `resolvePageClusterKeys`, a union on `phase`:
+ * - `pass0-signals`: reading the blocking signals; every 1,000 pages.
+ * - `block-sampled`: a block will be clustered from a sample, because its
+ *   similarity table exceeds the memory budget or because the blocks
+ *   clustered whole would hold more than `@nitpicker/core`'s
+ *   `TEMPLATE_EXACT_PAGE_LIMIT` pages and it is among the largest; once per
+ *   such block, before any HTML is read.
+ * - `pass1-block-complete`: one block's clusters are done.
+ * - `pass1b-assign`: only with a sampled block; every 1,000 pages assigned to
+ *   the sampled clusters.
+ * - `stage-b-start`: the merge across blocks began; once per call.
+ */
+export type ProgressEvent = {
+    readonly phase: 'pass0-signals';
+    readonly pagesSeen: number;
+} | {
+    readonly phase: 'block-sampled';
+    readonly blockKey: string;
+    readonly pageCount: number;
+    readonly pairBytes: number;
+    readonly budget: number;
+    readonly reason: 'table-over-budget' | 'pages-over-limit';
+} | {
+    readonly phase: 'pass1-block-complete';
+    readonly blockKey: string;
+    readonly blocksProcessed: number;
+    readonly totalBlocks: number;
+} | {
+    readonly phase: 'pass1b-assign';
+    readonly pagesAssigned: number;
+    readonly pagesToAssign: number;
+} | {
+    readonly phase: 'stage-b-start';
+    readonly unitCount: number;
+};
+/** Options of `resolvePageClusterKeys`. */
+export interface ResolvePageClusterKeysOptions {
+    /**
+     * Bytes a block's Stage A similarity table may take before the block is
+     * clustered from a sample instead (`@nitpicker/core`'s
+     * `TEMPLATE_MEMORY_BUDGET` by default; `0` samples every block).
+     */
+    readonly memoryBudget: number;
+    /** Receives every {@link ProgressEvent}. */
+    readonly onProgress: (event: ProgressEvent) => void;
+}
+/** What `resolvePageClusterKeys` returns. */
+export interface PageClusterKeys {
+    /** Each page's cluster key, in corpus order. */
+    readonly keys: string[];
+    /**
+     * Each final cluster's reason, by key. Built from the clusters' retained
+     * members (the samples, for a sampled block), sized by cluster count, not
+     * page count.
+     */
+    readonly reasons: ReadonlyMap<string, TemplateClusterReason>;
+    /** The blocks clustered from a sample, in block order. */
+    readonly sampledBlocks: readonly TemplateSampledBlock[];
+}
+/** One read of the corpus's HTML, as the driver hands it to the run. */
+export interface HtmlRead {
+    /** The corpus. */
+    readonly pages: PageFactory;
+    /**
+     * The run's `pageMask()` for this read: which pages it uses; its length
+     * is the page count of the first read.
+     */
+    readonly mask: Uint8Array;
+    /** Hands a batch to the run (`pushPages` or `pushAssignmentPages`). */
+    readonly push: (batch: TemplatePageHtml[]) => Promise<void>;
+}
+/** The read of every page's blocking signals into a run. */
+export interface SignalsRead {
+    /** The corpus. */
+    readonly pages: PageFactory;
+    /** The run the signals go to. */
+    readonly run: TemplateClassificationRun;
+    /** Receives the `pass0-signals` events. */
+    readonly onProgress: ((event: ProgressEvent) => void) | undefined;
+}

package/lib/template-classification/page-cluster/types.js

@@ -0,0 +1 @@
+export {};

package/lib/template-classification/types.d.ts

@@ -0,0 +1,234 @@
+import type Archive from '../archive.js';
+import type { TemplateClusterReason } from '../db-ops/templates/types.js';
+import type { ContentRootHint } from '@nitpicker/core/types';
+/**
+ * The four values of a page's detected main-content element that
+ * `createContentRootHint` builds the classifier's content-root hint from.
+ * Structurally the matching getters of `Page`, and the decoded
+ * `page_meta.main_content_*` columns `loadPageClusterCandidates` reads.
+ */
+export interface ContentRootSource {
+    /** The element's DOM node name (`'DIV'`), or `null`. */
+    readonly mainContentNodeName: string | null;
+    /** The element's `id`, or `null`. */
+    readonly mainContentId: string | null;
+    /** The element's `role`, or `null`. */
+    readonly mainContentRole: string | null;
+    /** The element's CSS classes, or `null` when no main region was detected. */
+    readonly mainContentClassList: readonly string[] | null;
+}
+/**
+ * What template classification keeps of one internal HTML page while it
+ * waits for the clustering: the page's id (to read its HTML by) and the
+ * signals the classifier reads without HTML. Deliberately not a `Page` — a
+ * `Page` holds the page's whole row (every meta column), and holding one per
+ * page of a several-hundred-thousand-page archive is what this type avoids.
+ * @example
+ * ```ts
+ * const candidate: PageClusterCandidate = {
+ * 	id: 12,
+ * 	href: 'https://example.com/news/1',
+ * 	paths: ['news', '1'],
+ * 	host: 'example.com',
+ * };
+ * ```
+ */
+export interface PageClusterCandidate {
+    /** `Page.id`; the key the page's HTML is read by. */
+    readonly id: number;
+    /** The page's `url.href`; the key of its result and of its stylesheet references. */
+    readonly href: string;
+    /** The page's URL path segments. */
+    readonly paths: readonly string[];
+    /** `hostname`, or `hostname:port` when the URL has a port. */
+    readonly host: string;
+    /** The crawler-detected main-content hint; absent when none was detected. */
+    readonly contentRoot?: ContentRootHint;
+}
+/**
+ * Progress of the clustering step itself. A copy of the page-cluster
+ * driver's `ProgressEvent` (`./page-cluster/resolve-page-cluster-keys.ts`,
+ * structurally assignable to this type), kept independent of it so
+ * `@nitpicker/cli` formats progress against a type archive exports rather
+ * than against the driver's internals, which are not part of archive's
+ * `exports` — the same reason {@link TemplateClusterReason} mirrors
+ * `@nitpicker/core`'s reason type.
+ */
+export type TemplateClusteringProgress = {
+    readonly phase: 'pass0-signals';
+    readonly pagesSeen: number;
+} | {
+    readonly phase: 'block-sampled';
+    readonly blockKey: string;
+    readonly pageCount: number;
+    readonly pairBytes: number;
+    readonly budget: number;
+    readonly reason: 'table-over-budget' | 'pages-over-limit';
+} | {
+    readonly phase: 'pass1-block-complete';
+    readonly blockKey: string;
+    readonly blocksProcessed: number;
+    readonly totalBlocks: number;
+} | {
+    readonly phase: 'pass1b-assign';
+    readonly pagesAssigned: number;
+    readonly pagesToAssign: number;
+} | {
+    readonly phase: 'stage-b-start';
+    readonly unitCount: number;
+};
+/**
+ * Progress of the whole crawl-end classification
+ * ({@link import('./classify-archive-page-templates.js').classifyArchivePageTemplates}):
+ * listing the pages with stored HTML, loading their signals out of the
+ * archive, gathering stylesheet references, the clustering phases, then
+ * writing the result.
+ *
+ * `loading-pages-start`, `collecting-stylesheets`, `listing-html-snapshots`
+ * and `writing-results` carry no count: each is a step with no natural
+ * progress unit (a count query followed by keyset batches that report their
+ * own `loading-pages` counts, three chained SQL passes, one join query, one
+ * write transaction), so they only announce that the step began — otherwise
+ * the previous step's last message would stay on screen and look like a stall.
+ */
+export type TemplateClassificationProgress = {
+    readonly phase: 'loading-pages-start';
+} | {
+    readonly phase: 'loading-pages';
+    readonly done: number;
+    readonly total: number;
+} | {
+    readonly phase: 'collecting-stylesheets';
+} | {
+    readonly phase: 'listing-html-snapshots';
+} | TemplateClusteringProgress | {
+    readonly phase: 'writing-results';
+    readonly templateCount: number;
+};
+/**
+ * A block clustered from a sample of its pages. Mirrors `@nitpicker/core`'s
+ * `TemplateSampledBlock`, for the same reason {@link TemplateClusterReason}
+ * mirrors its reason type.
+ * @example
+ * ```ts
+ * const block: SampledTemplateBlock = {
+ * 	blockKey: 'path:news',
+ * 	pageCount: 30_000,
+ * 	pairBytes: 3_599_880_000,
+ * 	reason: 'table-over-budget',
+ * };
+ * ```
+ */
+export interface SampledTemplateBlock {
+    /** The block's key. */
+    readonly blockKey: string;
+    /** Its pages. */
+    readonly pageCount: number;
+    /** Bytes its similarity table would have taken. */
+    readonly pairBytes: number;
+    /**
+     * Why it was sampled: its table exceeds the memory budget, or the blocks
+     * clustered whole would hold more than `@nitpicker/core`'s
+     * `TEMPLATE_EXACT_PAGE_LIMIT` pages and it is among the largest.
+     */
+    readonly reason: 'table-over-budget' | 'pages-over-limit';
+}
+/**
+ * Options for {@link import('./classify-page-templates.js').classifyPageTemplates}.
+ */
+export interface ClassifyPageTemplatesOptions {
+    /** The archive to read stylesheet references and the stored-HTML listing from. */
+    archive: Archive;
+    /**
+     * Every internal HTML page of the archive that has a stored HTML snapshot
+     * (`loadPageClusterCandidates`).
+     * Classification is a corpus-wide batch computation: template keys are
+     * only comparable within one call, so the pages must never be passed in
+     * per-batch slices.
+     */
+    candidates: readonly PageClusterCandidate[];
+    /**
+     * Bytes a block's Stage A similarity table may take before the block is
+     * clustered from a sample instead; see `resolvePageClusterKeys`. Defaults
+     * to `@nitpicker/core`'s `TEMPLATE_MEMORY_BUDGET`; `0` samples every block.
+     */
+    memoryBudget?: number;
+    /**
+     * Receives every {@link TemplateClassificationProgress} event of the call,
+     * including the clustering phases `resolvePageClusterKeys` reports, so
+     * long-running classification on large archives isn't silently
+     * unresponsive.
+     */
+    onProgress?: (event: TemplateClassificationProgress) => void;
+}
+/**
+ * Options for {@link import('./load-page-cluster-candidates.js').loadPageClusterCandidates}.
+ */
+export interface LoadPageClusterCandidatesOptions {
+    /**
+     * Output of `collectPageIdsWithHtml`: only pages whose id is in it are
+     * loaded, so pages with no stored HTML never take part (nor stay resident).
+     */
+    readonly pageIdsWithHtml: ReadonlySet<number>;
+    /** Receives a `loading-pages` event after each batch. */
+    readonly onProgress?: (event: TemplateClassificationProgress) => void;
+    /** Rows read per query. Defaults to 10,000; a bound on one query's transient garbage only. */
+    readonly batchSize?: number;
+}
+/**
+ * Options for {@link import('./create-page-cluster-factory.js').createPageClusterFactory}.
+ */
+export interface CreatePageClusterFactoryOptions {
+    /** Reads each page's HTML by id, only on the reads that use it. */
+    readonly archive: Pick<Archive, 'getHtmlOfPage'>;
+    /**
+     * The pages to cluster, as `loadPageClusterCandidates` returned them
+     * (already limited to pages with a stored HTML snapshot).
+     */
+    readonly candidates: readonly PageClusterCandidate[];
+    /** Output of `collectPageStylesheetUrls`, keyed by `url.href`. */
+    readonly stylesheetsByUrl: ReadonlyMap<string, readonly string[]>;
+}
+/**
+ * Result of {@link import('./classify-page-templates.js').classifyPageTemplates}.
+ */
+export interface PageTemplateClassification {
+    /**
+     * Page URL (`page.url.href`) → template key. Only internal HTML pages
+     * with retrievable HTML have an entry.
+     */
+    readonly templateKeysByUrl: ReadonlyMap<string, string>;
+    /**
+     * Template key → the classifier's cluster-selection evidence for that
+     * key, one per final cluster. Built from each cluster's retained members
+     * (the samples, for a sampled block), so `memberCount` can be smaller
+     * than the cluster's page count.
+     */
+    readonly clusterReasonsByTemplateKey: ReadonlyMap<string, TemplateClusterReason>;
+    /** The blocks clustered from a sample, in block order. */
+    readonly sampledBlocks: readonly SampledTemplateBlock[];
+}
+/**
+ * Options for {@link import('./classify-archive-page-templates.js').classifyArchivePageTemplates}.
+ */
+export interface ClassifyArchivePageTemplatesOptions {
+    /**
+     * Bytes a block's Stage A similarity table may take before the block is
+     * clustered from a sample instead; see `resolvePageClusterKeys`. Defaults
+     * to `@nitpicker/core`'s `TEMPLATE_MEMORY_BUDGET`; `0` samples every block.
+     */
+    readonly memoryBudget?: number;
+    /** Receives loading and clustering progress; omit for a silent run. */
+    readonly onProgress?: (event: TemplateClassificationProgress) => void;
+}
+/**
+ * Result of {@link import('./classify-archive-page-templates.js').classifyArchivePageTemplates}.
+ */
+export interface ClassifyArchivePageTemplatesResult {
+    /** Number of pages that received a template key. */
+    readonly classifiedPageCount: number;
+    /** Number of distinct template keys among them. */
+    readonly templateCount: number;
+    /** The blocks clustered from a sample, in block order; empty when none was. */
+    readonly sampledBlocks: readonly SampledTemplateBlock[];
+}

package/lib/template-classification/types.js

@@ -0,0 +1 @@
+export {};

package/lib/test-utils/build-full-schema-test-db.d.ts

@@ -0,0 +1,34 @@
+import type { Config } from '../types.js';
+import type { Knex } from 'knex';
+/**
+ * Builds a file-backed knex connection against the FULL current archive
+ * schema (every ref/entity/adjunct table, including columns only a
+ * migration adds to a legacy archive — `alias_of_id`, `dedupe_cap_event_id`,
+ * `body_hash`, …) by running the exact same {@link init} boot sequence a
+ * real `Archive.create`/`Archive.open` uses. Shared by every
+ * `archive/transfer/` spec that needs a realistic source or destination
+ * database — hand-rolling `createRefTables`/`createEntityTables`/
+ * `createAdjunctTables` directly would produce the schema `initSchema`
+ * creates but skip the migrations that add columns for archives created
+ * before those features existed (which happen to be a no-op on a fresh
+ * DB today, but drift is exactly the risk this helper avoids).
+ *
+ * A DB file must be used (not `:memory:`) because `ATTACH DATABASE` in
+ * the transfer primitives needs a real path — see `attach-source-database.ts`.
+ * @param filename - Absolute path to the SQLite file. The caller is
+ *   responsible for removing it (`afterEach`) — see existing specs for the
+ *   convention.
+ * @param config - Overrides merged over {@link DEFAULT_TEST_CONFIG} and
+ *   written via `setConfig` once the schema exists. Pass `null` to skip
+ *   writing an `info` row entirely (some specs seed `content_items`
+ *   directly and never read `getConfig()`).
+ * @returns The connected knex instance, on the current schema, with an
+ *   `info` row unless `config` is `null`.
+ * @example
+ * ```ts
+ * const db = await buildFullSchemaTestDb(path.resolve(workingDir, 'source.sqlite'), {
+ *   roots: ['https://a.example.com/'],
+ * });
+ * ```
+ */
+export declare function buildFullSchemaTestDb(filename: string, config?: Partial<Config> | null): Promise<Knex>;