@nitpicker/crawler 0.4.1

package/lib/archive/database.js

@@ -0,0 +1,972 @@
+var __runInitializers = (this && this.__runInitializers) || function (thisArg, initializers, value) {
+    var useValue = arguments.length > 2;
+    for (var i = 0; i < initializers.length; i++) {
+        value = useValue ? initializers[i].call(thisArg, value) : initializers[i].call(thisArg);
+    }
+    return useValue ? value : void 0;
+};
+var __esDecorate = (this && this.__esDecorate) || function (ctor, descriptorIn, decorators, contextIn, initializers, extraInitializers) {
+    function accept(f) { if (f !== void 0 && typeof f !== "function") throw new TypeError("Function expected"); return f; }
+    var kind = contextIn.kind, key = kind === "getter" ? "get" : kind === "setter" ? "set" : "value";
+    var target = !descriptorIn && ctor ? contextIn["static"] ? ctor : ctor.prototype : null;
+    var descriptor = descriptorIn || (target ? Object.getOwnPropertyDescriptor(target, contextIn.name) : {});
+    var _, done = false;
+    for (var i = decorators.length - 1; i >= 0; i--) {
+        var context = {};
+        for (var p in contextIn) context[p] = p === "access" ? {} : contextIn[p];
+        for (var p in contextIn.access) context.access[p] = contextIn.access[p];
+        context.addInitializer = function (f) { if (done) throw new TypeError("Cannot add initializers after decoration has completed"); extraInitializers.push(accept(f || null)); };
+        var result = (0, decorators[i])(kind === "accessor" ? { get: descriptor.get, set: descriptor.set } : descriptor[key], context);
+        if (kind === "accessor") {
+            if (result === void 0) continue;
+            if (result === null || typeof result !== "object") throw new TypeError("Object expected");
+            if (_ = accept(result.get)) descriptor.get = _;
+            if (_ = accept(result.set)) descriptor.set = _;
+            if (_ = accept(result.init)) initializers.unshift(_);
+        }
+        else if (_ = accept(result)) {
+            if (kind === "field") initializers.unshift(_);
+            else descriptor[key] = _;
+        }
+    }
+    if (target) Object.defineProperty(target, contextIn.name, descriptor);
+    done = true;
+};
+import path from 'node:path';
+import { tryParseUrl as parseUrl } from '@d-zero/shared/parse-url';
+import { retry } from '@d-zero/shared/retry';
+import { pathComparator } from '@d-zero/shared/sort/path';
+import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
+import knex from 'knex';
+import { ErrorEmitter, eachSplitted } from '../utils/index.js';
+import { dbLog } from './debug.js';
+import { mkdir } from './filesystem/index.js';
+const retrySetting = {
+    interval: 300,
+    retries: 3,
+};
+/**
+ * Low-level database abstraction layer for the archive's SQLite database.
+ *
+ * Manages the `pages`, `anchors`, `images`, `resources`, and `resources-referrers`
+ * tables. All public methods that perform database queries use the `@retryable`
+ * decorator for automatic retry on transient failures, and `@ErrorEmitter` to
+ * propagate errors as events.
+ *
+ * Use the static {@link Database.connect} factory method to create instances.
+ * The constructor is private.
+ */
+let Database = (() => {
+    let _classSuper = EventEmitter;
+    let _instanceExtraInitializers = [];
+    let _getAnchorsOnPage_decorators;
+    let _getBaseUrl_decorators;
+    let _getConfig_decorators;
+    let _getCrawlingState_decorators;
+    let _getHtmlPathOnPage_decorators;
+    let _getName_decorators;
+    let _getPageCount_decorators;
+    let _getPages_decorators;
+    let _getPagesWithRels_decorators;
+    let _getRedirectsForPages_decorators;
+    let _getReferrersOfPage_decorators;
+    let _getReferrersOfResource_decorators;
+    let _getResources_decorators;
+    let _getResourceUrlList_decorators;
+    let _insertResource_decorators;
+    let _insertResourceReferrers_decorators;
+    let _setConfig_decorators;
+    let _setSkippedPage_decorators;
+    let _updatePage_decorators;
+    return class Database extends _classSuper {
+        static {
+            const _metadata = typeof Symbol === "function" && Symbol.metadata ? Object.create(_classSuper[Symbol.metadata] ?? null) : void 0;
+            _getAnchorsOnPage_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _getBaseUrl_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _getConfig_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _getCrawlingState_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _getHtmlPathOnPage_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _getName_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _getPageCount_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _getPages_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _getPagesWithRels_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _getRedirectsForPages_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _getReferrersOfPage_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _getReferrersOfResource_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _getResources_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _getResourceUrlList_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _insertResource_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _insertResourceReferrers_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _setConfig_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _setSkippedPage_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _updatePage_decorators = [ErrorEmitter(), retry(retrySetting)];
+            __esDecorate(this, null, _getAnchorsOnPage_decorators, { kind: "method", name: "getAnchorsOnPage", static: false, private: false, access: { has: obj => "getAnchorsOnPage" in obj, get: obj => obj.getAnchorsOnPage }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _getBaseUrl_decorators, { kind: "method", name: "getBaseUrl", static: false, private: false, access: { has: obj => "getBaseUrl" in obj, get: obj => obj.getBaseUrl }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _getConfig_decorators, { kind: "method", name: "getConfig", static: false, private: false, access: { has: obj => "getConfig" in obj, get: obj => obj.getConfig }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _getCrawlingState_decorators, { kind: "method", name: "getCrawlingState", static: false, private: false, access: { has: obj => "getCrawlingState" in obj, get: obj => obj.getCrawlingState }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _getHtmlPathOnPage_decorators, { kind: "method", name: "getHtmlPathOnPage", static: false, private: false, access: { has: obj => "getHtmlPathOnPage" in obj, get: obj => obj.getHtmlPathOnPage }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _getName_decorators, { kind: "method", name: "getName", static: false, private: false, access: { has: obj => "getName" in obj, get: obj => obj.getName }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _getPageCount_decorators, { kind: "method", name: "getPageCount", static: false, private: false, access: { has: obj => "getPageCount" in obj, get: obj => obj.getPageCount }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _getPages_decorators, { kind: "method", name: "getPages", static: false, private: false, access: { has: obj => "getPages" in obj, get: obj => obj.getPages }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _getPagesWithRels_decorators, { kind: "method", name: "getPagesWithRels", static: false, private: false, access: { has: obj => "getPagesWithRels" in obj, get: obj => obj.getPagesWithRels }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _getRedirectsForPages_decorators, { kind: "method", name: "getRedirectsForPages", static: false, private: false, access: { has: obj => "getRedirectsForPages" in obj, get: obj => obj.getRedirectsForPages }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _getReferrersOfPage_decorators, { kind: "method", name: "getReferrersOfPage", static: false, private: false, access: { has: obj => "getReferrersOfPage" in obj, get: obj => obj.getReferrersOfPage }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _getReferrersOfResource_decorators, { kind: "method", name: "getReferrersOfResource", static: false, private: false, access: { has: obj => "getReferrersOfResource" in obj, get: obj => obj.getReferrersOfResource }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _getResources_decorators, { kind: "method", name: "getResources", static: false, private: false, access: { has: obj => "getResources" in obj, get: obj => obj.getResources }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _getResourceUrlList_decorators, { kind: "method", name: "getResourceUrlList", static: false, private: false, access: { has: obj => "getResourceUrlList" in obj, get: obj => obj.getResourceUrlList }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _insertResource_decorators, { kind: "method", name: "insertResource", static: false, private: false, access: { has: obj => "insertResource" in obj, get: obj => obj.insertResource }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _insertResourceReferrers_decorators, { kind: "method", name: "insertResourceReferrers", static: false, private: false, access: { has: obj => "insertResourceReferrers" in obj, get: obj => obj.insertResourceReferrers }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _setConfig_decorators, { kind: "method", name: "setConfig", static: false, private: false, access: { has: obj => "setConfig" in obj, get: obj => obj.setConfig }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _setSkippedPage_decorators, { kind: "method", name: "setSkippedPage", static: false, private: false, access: { has: obj => "setSkippedPage" in obj, get: obj => obj.setSkippedPage }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _updatePage_decorators, { kind: "method", name: "updatePage", static: false, private: false, access: { has: obj => "updatePage" in obj, get: obj => obj.updatePage }, metadata: _metadata }, null, _instanceExtraInitializers);
+            if (_metadata) Object.defineProperty(this, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
+        }
+        /** The Knex query builder instance connected to the SQLite database. */
+        #instance = __runInitializers(this, _instanceExtraInitializers);
+        /** Absolute path to the working directory, used for resolving relative snapshot paths. */
+        #workingDir;
+        // eslint-disable-next-line no-restricted-syntax
+        constructor(options) {
+            super();
+            this.#workingDir = options.workingDir;
+            switch (options.type) {
+                case 'sqlite3': {
+                    this.#instance = knex({
+                        client: options.type,
+                        connection: {
+                            filename: options.filename,
+                        },
+                        useNullAsDefault: true,
+                        pool: {
+                            acquireTimeoutMillis: 600_000,
+                        },
+                    });
+                    break;
+                }
+                case 'mysql': {
+                    throw new Error("Don't support MySQL yet.");
+                }
+            }
+        }
+        /**
+         * Adds the `order` column to the `pages` table for URL sort ordering.
+         * @deprecated Since v0.1.x. The column is now created during table initialization.
+         * @returns The result of the schema alteration.
+         */
+        async addOrderField() {
+            return await this.#instance.schema.table('pages', (t) => {
+                t.integer('order').unsigned().nullable().defaultTo(null);
+            });
+        }
+        /**
+         * Forces a WAL checkpoint, writing all pending WAL data back to the main database file.
+         * Uses TRUNCATE mode to reset the WAL file to zero bytes after checkpointing.
+         * This ensures the database is fully self-contained in `db.sqlite` before archiving.
+         */
+        async checkpoint() {
+            await this.#instance.raw('PRAGMA wal_checkpoint(TRUNCATE)');
+        }
+        async destroy() {
+            await this.#instance.destroy();
+        }
+        /**
+         * Retrieves all anchors (outgoing links) on a specific page.
+         * Joins the `anchors` table with the `pages` table to resolve link destinations.
+         * @param pageId - The database ID of the page whose anchors to retrieve.
+         * @returns An array of anchor records with resolved URL, title, status, and content type.
+         */
+        async getAnchorsOnPage(pageId) {
+            const res = await this.#instance
+                .select('pages.url', 'pages.title', 'pages.status', 'pages.statusText', 'pages.contentType', 'anchors.hash', 'anchors.textContent')
+                .from('anchors')
+                .join('pages', 'anchors.hrefId', '=', 'pages.id')
+                .where('anchors.pageId', pageId);
+            return res;
+        }
+        /**
+         * Retrieves the base URL of the crawl session from the `info` table.
+         * @returns The base URL string.
+         * @throws {Error} If no base URL is found in the database.
+         */
+        async getBaseUrl() {
+            const selected = await this.#instance.select('baseUrl').from('info');
+            if (!selected[0]) {
+                throw new Error('No baseUrl');
+            }
+            const [{ baseUrl }] = selected;
+            return baseUrl || '';
+        }
+        /**
+         * Retrieves the full crawl configuration from the `info` table.
+         * Deserializes JSON-encoded fields (`excludes`, `excludeKeywords`, `scope`).
+         * @returns The parsed {@link Config} object.
+         * @throws {Error} If no configuration is found in the database.
+         */
+        async getConfig() {
+            const [config] = await this.#instance.select('*').from('info');
+            if (!config) {
+                throw new Error('No config');
+            }
+            const opt = {
+                ...config,
+                excludes: getJSON(config.excludes, []),
+                excludeKeywords: getJSON(config.excludeKeywords, []),
+                excludeUrls: getJSON(config.excludeUrls, []),
+                scope: getJSON(config.scope, []),
+                retry: config.retry ?? 3,
+            };
+            // @ts-expect-error
+            delete opt.id;
+            dbLog('Table `info`: %O => %O', config, opt);
+            return opt;
+        }
+        /**
+         * Retrieves the current crawling state by listing scraped and pending URLs.
+         * @returns An object with `scraped` (completed URLs) and `pending` (remaining URLs) arrays.
+         */
+        async getCrawlingState() {
+            const ex = (r) => r.url;
+            const $scraped = await this.#instance
+                .select('url')
+                .from('pages')
+                .where('scraped', 1);
+            const scraped = $scraped.map(ex);
+            const $pending = await this.#instance
+                .select('url')
+                .from('pages')
+                .where('scraped', 0);
+            const pending = $pending.map(ex);
+            return {
+                scraped,
+                pending,
+            };
+        }
+        /**
+         * Retrieves the HTML snapshot file path for a specific page.
+         * @param pageId - The database ID of the page.
+         * @returns The relative file path to the HTML snapshot, or null if not saved.
+         */
+        async getHtmlPathOnPage(pageId) {
+            return await this.#instance.transaction(async (trx) => {
+                const [{ html }] = await trx
+                    .select('html')
+                    .from('pages')
+                    .where('id', pageId);
+                return html || null;
+            });
+        }
+        /**
+         * Retrieves the crawl session name from the `info` table.
+         * @returns The name string.
+         * @throws {Error} If no name is found in the database.
+         */
+        async getName() {
+            const selected = await this.#instance.select('name').from('info');
+            if (!selected[0]) {
+                throw new Error('No name');
+            }
+            const [{ name }] = selected;
+            return name;
+        }
+        /**
+         * Counts the total number of pages in the database.
+         * @returns The total page count.
+         * @throws {Error} If the count query fails.
+         */
+        async getPageCount() {
+            const selected = await this.#instance.count('id').from('pages');
+            if (!selected[0]) {
+                throw new Error('No count');
+            }
+            // @ts-expect-error
+            const count = selected[0]['count(`id`)'];
+            dbLog('Number of pages: %d', count);
+            return count;
+        }
+        /**
+         * Retrieves pages from the database with optional filtering, pagination via offset and limit.
+         * @param filter - An optional {@link PageFilter} to narrow results by content type and origin.
+         * @param offset - The number of rows to skip. Defaults to `0`.
+         * @param limit - The maximum number of rows to return. Defaults to `100000`.
+         * @returns An array of raw {@link DB_Page} rows.
+         */
+        async getPages(filter, offset = 0, limit = 100_000) {
+            const q = this.#instance.select('*').from('pages');
+            switch (filter) {
+                case 'page': {
+                    return q
+                        .where({
+                        contentType: 'text/html',
+                        isTarget: 1,
+                    })
+                        .limit(limit)
+                        .offset(offset);
+                }
+                case 'page-included-no-target': {
+                    return q
+                        .where({
+                        contentType: 'text/html',
+                    })
+                        .limit(limit)
+                        .offset(offset);
+                }
+                case 'external-page': {
+                    return q
+                        .where({
+                        contentType: 'text/html',
+                        isExternal: 1,
+                    })
+                        .limit(limit)
+                        .offset(offset);
+                }
+                case 'internal-page': {
+                    return q
+                        .where({
+                        contentType: 'text/html',
+                        isExternal: 0,
+                    })
+                        .limit(limit)
+                        .offset(offset);
+                }
+                case 'no-page': {
+                    return q
+                        .whereNull('contentType')
+                        .orWhereNot({
+                        contentType: 'text/html',
+                    })
+                        .limit(limit)
+                        .offset(offset);
+                }
+                case 'external-no-page': {
+                    return q
+                        .where((qb) => {
+                        qb.whereNull('contentType').orWhereNot({
+                            contentType: 'text/html',
+                        });
+                    })
+                        .andWhere({
+                        isExternal: 1,
+                    })
+                        .limit(limit)
+                        .offset(offset);
+                }
+                case 'internal-no-page': {
+                    return q
+                        .where((qb) => {
+                        qb.whereNull('contentType').orWhereNot({
+                            contentType: 'text/html',
+                        });
+                    })
+                        .andWhere({
+                        isExternal: 0,
+                    })
+                        .limit(limit)
+                        .offset(offset);
+                }
+            }
+            return q.limit(limit).offset(offset);
+        }
+        /**
+         * Retrieves pages along with their related redirect, anchor, and referrer data.
+         * Results are ordered by the natural URL sort order. Only non-redirected pages are returned.
+         * @param offset - The number of rows to skip.
+         * @param limit - The maximum number of pages to return.
+         * @returns An object containing `pages`, `redirects`, `anchors`, and `referrers` arrays.
+         */
+        async getPagesWithRels(offset, limit) {
+            await this.addOrderField().catch((error) => error);
+            await this.setUrlOrder();
+            dbLog('Get Pages');
+            const pages = await this.#instance
+                .select('*')
+                .from('pages')
+                .orderByRaw('`order` ASC NULLS LAST')
+                .whereNull('redirectDestId')
+                .limit(limit)
+                .offset(offset);
+            // When empty
+            if (pages.length === 0) {
+                return {
+                    pages: [],
+                    redirects: [],
+                    referrers: [],
+                    anchors: [],
+                };
+            }
+            dbLog('Get Pages: Redirects');
+            const redirects = await this.#instance
+                .with('limitedPages', limitedPageIds(limit, offset))
+                .with('redirect', redirectTable(false))
+                .select('id as pageId', 'from', 'fromId')
+                .from('redirect')
+                // Filter
+                .join('limitedPages', 'redirect.toId', '=', 'limitedPages.id')
+                // Sort
+                .orderBy('id', 'asc');
+            dbLog('Get Pages: Anchors');
+            const anchors = await this.#instance
+                .with('limitedPages', limitedPageIds(limit, offset))
+                .with('redirect', redirectTable())
+                .select('limitedPages.id as pageId', 'href.url', 'redirect.from as href', 'href.isExternal', 'href.title', 'href.status', 'href.statusText', 'href.contentType', 'anchors.hash', 'anchors.textContent')
+                .from('anchors')
+                // Filters
+                .join('limitedPages', 'anchors.pageId', '=', 'limitedPages.id')
+                // Resolves redirect
+                .join('redirect', 'anchors.hrefId', '=', 'redirect.fromId')
+                // Target
+                .join('pages as href', 'redirect.toId', '=', 'href.id')
+                // Sort
+                .orderBy('anchors.id', 'asc');
+            dbLog('Get Pages: Referrers');
+            const referrers = await this.#instance
+                .with('limitedPages', limitedPageIds(limit, offset))
+                .with('redirect', redirectTable())
+                .select('redirect.toId as pageId', 'referrer.url', 'redirect.from as through', 'redirect.fromId as throughId', 'anchors.hash', 'anchors.textContent')
+                .from('anchors')
+                // Resolves redirect
+                .join('redirect', 'anchors.hrefId', '=', 'redirect.fromId')
+                // Referrer
+                .join('pages as referrer', 'anchors.pageId', '=', 'referrer.id')
+                // Filters
+                .join('limitedPages', 'redirect.toId', '=', 'limitedPages.id')
+                // Sort
+                .orderBy('anchors.id', 'asc');
+            dbLog('Get Pages: Done');
+            return {
+                pages,
+                redirects,
+                anchors,
+                referrers,
+            };
+        }
+        /**
+         * Retrieves redirect sources for the given page IDs in bulk.
+         * @param pageIds - The database IDs of the destination pages.
+         * @returns An array of {@link DB_Redirect} records mapping destination pages to their redirect sources.
+         */
+        async getRedirectsForPages(pageIds) {
+            if (pageIds.length === 0)
+                return [];
+            return this.#instance
+                .select('redirectDestId as pageId', 'url as from', 'id as fromId')
+                .from('pages')
+                .whereIn('redirectDestId', pageIds);
+        }
+        /**
+         * Retrieves pages that link to a specific page (incoming links / referrers).
+         * @param pageId - The database ID of the target page.
+         * @returns An array of referrer records with URL, hash, and text content.
+         */
+        async getReferrersOfPage(pageId) {
+            const res = await this.#instance
+                .select('pages.url', 'anchors.hash', 'anchors.textContent')
+                .from('anchors')
+                .join('pages', 'anchors.pageId', '=', 'pages.id')
+                .where('anchors.hrefId', pageId);
+            return res;
+        }
+        /**
+         * Retrieves the page URLs that reference a specific resource.
+         * @param id - The database ID of the resource.
+         * @returns An array of page URL strings that reference the resource.
+         */
+        async getReferrersOfResource(id) {
+            const res = await this.#instance
+                .select('pages.url')
+                .from('resources-referrers')
+                .join('resources', 'resources.id', '=', 'resources-referrers.resourceId')
+                .join('pages', 'pages.id', '=', 'resources-referrers.pageId')
+                .where('resources.id', id);
+            return res.map((r) => r.url);
+        }
+        /**
+         * Retrieves all sub-resources from the `resources` table.
+         * @returns An array of raw {@link DB_Resource} rows.
+         */
+        async getResources() {
+            return this.#instance.select('*').from('resources');
+        }
+        /**
+         * Retrieves a flat list of all resource URLs from the `resources` table.
+         * @returns An array of resource URL strings.
+         */
+        async getResourceUrlList() {
+            const res = await this.#instance.select('url').from('resources');
+            return res.map((r) => r.url);
+        }
+        /**
+         * Inserts a sub-resource into the `resources` table.
+         * Ignores duplicate URLs (uses `ON CONFLICT IGNORE`).
+         * @param resource - The resource data to insert.
+         */
+        async insertResource(resource) {
+            await this.#instance
+                .from('resources')
+                .insert({
+                url: resource.url.href,
+                isExternal: resource.isExternal ? 1 : 0,
+                status: resource.status,
+                statusText: resource.statusText,
+                contentType: resource.contentType,
+                contentLength: resource.contentLength,
+                compress: resource.compress || 0,
+                cdn: resource.cdn || 0,
+                responseHeaders: JSON.stringify(resource.headers),
+            })
+                .onConflict('url')
+                .ignore();
+        }
+        /**
+         * Inserts a referrer relationship between a resource and a page into the
+         * `resources-referrers` table. Silently skips if the resource is not found.
+         * @param src - The URL of the resource.
+         * @param pageUrl - The URL of the page that references the resource.
+         */
+        async insertResourceReferrers(src, pageUrl) {
+            const selected = await this.#instance
+                .select('id')
+                .from('resources')
+                .where('url', src);
+            if (!selected[0]) {
+                // Ignore when the resource is not found
+                return;
+            }
+            const [{ id: resourceId }] = selected;
+            const pageId = await this.#getIdByUrl(pageUrl);
+            await this.#instance('resources-referrers').insert({
+                resourceId,
+                pageId,
+            });
+        }
+        /**
+         * Stores the crawl configuration in the `info` table.
+         * Serializes array fields (`excludes`, `excludeKeywords`, `scope`) as JSON strings.
+         * @param config - The {@link Config} object to store.
+         */
+        async setConfig(config) {
+            return this.#instance.from('info').insert({
+                ...config,
+                // @ts-expect-error
+                excludes: JSON.stringify(config.excludes),
+                // @ts-expect-error
+                excludeKeywords: JSON.stringify(config.excludeKeywords),
+                // @ts-expect-error
+                excludeUrls: JSON.stringify(config.excludeUrls),
+                // @ts-expect-error
+                scope: JSON.stringify(config.scope),
+            });
+        }
+        /**
+         * Marks a page as skipped in the database with the given reason.
+         * Creates the page row if it does not already exist.
+         * @param url - The URL of the skipped page.
+         * @param reason - The reason the page was skipped.
+         * @param isExternal - Whether the page is on an external domain. Defaults to `false`.
+         */
+        async setSkippedPage(url, reason, isExternal = false) {
+            const pageId = await this.#getIdByUrl(url, isExternal ? 1 : 0);
+            await this.#instance('pages')
+                .where('id', pageId)
+                .update({
+                scraped: 1,
+                isExternal: isExternal ? 1 : 0,
+                isSkipped: 1,
+                skipReason: reason,
+            });
+        }
+        /**
+         * Assigns natural URL sort order values to all internal pages.
+         * Pages are sorted using {@link pathComparator} and assigned sequential order numbers.
+         */
+        async setUrlOrder() {
+            dbLog('Set URL Order');
+            const res = await this.#instance
+                .select('id', 'url')
+                .from('pages')
+                .where('isExternal', '=', 0);
+            const sorted = res.toSorted((a, b) => pathComparator(a.url, b.url));
+            // Batch update using chunked CASE statements to avoid N+1 queries
+            const BATCH_SIZE = 500;
+            for (let i = 0; i < sorted.length; i += BATCH_SIZE) {
+                const batch = sorted.slice(i, i + BATCH_SIZE);
+                const ids = batch.map((row) => row.id);
+                const bindings = [];
+                const cases = batch
+                    .map((row, j) => {
+                    bindings.push(row.id, i + j + 1);
+                    return 'WHEN ? THEN ?';
+                })
+                    .join(' ');
+                const placeholders = ids.map(() => '?').join(',');
+                await this.#instance.raw(`UPDATE pages SET \`order\` = CASE id ${cases} END WHERE id IN (${placeholders})`, [...bindings, ...ids]);
+            }
+        }
+        /**
+         * Inserts or updates a crawled page in the database, including its redirect chain,
+         * anchors, and images. Optionally creates an HTML snapshot file path entry.
+         * @param page - The page data to store.
+         * @param snapshotDir - The directory for saving HTML snapshots, or null to skip snapshots.
+         * @param isTarget - Whether this page is a crawl target.
+         * @returns An object with the optional `html` snapshot file path and the page's database `pageId`.
+         */
+        async updatePage(page, snapshotDir, isTarget) {
+            let destUrl = page.url.withoutHashAndAuth;
+            const redirectPaths = [...page.redirectPaths];
+            if (redirectPaths.length > 0) {
+                destUrl = redirectPaths.pop();
+                redirectPaths.unshift(page.url.withoutHashAndAuth);
+            }
+            const destUrlObject = parseUrl(destUrl);
+            if (!destUrlObject) {
+                throw new Error(`Failed to parse URL: ${destUrl}`);
+            }
+            return await this.#instance.transaction(async (trx) => {
+                const pageId = await this.#insertPage({
+                    ...page,
+                    url: destUrlObject,
+                }, isTarget, trx);
+                for (const redirect of redirectPaths) {
+                    dbLog('Set redirected url: %s -> %s', redirect, destUrl);
+                    const redirectId = await this.#getIdByUrl(redirect, undefined, trx);
+                    await trx('pages')
+                        .where('id', redirectId)
+                        .update({
+                        scraped: 1,
+                        redirectDestId: pageId,
+                        isExternal: page.isExternal ? 1 : 0,
+                    });
+                }
+                let snapshot = { pageId };
+                if (isTarget && snapshotDir) {
+                    snapshot = await this.#updateSnapshotPath(pageId, snapshotDir, trx);
+                }
+                const anchors = await Promise.all(page.anchorList.map(async (anchor) => {
+                    const hrefId = await this.#getIdByUrl(anchor.href.withoutHashAndAuth, anchor.isExternal ? 1 : 0, trx);
+                    return {
+                        pageId,
+                        hrefId,
+                        hash: anchor.href.hash,
+                        textContent: anchor.textContent,
+                    };
+                }));
+                dbLog('Insert anchors.length: %d', anchors.length);
+                if (anchors.length > 0) {
+                    await eachSplitted(anchors, 100, async (_anchors) => {
+                        await trx('anchors').insert(_anchors);
+                    });
+                }
+                const images = page.imageList.map((image) => ({
+                    pageId,
+                    ...image,
+                }));
+                dbLog('Insert images.length: %d', images.length);
+                if (images.length > 0) {
+                    await eachSplitted(images, 100, async (_images) => {
+                        await trx('images').insert(_images);
+                    });
+                }
+                return snapshot;
+            });
+        }
+        /**
+         * Returns the database ID for a URL, creating a new page row if needed.
+         * Uses `ON CONFLICT IGNORE` to handle race conditions in concurrent inserts.
+         * @param url
+         * @param isExternal
+         * @param trx
+         */
+        async #getIdByUrl(url, isExternal, trx) {
+            const qb = trx ?? this.#instance;
+            const [record] = await qb.select('id').from('pages').where('url', url);
+            // Must use `?` because it may be `undefined`
+            const pageId = record?.id ?? Number.NaN;
+            if (Number.isFinite(pageId)) {
+                return pageId;
+            }
+            const insertedRows = await qb('pages')
+                .insert({
+                url,
+                scraped: 0,
+                isTarget: 0,
+                ...(isExternal != null && { isExternal }),
+            })
+                .onConflict('url')
+                .ignore();
+            const [insertedId] = insertedRows;
+            if (!insertedId) {
+                // onConflict.ignore() returns 0 on race condition — re-select
+                const [existing] = await qb.select('id').from('pages').where('url', url);
+                if (existing?.id) {
+                    return existing.id;
+                }
+                throw new Error(`Failed to insert a new page: ${url}`);
+            }
+            return insertedId;
+        }
+        /**
+         * Initializes the database schema if tables do not exist.
+         * Enables WAL journal mode and foreign keys, then creates all tables
+         * (`info`, `pages`, `anchors`, `images`, `resources`, `resources-referrers`).
+         */
+        async #init() {
+            const isExists = await this.#instance.schema.hasTable('info');
+            if (isExists) {
+                return;
+            }
+            // Enable WAL mode and foreign keys for better performance and data integrity
+            await this.#instance.raw('PRAGMA journal_mode = WAL');
+            await this.#instance.raw('PRAGMA foreign_keys = ON');
+            await this.#instance.schema
+                .createTable('info', (t) => {
+                t.increments('id');
+                t.string('version');
+                t.string('name');
+                t.string('baseUrl');
+                t.boolean('recursive');
+                t.boolean('useSubprocess');
+                t.integer('interval');
+                t.boolean('image');
+                t.boolean('fetchExternal');
+                t.integer('parallels');
+                t.json('scope');
+                t.json('excludes');
+                t.json('excludeKeywords');
+                t.json('excludeUrls');
+                t.integer('maxExcludedDepth');
+                t.integer('retry');
+                t.boolean('fromList');
+                t.boolean('disableQueries');
+            })
+                .createTable('pages', (t) => {
+                t.increments('id');
+                t.string('url', 8190).notNullable().unique();
+                t.integer('redirectDestId').unsigned().references('pages.id').defaultTo(null);
+                t.boolean('scraped').notNullable();
+                t.boolean('isTarget').notNullable();
+                t.boolean('isExternal');
+                t.integer('status');
+                t.string('statusText');
+                t.string('contentType').nullable();
+                t.integer('contentLength').unsigned().nullable();
+                t.json('responseHeaders').nullable();
+                t.string('lang');
+                t.string('title');
+                t.string('description');
+                t.string('keywords');
+                t.boolean('noindex');
+                t.boolean('nofollow');
+                t.boolean('noarchive');
+                t.string('canonical');
+                t.string('alternate');
+                t.string('og_type');
+                t.string('og_title');
+                t.string('og_site_name');
+                t.string('og_description');
+                t.string('og_url');
+                t.string('og_image');
+                t.string('twitter_card');
+                t.string('html');
+                t.boolean('isSkipped');
+                t.string('skipReason');
+                t.integer('order').unsigned().nullable();
+                t.index('isExternal');
+                t.index('contentType');
+                t.index('scraped');
+                t.index('redirectDestId');
+                t.index('order');
+            })
+                .createTable('anchors', (t) => {
+                t.increments('id');
+                t.integer('pageId').notNullable().unsigned().references('pages.id');
+                t.integer('hrefId').notNullable().unsigned().references('pages.id');
+                t.string('hash');
+                t.string('textContent').nullable();
+                t.index('pageId');
+                t.index('hrefId');
+            })
+                .createTable('images', (t) => {
+                t.increments('id');
+                t.integer('pageId').notNullable().unsigned().references('pages.id');
+                t.string('src', 8190);
+                t.string('currentSrc', 8190);
+                t.string('alt');
+                t.float('width').unsigned().notNullable();
+                t.float('height').unsigned().notNullable();
+                t.integer('naturalWidth').unsigned().notNullable();
+                t.integer('naturalHeight').unsigned().notNullable();
+                t.boolean('isLazy');
+                t.integer('viewportWidth').unsigned().notNullable();
+                t.string('sourceCode');
+                t.index('pageId');
+            })
+                .createTable('resources', (t) => {
+                t.increments('id');
+                t.string('url', 8190).notNullable().unique();
+                t.boolean('isExternal');
+                t.integer('status');
+                t.string('statusText');
+                t.string('contentType').nullable();
+                t.integer('contentLength').unsigned().nullable();
+                t.string('compress').nullable();
+                t.string('cdn').nullable();
+                t.json('responseHeaders').nullable();
+            })
+                .createTable('resources-referrers', (t) => {
+                t.increments('id');
+                t.integer('resourceId').notNullable().unsigned().references('resources.id');
+                t.integer('pageId').notNullable().unsigned().references('pages.id');
+                t.unique(['resourceId', 'pageId']);
+                t.index('resourceId');
+                t.index('pageId');
+            });
+        }
+        /**
+         * Upserts page data into the `pages` table (inserts if new, updates if existing).
+         * @param page
+         * @param isTarget
+         * @param trx
+         */
+        async #insertPage(page, isTarget, trx) {
+            const qb = trx ?? this.#instance;
+            const pageId = await this.#getIdByUrl(page.url.withoutHashAndAuth, undefined, trx);
+            await qb('pages')
+                .where('id', pageId)
+                .update({
+                scraped: true,
+                isTarget,
+                isExternal: page.isExternal,
+                status: page.status,
+                statusText: page.statusText,
+                contentType: page.contentType,
+                contentLength: page.contentLength,
+                responseHeaders: JSON.stringify(page.responseHeaders),
+                lang: page.meta.lang,
+                title: page.meta.title,
+                description: page.meta.description,
+                keywords: page.meta.keywords,
+                noindex: page.meta.noindex,
+                nofollow: page.meta.nofollow,
+                noarchive: page.meta.noarchive,
+                canonical: page.meta.canonical,
+                alternate: page.meta.alternate,
+                og_type: page.meta['og:type'],
+                og_title: page.meta['og:title'],
+                og_site_name: page.meta['og:site_name'],
+                og_description: page.meta['og:description'],
+                og_url: page.meta['og:url'],
+                og_image: page.meta['og:image'],
+                twitter_card: page.meta['twitter:card'],
+                isSkipped: page.isSkipped,
+            });
+            return pageId;
+        }
+        /**
+         * Assigns and persists the HTML snapshot file path for a page.
+         * @param pageId
+         * @param snapshotDir
+         * @param trx
+         */
+        async #updateSnapshotPath(pageId, snapshotDir, trx) {
+            const qb = trx ?? this.#instance;
+            const snapshotHtmlPath = path.resolve(snapshotDir, `${pageId}.html`);
+            const snapshotRelHtmlPath = path.relative(this.#workingDir, snapshotHtmlPath);
+            await qb('pages').where('id', pageId).update({
+                html: snapshotRelHtmlPath,
+            });
+            return {
+                html: snapshotHtmlPath,
+                pageId,
+            };
+        }
+        /**
+         * Creates and initializes a new Database instance.
+         * Creates the parent directory for the database file if needed,
+         * establishes the connection, and initializes tables if they do not exist.
+         * @param options - The database connection options specifying the type and file path.
+         * @returns A fully initialized Database instance.
+         */
+        static async connect(options) {
+            switch (options.type) {
+                case 'sqlite3': {
+                    mkdir(options.filename);
+                    break;
+                }
+            }
+            const db = new Database(options);
+            await db.#init();
+            return db;
+        }
+    };
+})();
+export { Database };
+// ----- ----- ----- ----- -----
+//
+// Common Queries
+//
+// ----- ----- ----- ----- -----
+/**
+ * Returns a Knex subquery builder that selects page IDs with pagination,
+ * ordered by the `order` column (nulls last), excluding redirected pages.
+ * @param limit - The maximum number of page IDs to return.
+ * @param offset - The number of page IDs to skip before returning results.
+ */
+function limitedPageIds(limit, offset) {
+    return async (qb) => {
+        await qb
+            .select('id')
+            .from('pages')
+            .orderByRaw('`order` ASC NULLS LAST')
+            .whereNull('redirectDestId')
+            .limit(limit)
+            .offset(offset);
+    };
+}
+/**
+ * Returns a Knex subquery builder that joins pages with their redirect destinations.
+ * When `includeNull` is true, also includes pages without redirects (self-referencing).
+ * @param includeNull - Whether to include non-redirected pages in the result. Defaults to `true`.
+ */
+function redirectTable(includeNull = true) {
+    return async (qb) => {
+        const list = qb
+            .select('A.id as fromId', 'A.url as from', 'B.url as to', 'B.id as toId')
+            .from('pages as A')
+            .join('pages as B', (j) => {
+            j.on('A.redirectDestId', '=', 'B.id').andOnNotNull('A.redirectDestId');
+        });
+        if (includeNull) {
+            await list.union(async (qb) => {
+                await qb
+                    .select('A.id as fromId', 'A.url as from', 'A.url as to', 'A.id as toId')
+                    .from('pages as A')
+                    .whereNull('A.redirectDestId');
+            });
+        }
+    };
+}
+// ----- ----- ----- ----- -----
+//
+// Utils
+//
+// ----- ----- ----- ----- -----
+/**
+ * Safely parses a JSON string, returning a fallback value if parsing fails or the input is not a string.
+ * @param data - The data to parse. Only string values are parsed; other types return the fallback.
+ * @param fallback - The value to return if parsing fails or the result is falsy.
+ * @returns The parsed JSON value, or the fallback.
+ */
+function getJSON(data, fallback) {
+    try {
+        if (typeof data === 'string') {
+            const result = JSON.parse(data);
+            if (result) {
+                return result;
+            }
+            return fallback;
+        }
+    }
+    catch {
+        // void
+    }
+    return fallback;
+}