@nitpicker/crawler 0.7.0 → 0.8.0

package/lib/archive/database.js

@@ -38,18 +38,57 @@ import { retry } from '@d-zero/shared/retry';
 import { pathComparator } from '@d-zero/shared/sort/path';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import knex from 'knex';
+import { findScopeEntry } from '../crawler/find-scope-entry.js';
 import { eachSplitted } from '../utils/array/each-splitted.js';
 import { ErrorEmitter } from '../utils/error/error-emitter.js';
 import { dbLog } from './debug.js';
 import { mkdir } from './filesystem/mkdir.js';
 import { getJSON } from './get-json.js';
 import { initSchema } from './init-schema.js';
+import { LibsqlDialect } from './libsql-dialect.js';
 import { limitedPageIds } from './limited-page-ids.js';
+import { migrateInfoRoots } from './migrate-info-roots.js';
 import { redirectTable } from './redirect-table.js';
 const retrySetting = {
     interval: 300,
     retries: 3,
 };
+/**
+ * Columns of the `info` table that `setConfig` / `updateConfig` are allowed to
+ * write. Any key outside this set is silently dropped so callers can splat a
+ * wider runtime config (with extras like `cwd`) without hitting "no such
+ * column" at the SQL layer.
+ */
+const INFO_COLUMN_ALLOWLIST = new Set([
+    'version',
+    'name',
+    'baseUrl',
+    'roots',
+    'recursive',
+    'interval',
+    'image',
+    'fetchExternal',
+    'parallels',
+    'excludes',
+    'excludeKeywords',
+    'excludeUrls',
+    'maxExcludedDepth',
+    'retry',
+    'fromList',
+    'disableQueries',
+    'userAgent',
+    'ignoreRobots',
+]);
+/**
+ * Subset of {@link INFO_COLUMN_ALLOWLIST} that is stored as a JSON-encoded
+ * string and therefore needs `JSON.stringify` on write.
+ */
+const INFO_JSON_COLUMNS = new Set([
+    'roots',
+    'excludes',
+    'excludeKeywords',
+    'excludeUrls',
+]);
 /**
  * Low-level database abstraction layer for the archive's SQLite database.
  *
@@ -81,8 +120,10 @@ let Database = (() => {
     let _getResourceUrlList_decorators;
     let _insertResource_decorators;
     let _insertResourceReferrers_decorators;
+    let _repromoteExternalPages_decorators;
     let _setConfig_decorators;
     let _setSkippedPage_decorators;
+    let _updateConfig_decorators;
     let _updatePage_decorators;
     return class Database extends _classSuper {
         static {
@@ -104,8 +145,10 @@ let Database = (() => {
             _getResourceUrlList_decorators = [ErrorEmitter(), retry(retrySetting)];
             _insertResource_decorators = [ErrorEmitter(), retry(retrySetting)];
             _insertResourceReferrers_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _repromoteExternalPages_decorators = [ErrorEmitter(), retry(retrySetting)];
             _setConfig_decorators = [ErrorEmitter(), retry(retrySetting)];
             _setSkippedPage_decorators = [ErrorEmitter(), retry(retrySetting)];
+            _updateConfig_decorators = [ErrorEmitter(), retry(retrySetting)];
             _updatePage_decorators = [ErrorEmitter(), retry(retrySetting)];
             __esDecorate(this, null, _clearHtmlPath_decorators, { kind: "method", name: "clearHtmlPath", static: false, private: false, access: { has: obj => "clearHtmlPath" in obj, get: obj => obj.clearHtmlPath }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _getAnchorsOnPage_decorators, { kind: "method", name: "getAnchorsOnPage", static: false, private: false, access: { has: obj => "getAnchorsOnPage" in obj, get: obj => obj.getAnchorsOnPage }, metadata: _metadata }, null, _instanceExtraInitializers);
@@ -124,8 +167,10 @@ let Database = (() => {
             __esDecorate(this, null, _getResourceUrlList_decorators, { kind: "method", name: "getResourceUrlList", static: false, private: false, access: { has: obj => "getResourceUrlList" in obj, get: obj => obj.getResourceUrlList }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _insertResource_decorators, { kind: "method", name: "insertResource", static: false, private: false, access: { has: obj => "insertResource" in obj, get: obj => obj.insertResource }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _insertResourceReferrers_decorators, { kind: "method", name: "insertResourceReferrers", static: false, private: false, access: { has: obj => "insertResourceReferrers" in obj, get: obj => obj.insertResourceReferrers }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _repromoteExternalPages_decorators, { kind: "method", name: "repromoteExternalPages", static: false, private: false, access: { has: obj => "repromoteExternalPages" in obj, get: obj => obj.repromoteExternalPages }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _setConfig_decorators, { kind: "method", name: "setConfig", static: false, private: false, access: { has: obj => "setConfig" in obj, get: obj => obj.setConfig }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _setSkippedPage_decorators, { kind: "method", name: "setSkippedPage", static: false, private: false, access: { has: obj => "setSkippedPage" in obj, get: obj => obj.setSkippedPage }, metadata: _metadata }, null, _instanceExtraInitializers);
+            __esDecorate(this, null, _updateConfig_decorators, { kind: "method", name: "updateConfig", static: false, private: false, access: { has: obj => "updateConfig" in obj, get: obj => obj.updateConfig }, metadata: _metadata }, null, _instanceExtraInitializers);
             __esDecorate(this, null, _updatePage_decorators, { kind: "method", name: "updatePage", static: false, private: false, access: { has: obj => "updatePage" in obj, get: obj => obj.updatePage }, metadata: _metadata }, null, _instanceExtraInitializers);
             if (_metadata) Object.defineProperty(this, Symbol.metadata, { enumerable: true, configurable: true, writable: true, value: _metadata });
         }
@@ -137,24 +182,16 @@ let Database = (() => {
         constructor(options) {
             super();
             this.#workingDir = options.workingDir;
-            switch (options.type) {
-                case 'sqlite3': {
-                    this.#instance = knex({
-                        client: options.type,
-                        connection: {
-                            filename: options.filename,
-                        },
-                        useNullAsDefault: true,
-                        pool: {
-                            acquireTimeoutMillis: 600_000,
-                        },
-                    });
-                    break;
-                }
-                case 'mysql': {
-                    throw new Error("Don't support MySQL yet.");
-                }
-            }
+            this.#instance = knex({
+                client: LibsqlDialect,
+                connection: {
+                    filename: options.filename,
+                },
+                useNullAsDefault: true,
+                pool: {
+                    acquireTimeoutMillis: 600_000,
+                },
+            });
         }
         /**
          * Adds the `order` column to the `pages` table for URL sort ordering.
@@ -222,7 +259,7 @@ let Database = (() => {
         }
         /**
          * Retrieves the full crawl configuration from the `info` table.
-         * Deserializes JSON-encoded fields (`excludes`, `excludeKeywords`, `scope`).
+         * Deserializes JSON-encoded fields (`roots`, `excludes`, `excludeKeywords`, `excludeUrls`).
          * @returns The parsed {@link Config} object.
          * @throws {Error} If no configuration is found in the database.
          */
@@ -236,10 +273,10 @@ let Database = (() => {
                 excludes: getJSON(config.excludes, []),
                 excludeKeywords: getJSON(config.excludeKeywords, []),
                 excludeUrls: getJSON(config.excludeUrls, []),
-                scope: getJSON(config.scope, []),
+                roots: getJSON(config.roots, []),
                 retry: config.retry ?? 3,
             };
-            // @ts-expect-error
+            // @ts-expect-error — `id` is the primary key, not part of the public Config shape
             delete opt.id;
             dbLog('Table `info`: %O => %O', config, opt);
             return opt;
@@ -574,23 +611,88 @@ let Database = (() => {
                 .onConflict(['resourceId', 'pageId'])
                 .ignore();
         }
+        /**
+         * Promote previously-external pages whose URL falls under any of the new scope
+         * entries back to a "needs scraping" state so that the next crawl picks them up
+         * as full internal pages.
+         *
+         * For each matching page:
+         * - clears the scrape metadata (status, headers, snapshot path, etc.),
+         * - flips `isExternal` to `0` and `scraped` to `0`,
+         * - removes stale `anchors`, `images`, and `resources-referrers` rows so that
+         *   the re-scrape can re-insert fresh ones without duplicates.
+         *
+         * The page row itself is kept (id is preserved) so existing referrers via
+         * `anchors.hrefId` remain valid. SELECT and UPDATE/DELETE statements are
+         * chunked to stay below SQLite's `SQLITE_LIMIT_VARIABLE_NUMBER`.
+         * @param scopes - The hostname-indexed scope map after the new roots are merged.
+         * @param options - URL parsing options forwarded to {@link findScopeEntry}.
+         * @returns The URLs of the pages that were promoted.
+         */
+        async repromoteExternalPages(scopes, options) {
+            if (scopes.size === 0) {
+                return [];
+            }
+            const candidates = await this.#instance
+                .select('id', 'url')
+                .from('pages')
+                .where('isExternal', 1);
+            const promotedIds = [];
+            const promotedUrls = [];
+            for (const row of candidates) {
+                const parsed = parseUrl(row.url, options);
+                if (!parsed) {
+                    continue;
+                }
+                if (findScopeEntry(parsed, scopes, options) === null) {
+                    continue;
+                }
+                promotedIds.push(row.id);
+                promotedUrls.push(row.url);
+            }
+            if (promotedIds.length === 0) {
+                return [];
+            }
+            const chunkSize = 500;
+            for (let i = 0; i < promotedIds.length; i += chunkSize) {
+                const chunk = promotedIds.slice(i, i + chunkSize);
+                await this.#instance('pages').whereIn('id', chunk).update({
+                    scraped: 0,
+                    isExternal: 0,
+                    isSkipped: 0,
+                    skipReason: null,
+                    html: null,
+                    status: null,
+                    statusText: null,
+                    contentType: null,
+                    contentLength: null,
+                    responseHeaders: '{}',
+                    redirectDestId: null,
+                });
+                await this.#instance('anchors').whereIn('pageId', chunk).delete();
+                await this.#instance('images').whereIn('pageId', chunk).delete();
+                await this.#instance('resources-referrers').whereIn('pageId', chunk).delete();
+            }
+            dbLog('Repromoted %d external pages back to pending', promotedUrls.length);
+            return promotedUrls;
+        }
         /**
          * Stores the crawl configuration in the `info` table.
-         * Serializes array fields (`excludes`, `excludeKeywords`, `scope`) as JSON strings.
+         * Only fields in {@link INFO_COLUMN_ALLOWLIST} are forwarded — any extra
+         * runtime-only field on the input is silently dropped so callers can splat
+         * a wider config object without producing SQL errors. JSON-array fields
+         * are serialized via `JSON.stringify`.
          * @param config - The {@link Config} object to store.
          */
         async setConfig(config) {
-            return this.#instance.from('info').insert({
-                ...config,
-                // @ts-expect-error
-                excludes: JSON.stringify(config.excludes),
-                // @ts-expect-error
-                excludeKeywords: JSON.stringify(config.excludeKeywords),
-                // @ts-expect-error
-                excludeUrls: JSON.stringify(config.excludeUrls),
-                // @ts-expect-error
-                scope: JSON.stringify(config.scope),
-            });
+            const payload = {};
+            for (const [key, value] of Object.entries(config)) {
+                if (!INFO_COLUMN_ALLOWLIST.has(key)) {
+                    continue;
+                }
+                payload[key] = INFO_JSON_COLUMNS.has(key) ? JSON.stringify(value) : value;
+            }
+            return this.#instance.from('info').insert(payload);
         }
         /**
          * Marks a page as skipped in the database with the given reason.
@@ -637,6 +739,39 @@ let Database = (() => {
                 await this.#instance.raw(`UPDATE pages SET \`order\` = CASE id ${cases} END WHERE id IN (${placeholders})`, [...bindings, ...ids]);
             }
         }
+        /**
+         * Update the single row in the `info` table with a partial config patch.
+         *
+         * Used by the append flow to extend `roots` (and any other tweakable
+         * field) without replacing the entire row. JSON-array fields are serialized on
+         * the fly; primitive fields are written verbatim. Unspecified fields stay as-is.
+         *
+         * Unknown keys (anything outside the allow-list of `info`-table columns) are
+         * silently dropped instead of being passed to SQL, so callers that splat a
+         * wider runtime config (e.g. `CrawlConfig` with `cwd` / `executablePath`)
+         * cannot accidentally trigger a "no such column" SQL error.
+         * @param patch - Partial {@link Config} fields to overwrite. `undefined` values are skipped.
+         */
+        async updateConfig(patch) {
+            const payload = {};
+            for (const [key, value] of Object.entries(patch)) {
+                if (value === undefined) {
+                    continue;
+                }
+                if (!INFO_COLUMN_ALLOWLIST.has(key)) {
+                    continue;
+                }
+                if (INFO_JSON_COLUMNS.has(key)) {
+                    payload[key] = JSON.stringify(value);
+                    continue;
+                }
+                payload[key] = value;
+            }
+            if (Object.keys(payload).length === 0) {
+                return;
+            }
+            await this.#instance.from('info').update(payload);
+        }
         /**
          * Inserts or updates a crawled page in the database, including its redirect chain,
          * anchors, and images. Optionally creates an HTML snapshot file path entry.
@@ -750,11 +885,15 @@ let Database = (() => {
             return insertedId;
         }
         /**
-         * Initializes the database schema if tables do not exist.
-         * Delegates to {@link initSchema} for the actual table creation.
+         * Initializes the database schema if tables do not exist, then runs lightweight
+         * migrations that bring older archives up to the current schema.
+         *
+         * Migrations are idempotent and run on every {@link Database.connect}, so the
+         * same DB can be opened safely from both writer and reader code paths.
          */
         async #init() {
             await initSchema(this.#instance);
+            await migrateInfoRoots(this.#instance);
         }
         /**
          * Upserts page data into the `pages` table (inserts if new, updates if existing).
@@ -818,16 +957,11 @@ let Database = (() => {
          * Creates and initializes a new Database instance.
          * Creates the parent directory for the database file if needed,
          * establishes the connection, and initializes tables if they do not exist.
-         * @param options - The database connection options specifying the type and file path.
+         * @param options - Database connection options (working directory + SQLite file path).
          * @returns A fully initialized Database instance.
          */
         static async connect(options) {
-            switch (options.type) {
-                case 'sqlite3': {
-                    mkdir(options.filename);
-                    break;
-                }
-            }
+            mkdir(options.filename);
             const db = new Database(options);
             await db.#init();
             return db;

package/lib/archive/init-schema.js

@@ -18,12 +18,12 @@ export async function initSchema(instance) {
         t.string('version');
         t.string('name');
         t.string('baseUrl');
+        t.json('roots');
         t.boolean('recursive');
         t.integer('interval');
         t.boolean('image');
         t.boolean('fetchExternal');
         t.integer('parallels');
-        t.json('scope');
         t.json('excludes');
         t.json('excludeKeywords');
         t.json('excludeUrls');

package/lib/archive/libsql-dialect.d.ts

@@ -0,0 +1,25 @@
+import type { Knex } from 'knex';
+/**
+ * Untyped reference to Knex's internal `better-sqlite3` dialect.
+ *
+ * Knex's published `exports` do not include this subpath, so TypeScript cannot
+ * resolve a declaration file for it. The dialect is a concrete subclass of
+ * `Knex.Client` at runtime, so the safe cast is intentional.
+ */
+declare const Base: typeof Knex.Client;
+/**
+ * Knex dialect that reuses the `better-sqlite3` adapter logic but swaps the
+ * underlying driver for `libsql`. `libsql` ships pre-compiled binaries via the
+ * `@libsql/<platform>-<arch>` optionalDependencies pattern, so no postinstall
+ * download from GitHub Releases is required.
+ */
+export declare class LibsqlDialect extends Base {
+    /**
+     * Returns the `libsql` constructor as the SQLite driver.
+     *
+     * Typed as `unknown` to avoid leaking the `Libsql.DatabaseConstructor`
+     * namespace from a non-re-exported module across the public API surface.
+     */
+    protected _driver(): unknown;
+}
+export {};

package/lib/archive/libsql-dialect.js

@@ -0,0 +1,28 @@
+// @ts-expect-error - Internal Knex subpath without published type declarations
+import Client_BetterSQLite3 from 'knex/lib/dialects/better-sqlite3/index.js';
+import libsql from 'libsql';
+/**
+ * Untyped reference to Knex's internal `better-sqlite3` dialect.
+ *
+ * Knex's published `exports` do not include this subpath, so TypeScript cannot
+ * resolve a declaration file for it. The dialect is a concrete subclass of
+ * `Knex.Client` at runtime, so the safe cast is intentional.
+ */
+const Base = Client_BetterSQLite3;
+/**
+ * Knex dialect that reuses the `better-sqlite3` adapter logic but swaps the
+ * underlying driver for `libsql`. `libsql` ships pre-compiled binaries via the
+ * `@libsql/<platform>-<arch>` optionalDependencies pattern, so no postinstall
+ * download from GitHub Releases is required.
+ */
+export class LibsqlDialect extends Base {
+    /**
+     * Returns the `libsql` constructor as the SQLite driver.
+     *
+     * Typed as `unknown` to avoid leaking the `Libsql.DatabaseConstructor`
+     * namespace from a non-re-exported module across the public API surface.
+     */
+    _driver() {
+        return libsql;
+    }
+}

package/lib/archive/migrate-info-roots.d.ts

@@ -0,0 +1,15 @@
+import type { Knex } from 'knex';
+/**
+ * Bring an archive's `info` table to the current shape: add `roots` (seeded
+ * from `baseUrl`) and drop the obsolete `scope` column.
+ *
+ * Idempotent: calling this multiple times on an up-to-date schema is a no-op.
+ * Archives where `baseUrl` is NULL receive an empty `roots` array rather than
+ * throwing.
+ *
+ * When the migration actually runs (i.e. either column transition was
+ * performed), a single notice is written to stderr so the user knows the
+ * file was upgraded.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export declare function migrateInfoRoots(instance: Knex): Promise<void>;

package/lib/archive/migrate-info-roots.js

@@ -0,0 +1,41 @@
+/**
+ * Bring an archive's `info` table to the current shape: add `roots` (seeded
+ * from `baseUrl`) and drop the obsolete `scope` column.
+ *
+ * Idempotent: calling this multiple times on an up-to-date schema is a no-op.
+ * Archives where `baseUrl` is NULL receive an empty `roots` array rather than
+ * throwing.
+ *
+ * When the migration actually runs (i.e. either column transition was
+ * performed), a single notice is written to stderr so the user knows the
+ * file was upgraded.
+ * @param instance - The Knex query builder instance connected to the database.
+ */
+export async function migrateInfoRoots(instance) {
+    const hasInfo = await instance.schema.hasTable('info');
+    if (!hasInfo) {
+        return;
+    }
+    const hasRoots = await instance.schema.hasColumn('info', 'roots');
+    const hasScope = await instance.schema.hasColumn('info', 'scope');
+    if (hasRoots && !hasScope) {
+        return;
+    }
+    const changes = [];
+    if (!hasRoots) {
+        await instance.schema.table('info', (t) => {
+            t.json('roots');
+        });
+        await instance.raw(`UPDATE info SET roots = json_array(baseUrl) WHERE roots IS NULL AND baseUrl IS NOT NULL`);
+        await instance.raw(`UPDATE info SET roots = '[]' WHERE roots IS NULL`);
+        changes.push('roots seeded');
+    }
+    if (hasScope) {
+        await instance.schema.table('info', (t) => {
+            t.dropColumn('scope');
+        });
+        changes.push('scope dropped');
+    }
+    // eslint-disable-next-line no-console
+    console.error(`[migrate] info table upgraded (${changes.join(', ')})`);
+}

package/lib/archive/types.d.ts

@@ -11,12 +11,12 @@ export interface DatabaseEvent {
  * Represents all crawling options that were used for the crawl session.
  */
 export interface Config extends Required<Pick<ParseURLOptions, 'disableQueries'>> {
-    /** The starting URL for the crawl. */
+    /** The starting URL for the crawl. Stored as a denormalised mirror of `roots[0]` so summary consumers can read a single URL without parsing the array. */
     baseUrl: string;
+    /** The user-provided root URLs that seeded the crawl. Each root is both a recursive starting point and a scope entry. Always non-empty. */
+    roots: string[];
     /** Maximum directory depth for excluded paths. */
     maxExcludedDepth: number;
-    /** URL patterns defining the crawl scope. */
-    scope: string[];
     /** Keywords used to exclude pages from crawling. */
     excludeKeywords: string[];
     /** URL patterns to exclude from crawling. */
@@ -285,31 +285,11 @@ export interface DB_Resource {
     responseHeaders: string | null;
 }
 /**
- * Base options shared by all database connection configurations.
+ * Connection options for the archive's libsql-backed database.
  */
-type AbsDatabaseOption = {
+export interface DatabaseOption {
     /** The working directory for the database (used for resolving relative paths). */
     workingDir: string;
-};
-/**
- * Union type for all supported database connection options.
- */
-export type DatabaseOption = DatabaseSqlite3Option | DatabaseMySqlOption;
-/**
- * Connection options for a SQLite3 database.
- */
-type DatabaseSqlite3Option = AbsDatabaseOption & {
-    /** The database type identifier. */
-    type: 'sqlite3';
     /** The absolute file path to the SQLite database file. */
     filename: string;
-};
-/**
- * Connection options for a MySQL database.
- * Note: MySQL support is not yet implemented.
- */
-type DatabaseMySqlOption = AbsDatabaseOption & {
-    /** The database type identifier. */
-    type: 'mysql';
-};
-export {};
+}

package/lib/crawler/crawler.d.ts

@@ -56,23 +56,27 @@ export default class Crawler extends EventEmitter<CrawlerEventTypes> {
      */
     resume(pending: string[], scraped: string[], resources: string[]): void;
     /**
-     * Start crawling from a single root URL.
+     * Start crawling from one or more root URLs.
      *
-     * Adds the root URL to the scope (if not already present) and the link list,
-     * then begins the deal-based concurrent crawl. Discovered child pages are
-     * automatically added to the queue when recursive mode is enabled.
-     * @param url - The root URL to begin crawling from.
-     */
-    start(url: ExURL): void;
-    /**
-     * Start crawling a pre-defined list of URLs in non-recursive mode.
+     * Each URL is registered as a scope entry (if not already present) and added
+     * to the link list. When `opts.recursive` is `false`, recursion is disabled
+     * and the crawler behaves like the former `startMultiple` (list mode);
+     * otherwise discovered child pages within the scope are followed.
      *
-     * Each URL in the list is added to the scope and the link list. Recursive
-     * crawling is disabled; only the provided URLs will be scraped.
-     * @param pageList - The list of URLs to crawl. Must contain at least one URL.
-     * @throws {Error} If the page list is empty.
+     * When resume state is present, the resumed pending URLs are merged with the
+     * newly-provided roots. The merge is deduplicated by protocol-agnostic key
+     * before reaching the dealer so a URL that exists in both sources — which
+     * is common in append-mode when a new root coincides with a repromoted
+     * previously-external page — does not race on two parallel slots.
+     * @param urls - The list of root URLs to begin crawling from. Must be non-empty.
+     * @param opts - Optional overrides; currently only `recursive` is honoured.
+     * @param opts.recursive - When `false`, disables recursive discovery and forces list-mode.
+     *   Defaults to the constructor option's `recursive` value.
+     * @throws {Error} If the URL list is empty.
      */
-    startMultiple(pageList: ExURL[]): void;
+    start(urls: ExURL[], opts?: {
+        recursive?: boolean;
+    }): void;
     /**
      * The default maximum number of concurrent scraping processes.
      *