@nitpicker/crawler 0.4.1

package/lib/archive/database.d.ts

@@ -0,0 +1,207 @@
+import type { Config, DB_Anchor, DB_Page, DB_Redirect, DB_Referrer, DB_Resource, DatabaseEvent, PageFilter } from './types.js';
+import type { PageData, Resource } from '../utils/index.js';
+import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
+/**
+ * Low-level database abstraction layer for the archive's SQLite database.
+ *
+ * Manages the `pages`, `anchors`, `images`, `resources`, and `resources-referrers`
+ * tables. All public methods that perform database queries use the `@retryable`
+ * decorator for automatic retry on transient failures, and `@ErrorEmitter` to
+ * propagate errors as events.
+ *
+ * Use the static {@link Database.connect} factory method to create instances.
+ * The constructor is private.
+ */
+export declare class Database extends EventEmitter<DatabaseEvent> {
+    #private;
+    private constructor();
+    /**
+     * Adds the `order` column to the `pages` table for URL sort ordering.
+     * @deprecated Since v0.1.x. The column is now created during table initialization.
+     * @returns The result of the schema alteration.
+     */
+    addOrderField(): Promise<void>;
+    /**
+     * Forces a WAL checkpoint, writing all pending WAL data back to the main database file.
+     * Uses TRUNCATE mode to reset the WAL file to zero bytes after checkpointing.
+     * This ensures the database is fully self-contained in `db.sqlite` before archiving.
+     */
+    checkpoint(): Promise<void>;
+    destroy(): Promise<void>;
+    /**
+     * Retrieves all anchors (outgoing links) on a specific page.
+     * Joins the `anchors` table with the `pages` table to resolve link destinations.
+     * @param pageId - The database ID of the page whose anchors to retrieve.
+     * @returns An array of anchor records with resolved URL, title, status, and content type.
+     */
+    getAnchorsOnPage(pageId: number): Promise<any[]>;
+    /**
+     * Retrieves the base URL of the crawl session from the `info` table.
+     * @returns The base URL string.
+     * @throws {Error} If no base URL is found in the database.
+     */
+    getBaseUrl(): Promise<any>;
+    /**
+     * Retrieves the full crawl configuration from the `info` table.
+     * Deserializes JSON-encoded fields (`excludes`, `excludeKeywords`, `scope`).
+     * @returns The parsed {@link Config} object.
+     * @throws {Error} If no configuration is found in the database.
+     */
+    getConfig(): Promise<Config>;
+    /**
+     * Retrieves the current crawling state by listing scraped and pending URLs.
+     * @returns An object with `scraped` (completed URLs) and `pending` (remaining URLs) arrays.
+     */
+    getCrawlingState(): Promise<{
+        scraped: string[];
+        pending: string[];
+    }>;
+    /**
+     * Retrieves the HTML snapshot file path for a specific page.
+     * @param pageId - The database ID of the page.
+     * @returns The relative file path to the HTML snapshot, or null if not saved.
+     */
+    getHtmlPathOnPage(pageId: number): Promise<any>;
+    /**
+     * Retrieves the crawl session name from the `info` table.
+     * @returns The name string.
+     * @throws {Error} If no name is found in the database.
+     */
+    getName(): Promise<any>;
+    /**
+     * Counts the total number of pages in the database.
+     * @returns The total page count.
+     * @throws {Error} If the count query fails.
+     */
+    getPageCount(): Promise<number>;
+    /**
+     * Retrieves pages from the database with optional filtering, pagination via offset and limit.
+     * @param filter - An optional {@link PageFilter} to narrow results by content type and origin.
+     * @param offset - The number of rows to skip. Defaults to `0`.
+     * @param limit - The maximum number of rows to return. Defaults to `100000`.
+     * @returns An array of raw {@link DB_Page} rows.
+     */
+    getPages(filter?: PageFilter, offset?: number, limit?: number): Promise<DB_Page[]>;
+    /**
+     * Retrieves pages along with their related redirect, anchor, and referrer data.
+     * Results are ordered by the natural URL sort order. Only non-redirected pages are returned.
+     * @param offset - The number of rows to skip.
+     * @param limit - The maximum number of pages to return.
+     * @returns An object containing `pages`, `redirects`, `anchors`, and `referrers` arrays.
+     */
+    getPagesWithRels(offset: number, limit: number): Promise<{
+        pages: DB_Page[];
+        redirects: DB_Redirect[];
+        anchors: DB_Anchor[];
+        referrers: DB_Referrer[];
+    }>;
+    /**
+     * Retrieves redirect sources for the given page IDs in bulk.
+     * @param pageIds - The database IDs of the destination pages.
+     * @returns An array of {@link DB_Redirect} records mapping destination pages to their redirect sources.
+     */
+    getRedirectsForPages(pageIds: number[]): Promise<DB_Redirect[]>;
+    /**
+     * Retrieves pages that link to a specific page (incoming links / referrers).
+     * @param pageId - The database ID of the target page.
+     * @returns An array of referrer records with URL, hash, and text content.
+     */
+    getReferrersOfPage(pageId: number): Promise<any[]>;
+    /**
+     * Retrieves the page URLs that reference a specific resource.
+     * @param id - The database ID of the resource.
+     * @returns An array of page URL strings that reference the resource.
+     */
+    getReferrersOfResource(id: number): Promise<string[]>;
+    /**
+     * Retrieves all sub-resources from the `resources` table.
+     * @returns An array of raw {@link DB_Resource} rows.
+     */
+    getResources(): Promise<DB_Resource[]>;
+    /**
+     * Retrieves a flat list of all resource URLs from the `resources` table.
+     * @returns An array of resource URL strings.
+     */
+    getResourceUrlList(): Promise<any[]>;
+    /**
+     * Inserts a sub-resource into the `resources` table.
+     * Ignores duplicate URLs (uses `ON CONFLICT IGNORE`).
+     * @param resource - The resource data to insert.
+     */
+    insertResource(resource: Resource): Promise<void>;
+    /**
+     * Inserts a referrer relationship between a resource and a page into the
+     * `resources-referrers` table. Silently skips if the resource is not found.
+     * @param src - The URL of the resource.
+     * @param pageUrl - The URL of the page that references the resource.
+     */
+    insertResourceReferrers(src: string, pageUrl: string): Promise<void>;
+    /**
+     * Stores the crawl configuration in the `info` table.
+     * Serializes array fields (`excludes`, `excludeKeywords`, `scope`) as JSON strings.
+     * @param config - The {@link Config} object to store.
+     */
+    setConfig(config: Config): Promise<Config[]>;
+    /**
+     * Marks a page as skipped in the database with the given reason.
+     * Creates the page row if it does not already exist.
+     * @param url - The URL of the skipped page.
+     * @param reason - The reason the page was skipped.
+     * @param isExternal - Whether the page is on an external domain. Defaults to `false`.
+     */
+    setSkippedPage(url: string, reason: string, isExternal?: boolean): Promise<void>;
+    /**
+     * Assigns natural URL sort order values to all internal pages.
+     * Pages are sorted using {@link pathComparator} and assigned sequential order numbers.
+     */
+    setUrlOrder(): Promise<void>;
+    /**
+     * Inserts or updates a crawled page in the database, including its redirect chain,
+     * anchors, and images. Optionally creates an HTML snapshot file path entry.
+     * @param page - The page data to store.
+     * @param snapshotDir - The directory for saving HTML snapshots, or null to skip snapshots.
+     * @param isTarget - Whether this page is a crawl target.
+     * @returns An object with the optional `html` snapshot file path and the page's database `pageId`.
+     */
+    updatePage(page: PageData, snapshotDir: string | null, isTarget: boolean): Promise<{
+        html?: string | undefined;
+        pageId: number;
+    }>;
+    /**
+     * Creates and initializes a new Database instance.
+     * Creates the parent directory for the database file if needed,
+     * establishes the connection, and initializes tables if they do not exist.
+     * @param options - The database connection options specifying the type and file path.
+     * @returns A fully initialized Database instance.
+     */
+    static connect(options: DatabaseOption): Promise<Database>;
+}
+/**
+ * Base options shared by all database connection configurations.
+ */
+type AbsDatabaseOption = {
+    /** The working directory for the database (used for resolving relative paths). */
+    workingDir: string;
+};
+/**
+ * Union type for all supported database connection options.
+ */
+type DatabaseOption = DatabaseSqlite3Option | DatabaseMySqlOption;
+/**
+ * Connection options for a SQLite3 database.
+ */
+type DatabaseSqlite3Option = AbsDatabaseOption & {
+    /** The database type identifier. */
+    type: 'sqlite3';
+    /** The absolute file path to the SQLite database file. */
+    filename: string;
+};
+/**
+ * Connection options for a MySQL database.
+ * Note: MySQL support is not yet implemented.
+ */
+type DatabaseMySqlOption = AbsDatabaseOption & {
+    /** The database type identifier. */
+    type: 'mysql';
+};
+export {};