@nitpicker/crawler 0.6.3 → 0.7.0

package/lib/crawler/speculative-pagination.js

@@ -1,215 +0,0 @@
-import { isError } from '@nitpicker/beholder';
-/**
- * Compares two consecutive URL strings and detects a single-token numeric
- * pagination pattern (e.g. `/page/1` → `/page/2`, or `?p=1` → `?p=2`).
- *
- * The algorithm decomposes each URL into tokens (path segments + sorted query values),
- * then checks that exactly one token differs and both values are integers with a
- * positive step. Returns `null` when no pattern is detected.
- *
- * WHY single-token constraint: Multi-token differences (e.g. both path and query
- * changing) indicate different routes rather than pagination, so they are rejected.
- * @param prevUrl - The previously pushed URL (protocol-agnostic, without hash/auth)
- * @param currentUrl - The newly discovered URL
- * @returns The detected pattern, or `null` if no pagination pattern was found
- */
-export function detectPaginationPattern(prevUrl, currentUrl) {
-    const prev = decomposeUrl(prevUrl);
-    const curr = decomposeUrl(currentUrl);
-    if (!prev || !curr)
-        return null;
-    // Host (including port) must match
-    if (prev.host !== curr.host)
-        return null;
-    // Path segment count must match
-    if (prev.pathSegments.length !== curr.pathSegments.length)
-        return null;
-    // Query key sets must match in count and identity
-    if (prev.queryKeys.length !== curr.queryKeys.length)
-        return null;
-    for (let i = 0; i < prev.queryKeys.length; i++) {
-        if (prev.queryKeys[i] !== curr.queryKeys[i])
-            return null;
-    }
-    // Build combined token arrays: path segments + query values (sorted by key)
-    const prevTokens = [...prev.pathSegments, ...prev.queryValues];
-    const currTokens = [...curr.pathSegments, ...curr.queryValues];
-    let diffIndex = -1;
-    for (const [i, prevToken] of prevTokens.entries()) {
-        if (prevToken !== currTokens[i]) {
-            if (diffIndex !== -1)
-                return null; // more than one difference
-            diffIndex = i;
-        }
-    }
-    if (diffIndex === -1)
-        return null; // identical URLs
-    const prevNum = Number(prevTokens[diffIndex]);
-    const currNum = Number(currTokens[diffIndex]);
-    if (!Number.isFinite(prevNum) || !Number.isFinite(currNum))
-        return null;
-    if (!Number.isInteger(prevNum) || !Number.isInteger(currNum))
-        return null;
-    const step = currNum - prevNum;
-    if (step <= 0)
-        return null;
-    return {
-        tokenIndex: diffIndex,
-        step,
-        currentNumber: currNum,
-    };
-}
-/**
- * Generates speculative URLs by extrapolating the detected pagination pattern.
- *
- * Starting from `currentUrl`, applies the pattern's step `count` times to produce
- * future page URLs (e.g. if step=1 and currentNumber=2, generates page 3, 4, ...).
- * These URLs are pushed into the crawl queue and discarded later if they 404.
- * @param pattern - The detected pagination pattern from {@link detectPaginationPattern}
- * @param currentUrl - The URL to extrapolate from (protocol-agnostic, without hash/auth)
- * @param count - Number of speculative URLs to generate (typically equals concurrency)
- * @returns Array of speculative URL strings
- */
-export function generateSpeculativeUrls(pattern, currentUrl, count) {
-    if (count <= 0)
-        return [];
-    const decomposed = decomposeUrl(currentUrl);
-    if (!decomposed)
-        return [];
-    const results = [];
-    for (let i = 1; i <= count; i++) {
-        const nextNum = pattern.currentNumber + pattern.step * i;
-        const url = reconstructUrl(decomposed, pattern.tokenIndex, String(nextNum));
-        results.push(url);
-    }
-    return results;
-}
-/**
- * Determines whether a speculative URL's scrape result should be discarded.
- *
- * Speculative URLs are pre-emptively pushed into the crawl queue before
- * knowing if they exist. This function filters out invalid results:
- * - `error` type → discard (server unreachable, timeout, etc.)
- * - `ignoreAndSkip` type → discard (matched exclusion rule)
- * - `scrapeEnd` with HTTP error status (4xx/5xx) → discard
- * - `scrapeEnd` with 2xx/3xx → keep
- * @param result - The scrape result for the speculative URL
- * @returns `true` if the result should be discarded (not saved to archive)
- */
-export function shouldDiscardSpeculative(result) {
-    switch (result.type) {
-        case 'error': {
-            return true;
-        }
-        case 'ignoreAndSkip': {
-            return true;
-        }
-        case 'scrapeEnd': {
-            if (!result.pageData)
-                return true;
-            return isError(result.pageData.status);
-        }
-        default: {
-            return true;
-        }
-    }
-}
-/**
- * Decomposes a URL string into its constituent tokens for comparison.
- * Handles both full URLs (`https://host/path?q=v`) and protocol-agnostic
- * URLs (`//host/path?q=v`). Query parameters are sorted by key for
- * consistent comparison.
- * @param url - The URL string to decompose
- * @returns The decomposed URL, or `null` if the format is invalid
- */
-function decomposeUrl(url) {
-    // URL format: //host/path?query  or  //host?query  (protocol-agnostic)
-    // Also handle protocol://host/path?query
-    let work = url;
-    let protocol = '';
-    // Strip protocol
-    const protoMatch = /^(https?:)?\/\//.exec(work);
-    if (!protoMatch)
-        return null;
-    protocol = protoMatch[1] ?? '';
-    work = work.slice(protoMatch[0].length);
-    // Split host from rest
-    const slashIdx = work.indexOf('/');
-    const qmarkIdx = work.indexOf('?');
-    let host;
-    let pathPart;
-    let queryPart;
-    if (slashIdx === -1 && qmarkIdx === -1) {
-        host = work;
-        pathPart = '';
-        queryPart = '';
-    }
-    else if (slashIdx === -1) {
-        host = work.slice(0, qmarkIdx);
-        pathPart = '';
-        queryPart = work.slice(qmarkIdx + 1);
-    }
-    else {
-        host = work.slice(0, slashIdx);
-        const pathAndQuery = work.slice(slashIdx + 1);
-        const pq = pathAndQuery.indexOf('?');
-        if (pq === -1) {
-            pathPart = pathAndQuery;
-            queryPart = '';
-        }
-        else {
-            pathPart = pathAndQuery.slice(0, pq);
-            queryPart = pathAndQuery.slice(pq + 1);
-        }
-    }
-    const pathSegments = pathPart ? pathPart.split('/') : [];
-    // Parse query into sorted key-value pairs
-    const queryPairs = [];
-    if (queryPart) {
-        for (const pair of queryPart.split('&')) {
-            const eqIdx = pair.indexOf('=');
-            if (eqIdx === -1) {
-                queryPairs.push([pair, '']);
-            }
-            else {
-                queryPairs.push([pair.slice(0, eqIdx), pair.slice(eqIdx + 1)]);
-            }
-        }
-    }
-    queryPairs.sort((a, b) => a[0].localeCompare(b[0]));
-    return {
-        host,
-        pathSegments,
-        queryKeys: queryPairs.map(([k]) => k),
-        queryValues: queryPairs.map(([, v]) => v),
-        protocol,
-    };
-}
-/**
- * Reconstructs a URL string from a decomposed representation with one
- * token replaced at the specified index.
- * @param decomposed - The decomposed URL to reconstruct
- * @param tokenIndex - Index in the combined token array (path segments + query values)
- * @param newValue - The replacement value for the token at `tokenIndex`
- * @returns The reconstructed URL string
- */
-function reconstructUrl(decomposed, tokenIndex, newValue) {
-    const { host, pathSegments, queryKeys, queryValues, protocol } = decomposed;
-    const newPathSegments = [...pathSegments];
-    const newQueryValues = [...queryValues];
-    if (tokenIndex < pathSegments.length) {
-        newPathSegments[tokenIndex] = newValue;
-    }
-    else {
-        newQueryValues[tokenIndex - pathSegments.length] = newValue;
-    }
-    let url = `${protocol}//${host}`;
-    if (newPathSegments.length > 0) {
-        url += `/${newPathSegments.join('/')}`;
-    }
-    if (queryKeys.length > 0) {
-        const pairs = queryKeys.map((k, i) => `${k}=${newQueryValues[i]}`);
-        url += `?${pairs.join('&')}`;
-    }
-    return url;
-}

package/lib/crawler/url-filter.d.ts

@@ -1,56 +0,0 @@
-import type { ExURL, ParseURLOptions } from '@d-zero/shared/parse-url';
-/**
- * Determine whether a URL should be skipped during crawling.
- *
- * A URL is skipped if it matches any user-defined exclude glob pattern
- * or starts with any of the excluded URL prefixes.
- * @param url - The parsed URL to check.
- * @param excludes - Array of glob patterns for URLs to exclude.
- * @param excludeUrls - Array of URL prefixes to exclude (matched via `startsWith`).
- * @param options - URL parsing options used for pattern matching.
- * @returns `true` if the URL should be skipped.
- */
-export declare function shouldSkipUrl(url: ExURL, excludes: readonly string[], excludeUrls: readonly string[], options: ParseURLOptions): boolean;
-/**
- * Determine whether a URL is external to the crawl scope.
- *
- * A URL is considered external if its hostname does not appear
- * as a key in the scope map.
- * @param url - The parsed URL to check.
- * @param scope - Map of hostnames to their scope URLs.
- * @returns `true` if the URL is outside the crawl scope.
- */
-export declare function isExternalUrl(url: ExURL, scope: ReadonlyMap<string, readonly ExURL[]>): boolean;
-/**
- * Inject authentication credentials from a matching scope URL into the target URL.
- *
- * Finds the best-matching scope URL (deepest path match) for the given URL's
- * hostname and copies its `username` and `password` properties. This mutates
- * the `url` parameter in place.
- * @param url - The parsed URL to receive authentication credentials (mutated in place).
- * @param scope - Map of hostnames to their scope URLs.
- */
-export declare function injectScopeAuth(url: ExURL, scope: ReadonlyMap<string, readonly ExURL[]>): void;
-/**
- * Find the scope URL with the deepest matching path for a given URL.
- *
- * Among all scope URLs sharing the same hostname, returns the one whose
- * path segments are a prefix of the target URL's path segments and which
- * has the greatest depth. Returns `null` if no scope URL matches.
- * @param url - The parsed URL to match against scope URLs.
- * @param scopes - The list of scope URLs to search.
- * @returns The best-matching scope URL, or `null` if none match.
- */
-export declare function findBestMatchingScope(url: ExURL, scopes: readonly ExURL[]): ExURL | null;
-/**
- * Check whether a URL is in a lower layer (subdirectory) of any scope URL.
- *
- * Tests the URL against each scope URL using the `isLowerLayer` utility,
- * which checks if the URL's path is at the same level or deeper than
- * the scope URL's path.
- * @param url - The parsed URL to check.
- * @param scopes - The list of scope URLs to test against.
- * @param options - URL parsing options used for layer comparison.
- * @returns `true` if the URL is in a lower layer of at least one scope URL.
- */
-export declare function isInAnyLowerLayer(url: ExURL, scopes: readonly ExURL[], options: ParseURLOptions): boolean;

package/lib/crawler/url-filter.js

@@ -1,110 +0,0 @@
-import { isLowerLayer } from '@d-zero/shared/is-lower-layer';
-import { pathMatch } from '@d-zero/shared/path-match';
-import { protocolAgnosticKey } from './protocol-agnostic-key.js';
-/**
- * Determine whether a URL should be skipped during crawling.
- *
- * A URL is skipped if it matches any user-defined exclude glob pattern
- * or starts with any of the excluded URL prefixes.
- * @param url - The parsed URL to check.
- * @param excludes - Array of glob patterns for URLs to exclude.
- * @param excludeUrls - Array of URL prefixes to exclude (matched via `startsWith`).
- * @param options - URL parsing options used for pattern matching.
- * @returns `true` if the URL should be skipped.
- */
-export function shouldSkipUrl(url, excludes, excludeUrls, options) {
-    return (excludes.some((excludeGlobPattern) => pathMatch(url, excludeGlobPattern, options)) ||
-        excludeUrls.some((prefix) => protocolAgnosticKey(url.href).startsWith(protocolAgnosticKey(prefix))));
-}
-/**
- * Determine whether a URL is external to the crawl scope.
- *
- * A URL is considered external if its hostname does not appear
- * as a key in the scope map.
- * @param url - The parsed URL to check.
- * @param scope - Map of hostnames to their scope URLs.
- * @returns `true` if the URL is outside the crawl scope.
- */
-export function isExternalUrl(url, scope) {
-    return !scope.has(url.hostname);
-}
-/**
- * Inject authentication credentials from a matching scope URL into the target URL.
- *
- * Finds the best-matching scope URL (deepest path match) for the given URL's
- * hostname and copies its `username` and `password` properties. This mutates
- * the `url` parameter in place.
- * @param url - The parsed URL to receive authentication credentials (mutated in place).
- * @param scope - Map of hostnames to their scope URLs.
- */
-export function injectScopeAuth(url, scope) {
-    const scopes = scope.get(url.hostname);
-    if (!scopes) {
-        return;
-    }
-    const matchedScope = findBestMatchingScope(url, scopes);
-    if (matchedScope) {
-        url.username = matchedScope.username;
-        url.password = matchedScope.password;
-    }
-}
-/**
- * Find the scope URL with the deepest matching path for a given URL.
- *
- * Among all scope URLs sharing the same hostname, returns the one whose
- * path segments are a prefix of the target URL's path segments and which
- * has the greatest depth. Returns `null` if no scope URL matches.
- * @param url - The parsed URL to match against scope URLs.
- * @param scopes - The list of scope URLs to search.
- * @returns The best-matching scope URL, or `null` if none match.
- */
-export function findBestMatchingScope(url, scopes) {
-    let bestMatch = null;
-    let maxDepth = -1;
-    for (const scope of scopes) {
-        if (url.hostname !== scope.hostname) {
-            continue;
-        }
-        const isMatch = isPathMatch(url.paths, scope.paths);
-        if (isMatch && scope.depth > maxDepth) {
-            bestMatch = scope;
-            maxDepth = scope.depth;
-        }
-    }
-    return bestMatch;
-}
-/**
- * Check whether a target path is equal to or is a descendant of a base path.
- *
- * Compares path segments element by element. The target path matches if
- * all segments of the base path appear in the same positions at the
- * beginning of the target path.
- * @param targetPaths - The path segments of the URL being checked.
- * @param basePaths - The path segments of the scope URL to match against.
- * @returns `true` if the target path starts with or equals the base path.
- */
-function isPathMatch(targetPaths, basePaths) {
-    if (targetPaths.length < basePaths.length) {
-        return false;
-    }
-    for (const [i, basePath] of basePaths.entries()) {
-        if (targetPaths[i] !== basePath) {
-            return false;
-        }
-    }
-    return true;
-}
-/**
- * Check whether a URL is in a lower layer (subdirectory) of any scope URL.
- *
- * Tests the URL against each scope URL using the `isLowerLayer` utility,
- * which checks if the URL's path is at the same level or deeper than
- * the scope URL's path.
- * @param url - The parsed URL to check.
- * @param scopes - The list of scope URLs to test against.
- * @param options - URL parsing options used for layer comparison.
- * @returns `true` if the URL is in a lower layer of at least one scope URL.
- */
-export function isInAnyLowerLayer(url, scopes, options) {
-    return scopes.some((scope) => isLowerLayer(url.href, scope.href, options));
-}

package/lib/index.d.ts

@@ -1,16 +0,0 @@
-/**
- * @module @nitpicker/crawler
- *
- * Core module of Nitpicker that provides the main crawling engine,
- * utility functions, type definitions, and archive storage layer.
- */
-export * from './utils/index.js';
-export { ArchiveAccessor } from './archive/archive-accessor.js';
-export type { Redirect, Referrer, Anchor, StaticPageData } from './archive/page.js';
-export { default as Page } from './archive/page.js';
-export { default as ArchiveResource } from './archive/resource.js';
-export * from './archive/types.js';
-export { default as Archive } from './archive/archive.js';
-export { DEFAULT_EXCLUDED_EXTERNAL_URLS, CrawlerOrchestrator, } from './crawler-orchestrator.js';
-export * from './types.js';
-export * from './crawler/types.js';

package/lib/index.js

@@ -1,18 +0,0 @@
-/**
- * @module @nitpicker/crawler
- *
- * Core module of Nitpicker that provides the main crawling engine,
- * utility functions, type definitions, and archive storage layer.
- */
-// Types + Utils (旧 @nitpicker/types + utils)
-export * from './utils/index.js';
-// Archive
-export { ArchiveAccessor } from './archive/archive-accessor.js';
-export { default as Page } from './archive/page.js';
-export { default as ArchiveResource } from './archive/resource.js';
-export * from './archive/types.js';
-export { default as Archive } from './archive/archive.js';
-// Core
-export { DEFAULT_EXCLUDED_EXTERNAL_URLS, CrawlerOrchestrator, } from './crawler-orchestrator.js';
-export * from './types.js';
-export * from './crawler/types.js';

package/lib/qzilla.d.ts

@@ -1,136 +0,0 @@
-import type { QzillaEvent } from './types.js';
-import type { Config } from './archive/types.js';
-import Archive from './archive/archive.js';
-import { EventEmitter } from './utils/index.js';
-import type { ExURL } from './utils/index.js';
-/**
- * Default list of external URL prefixes excluded from crawling.
- * Includes social media sharing endpoints that are commonly linked
- * but provide no useful crawl data.
- */
-export declare const DEFAULT_EXCLUDED_EXTERNAL_URLS: string[];
-/**
- * Configuration options for the Qzilla crawler.
- *
- * Extends the archive {@link Config} with additional runtime settings
- * such as working directory, browser executable path, and output options.
- */
-type QzillaConfig = {
-    /** The working directory for output files. Defaults to `process.cwd()`. */
-    cwd: string;
-    /** Path to a Chromium/Chrome executable for Puppeteer. */
-    executablePath: string;
-    /** Output file path for the archive. */
-    filePath: string;
-    /** Whether to capture image resources during crawling. */
-    image: boolean;
-    /** File-size threshold (in bytes) above which images are excluded. */
-    imageFileSizeThreshold: number;
-    /** Delay in milliseconds between each page request. */
-    interval: number;
-    /** Whether the input is a pre-defined URL list (non-recursive mode). */
-    list: boolean;
-    /** Whether to enable verbose logging output. */
-    verbose: boolean;
-} & Config;
-/**
- * Callback invoked after the Qzilla instance is fully initialized
- * but before crawling begins.
- * @param qzilla - The initialized Qzilla instance.
- * @param config - The resolved archive configuration.
- */
-type QzillaInitializedCallback = (qzilla: Qzilla, config: Config) => void | Promise<void>;
-/**
- * The main entry point for Qzilla web crawling and archiving.
- *
- * Qzilla orchestrates the full lifecycle of a crawl session: it creates an archive,
- * configures a {@link Crawler}, processes discovered pages and resources, and
- * writes the final archive file. It emits events defined by {@link QzillaEvent}.
- *
- * Instances are created via the static factory methods {@link Qzilla.crawling}
- * or {@link Qzilla.resume}; the constructor is private.
- * @example
- * ```ts
- * const qzilla = await Qzilla.crawling(['https://example.com'], { recursive: true });
- * await qzilla.write();
- * ```
- */
-export declare class Qzilla extends EventEmitter<QzillaEvent> {
-    #private;
-    /**
-     * The underlying archive instance used for storing crawl results.
-     */
-    get archive(): Archive;
-    private constructor();
-    /**
-     * Abort the current crawl and archive operations.
-     *
-     * Delegates to the archive's abort method, which stops all in-progress
-     * database writes and cleans up temporary resources.
-     * @returns The result of the archive abort operation.
-     */
-    abort(): void;
-    /**
-     * Execute the crawl for the given list of URLs.
-     *
-     * Sets up event listeners on the crawler, starts crawling, and resolves
-     * when the crawl completes. Discovered pages, external pages, skipped pages,
-     * and resources are forwarded to the archive for storage.
-     * @param list - The list of parsed URLs to crawl. The first URL is used as the root.
-     * @returns A promise that resolves when crawling is complete.
-     * @throws {Error} If the URL list is empty.
-     */
-    crawling(list: ExURL[]): Promise<void>;
-    /**
-     * Kill any zombie Chromium processes that were not properly cleaned up.
-     *
-     * Retrieves the list of undead process IDs from the crawler and sends
-     * a SIGTERM signal to each one. Chromium is intentionally sent SIGTERM
-     * (not SIGKILL) to avoid leaving zombie processes.
-     */
-    garbageCollect(): void;
-    /**
-     * Retrieve the list of process IDs for Chromium instances that are
-     * still running after crawling has ended.
-     * @returns An array of process IDs that should be terminated.
-     */
-    getUndeadPid(): never[];
-    /**
-     * Write the archive to its configured file path.
-     *
-     * Emits `writeFileStart` before writing and `writeFileEnd` after
-     * the write completes successfully.
-     */
-    write(): Promise<void>;
-    /**
-     * Create a new Qzilla instance and start crawling the given URLs.
-     *
-     * This is the primary factory method for starting a fresh crawl. It:
-     * 1. Parses and sorts the input URLs
-     * 2. Creates an archive file
-     * 3. Saves the crawl configuration
-     * 4. Runs the optional initialized callback
-     * 5. Executes the crawl
-     * 6. Sorts the archived URLs in natural order
-     * @param url - One or more URL strings to crawl.
-     * @param options - Optional configuration overrides for the crawl session.
-     * @param initializedCallback - Optional callback invoked after initialization but before crawling starts.
-     * @returns A promise that resolves to the Qzilla instance after crawling completes.
-     * @throws {Error} If the URL list is empty or contains no valid URLs.
-     */
-    static crawling(url: string[], options?: Partial<QzillaConfig>, initializedCallback?: QzillaInitializedCallback): Promise<Qzilla>;
-    /**
-     * Resume a previously interrupted crawl from an existing archive file.
-     *
-     * Restores the crawl state (pending URLs, scraped URLs, and resources)
-     * from the archive, merges any option overrides, and continues crawling
-     * from where it left off.
-     * @param stubPath - Path to the existing archive file to resume from.
-     * @param options - Optional configuration overrides to apply on top of the archived config.
-     * @param initializedCallback - Optional callback invoked after initialization but before crawling resumes.
-     * @returns A promise that resolves to the Qzilla instance after crawling completes.
-     * @throws {Error} If the archived URL is invalid.
-     */
-    static resume(stubPath: string, options?: Partial<QzillaConfig>, initializedCallback?: QzillaInitializedCallback): Promise<Qzilla>;
-}
-export {};