@nitpicker/crawler 0.4.3 → 0.4.4

package/lib/archive/archive-accessor.d.ts

@@ -1,5 +1,5 @@
 import type { Database } from './database.js';
-import type { DatabaseEvent, PageFilter } from './types.js';
+import type { Config, DatabaseEvent, PageFilter } from './types.js';
 import type { ParseURLOptions } from '@d-zero/shared/parse-url';
 import { TypedAwaitEventEmitter as EventEmitter } from '@d-zero/shared/typed-await-event-emitter';
 import Page from './page.js';
@@ -25,6 +25,11 @@ export declare class ArchiveAccessor extends EventEmitter<DatabaseEvent> {
      *                    When null, `setData` is not available.
      */
     constructor(tmpDir: string, db: Database, namespace?: string | null);
+    /**
+     * Retrieves the crawl configuration stored in the archive database.
+     * @returns The parsed {@link Config} object.
+     */
+    getConfig(): Promise<Config>;
     /**
      * Retrieves anchor (link) data for a specific page by its database ID.
      * @param pageId - The database ID of the page whose anchors to retrieve.

package/lib/archive/archive-accessor.js

@@ -41,6 +41,13 @@ export class ArchiveAccessor extends EventEmitter {
             void this.emit('error', e);
         });
     }
+    /**
+     * Retrieves the crawl configuration stored in the archive database.
+     * @returns The parsed {@link Config} object.
+     */
+    async getConfig() {
+        return this.#db.getConfig();
+    }
     /**
      * Retrieves anchor (link) data for a specific page by its database ID.
      * @param pageId - The database ID of the page whose anchors to retrieve.

package/lib/archive/database.js

@@ -723,7 +723,6 @@ let Database = (() => {
                 t.string('name');
                 t.string('baseUrl');
                 t.boolean('recursive');
-                t.boolean('useSubprocess');
                 t.integer('interval');
                 t.boolean('image');
                 t.boolean('fetchExternal');
@@ -736,6 +735,8 @@ let Database = (() => {
                 t.integer('retry');
                 t.boolean('fromList');
                 t.boolean('disableQueries');
+                t.string('userAgent');
+                t.boolean('ignoreRobots');
             })
                 .createTable('pages', (t) => {
                 t.increments('id');

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nitpicker/crawler",
-	"version": "0.4.3",
+	"version": "0.4.4",
 	"description": "Web crawler engine with headless browser rendering and archive storage",
 	"author": "D-ZERO",
 	"license": "Apache-2.0",
@@ -48,5 +48,5 @@
 		"@types/tar": "7.0.87",
 		"@types/unzipper": "0.10.11"
 	},
-	"gitHead": "0f4ca55751be2f83dd5b6622c3502503fc7dfb41"
+	"gitHead": "7c9bad351d1a3f4eb6ea7233f7143140d0989ef2"
 }