@arcblock/crawler 1.1.5 → 1.2.0

package/README.md

@@ -43,8 +43,7 @@ await initCrawler({
     immediate: !!env.preferences.cronImmediate,
     sites: env.preferences.cronSites,
     time: env.preferences.cronTime,
-    crawlConcurrency: env.preferences.crawlConcurrency,
-    sitemapConcurrency: env.preferences.sitemapConcurrency,
+    concurrency: env.preferences.concurrency,
   },
 });
 ```

package/lib/cjs/config.d.ts

@@ -1,3 +1,4 @@
+import { CookieParam } from '@blocklet/puppeteer';
 export type Site = {
     url: string;
     pathname: string;
@@ -11,14 +12,19 @@ export type Config = {
     appUrl: string;
     cacheDir: string;
     puppeteerPath?: string;
-    siteCron: {
+    concurrency: number;
+    siteCron?: {
         sites: Site[];
         time: string;
         enabled: boolean;
         immediate: boolean;
-        crawlConcurrency: number;
-        sitemapConcurrency: number;
+        concurrency: number;
     };
+    cookies?: CookieParam[];
+    localStorage?: {
+        key: string;
+        value: string;
+    }[];
 };
 export declare const logger: any;
 export declare const config: Config;

package/lib/cjs/config.js

@@ -9,17 +9,9 @@ exports.logger = (0, logger_1.default)('@arcblock/crawler', { level: process.env
 exports.config = {
     isProd: process.env.NODE_ENV === 'production',
     dataDir: process.env.BLOCKLET_DATA_DIR,
-    appDir: process.env.BLOCKLET_APP_DIR || process.cwd(),
     cacheDir: process.env.BLOCKLET_CACHE_DIR || process.cwd(),
+    appDir: process.env.BLOCKLET_APP_DIR || process.cwd(),
     appUrl: process.env.BLOCKLET_APP_URL || '/',
     puppeteerPath: process.env.PUPPETEER_EXECUTABLE_PATH,
-    // cron
-    siteCron: {
-        sites: [],
-        enabled: true,
-        time: '0 0 0 * * *',
-        immediate: false,
-        crawlConcurrency: 2,
-        sitemapConcurrency: 30,
-    },
+    concurrency: 2,
 };

package/lib/cjs/crawler.d.ts

@@ -1,11 +1,10 @@
-import { JobState } from './store/job';
-import { SnapshotModel } from './store/snapshot';
-export declare function createCrawlQueue(): void;
+import { JobState, SnapshotModel } from './store';
+export declare function createCrawlQueue(queue: string): any;
 export declare function getDataDir(): Promise<{
     htmlDir: string;
     screenshotDir: string;
 }>;
-export declare const getPageContent: ({ url, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage, headers, cookies, localStorage, }: JobState) => Promise<{
+export declare const getPageContent: ({ url, includeScreenshot, includeHtml, width, height, quality, timeout, waitTime, fullPage, headers, cookies, localStorage, }: JobState) => Promise<{
     html: string | null;
     screenshot: Uint8Array<ArrayBufferLike> | null;
     meta: {

package/lib/cjs/crawler.js

@@ -24,16 +24,16 @@ const path_1 = __importDefault(require("path"));
 const config_1 = require("./config");
 const puppeteer_1 = require("./puppeteer");
 const snapshot_1 = require("./services/snapshot");
-const job_1 = require("./store/job");
-const snapshot_2 = require("./store/snapshot");
+const store_1 = require("./store");
 const utils_1 = require("./utils");
 const { BaseState } = require('@abtnode/models');
-let crawlQueue;
-function createCrawlQueue() {
-    const db = new BaseState(job_1.Job);
-    crawlQueue = (0, queue_1.default)({
-        store: new sequelize_1.default(db, 'crawler'),
-        concurrency: config_1.config.siteCron.crawlConcurrency,
+// eslint-disable-next-line import/no-mutable-exports
+const crawlQueue = createCrawlQueue('urlCrawler');
+function createCrawlQueue(queue) {
+    const db = new BaseState(store_1.Job);
+    return (0, queue_1.default)({
+        store: new sequelize_1.default(db, queue),
+        concurrency: config_1.config.concurrency,
         onJob: (job) => __awaiter(this, void 0, void 0, function* () {
             config_1.logger.info('Starting to execute crawl job', job);
             const canCrawl = yield (0, utils_1.isAcceptCrawler)(job.url);
@@ -46,7 +46,7 @@ function createCrawlQueue() {
                         error: 'Denied by robots.txt',
                     },
                 });
-                yield snapshot_2.Snapshot.upsert(snapshot);
+                yield store_1.Snapshot.upsert(snapshot);
                 return snapshot;
             }
             // if index reach autoCloseBrowserCount, close browser
@@ -57,54 +57,67 @@ function createCrawlQueue() {
             // } catch (error) {
             //   logger.error('failed to close browser when queue index reached autoCloseBrowserCount:', error);
             // }
+            const formattedJob = Object.assign(Object.assign({}, job), { cookies: (config_1.config.cookies || []).concat(job.cookies || []), localStorage: (config_1.config.localStorage || []).concat(job.localStorage || []), url: (0, utils_1.formatUrl)(job.url) });
             try {
                 // get page content later
-                const result = yield (0, exports.getPageContent)(Object.assign({ localStorage: {
-                        // for blocklet theme
-                        blocklet_theme_prefer: 'light',
-                        // for blocklet domain warning
-                        'domain-warning-skip': Date.now().toString(),
-                    } }, job));
+                const result = yield (0, exports.getPageContent)(formattedJob);
                 if (!result || (!result.html && !result.screenshot)) {
-                    config_1.logger.error(`failed to crawl ${job.url}, empty content`, job);
+                    config_1.logger.error(`failed to crawl ${formattedJob.url}, empty content`, formattedJob);
                     const snapshot = (0, snapshot_1.convertJobToSnapshot)({
-                        job,
+                        job: formattedJob,
                         snapshot: {
                             status: 'failed',
                             error: 'Failed to crawl content',
                         },
                     });
-                    yield snapshot_2.Snapshot.upsert(snapshot);
+                    yield store_1.Snapshot.upsert(snapshot);
                     return snapshot;
                 }
-                // save html and screenshot to data dir
-                const { screenshotPath, htmlPath } = yield saveSnapshotToLocal({
-                    screenshot: result.screenshot,
-                    html: result.html,
-                });
-                // const lastModified = job.lastmodMap?.get(url) || new Date().toISOString();
-                const snapshot = (0, snapshot_1.convertJobToSnapshot)({
-                    job,
-                    snapshot: {
-                        status: 'success',
-                        screenshot: screenshotPath === null || screenshotPath === void 0 ? void 0 : screenshotPath.replace(config_1.config.dataDir, ''),
-                        html: htmlPath === null || htmlPath === void 0 ? void 0 : htmlPath.replace(config_1.config.dataDir, ''),
-                        meta: result.meta,
-                    },
-                });
-                yield snapshot_2.Snapshot.upsert(snapshot);
+                const snapshot = yield store_1.sequelize.transaction((txn) => __awaiter(this, void 0, void 0, function* () {
+                    // delete old snapshot
+                    if (formattedJob.replace) {
+                        try {
+                            const deletedJobIds = yield (0, snapshot_1.deleteSnapshots)({
+                                url: formattedJob.url,
+                                replace: true,
+                            }, { txn });
+                            if (deletedJobIds) {
+                                config_1.logger.info('Deleted old snapshot', { deletedJobIds });
+                            }
+                        }
+                        catch (error) {
+                            config_1.logger.error('Failed to delete old snapshot', { error, formattedJob });
+                        }
+                    }
+                    // save html and screenshot to data dir
+                    const { screenshotPath, htmlPath } = yield saveSnapshotToLocal({
+                        screenshot: result.screenshot,
+                        html: result.html,
+                    });
+                    const snapshot = (0, snapshot_1.convertJobToSnapshot)({
+                        job: formattedJob,
+                        snapshot: {
+                            status: 'success',
+                            screenshot: screenshotPath === null || screenshotPath === void 0 ? void 0 : screenshotPath.replace(config_1.config.dataDir, ''),
+                            html: htmlPath === null || htmlPath === void 0 ? void 0 : htmlPath.replace(config_1.config.dataDir, ''),
+                            meta: result.meta,
+                        },
+                    });
+                    yield store_1.Snapshot.upsert(snapshot, { transaction: txn });
+                    return snapshot;
+                }));
                 return snapshot;
             }
             catch (error) {
-                config_1.logger.error(`Failed to crawl ${job.url}`, { error, job });
+                config_1.logger.error(`Failed to crawl ${formattedJob.url}`, { error, formattedJob });
                 const snapshot = (0, snapshot_1.convertJobToSnapshot)({
-                    job,
+                    job: formattedJob,
                     snapshot: {
                         status: 'failed',
                         error: 'Internal error',
                     },
                 });
-                yield snapshot_2.Snapshot.upsert(snapshot);
+                yield store_1.Snapshot.upsert(snapshot);
                 return snapshot;
             }
         }),
@@ -142,7 +155,7 @@ function saveSnapshotToLocal(_a) {
         };
     });
 }
-const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, timeout = 90 * 1000, fullPage = false, headers, cookies = [], localStorage, }) {
+const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, timeout = 90 * 1000, waitTime = 0, fullPage = false, headers, cookies, localStorage, }) {
     const page = yield (0, puppeteer_1.initPage)();
     if (width && height) {
         yield page.setViewport({ width, height, deviceScaleFactor: 2 });
@@ -150,13 +163,18 @@ const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url,
     if (headers) {
         yield page.setExtraHTTPHeaders(headers);
     }
-    if (cookies === null || cookies === void 0 ? void 0 : cookies.length) {
-        yield page.setCookie(...cookies);
+    // handle cookies
+    if (cookies) {
+        const { hostname } = new URL(url);
+        const cookieParams = cookies.map((item) => (Object.assign(Object.assign({}, item), { expires: item.expires ? new Date(item.expires).getTime() : undefined, domain: item.domain || hostname, path: item.path || '/' })));
+        yield page.setCookie(...cookieParams);
     }
+    // handle localStorage
     if (localStorage) {
         yield page.evaluateOnNewDocument((items) => {
-            Object.entries(items).forEach(([key, value]) => {
-                window.localStorage.setItem(key, value);
+            items.forEach((item) => {
+                const value = item.value === 'now()' ? new Date().toISOString() : item.value;
+                window.localStorage.setItem(item.key, value);
             });
         }, localStorage);
     }
@@ -174,9 +192,18 @@ const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url,
         }
         // await for networkidle0
         // https://pptr.dev/api/puppeteer.page.waitfornetworkidle
-        yield page.waitForNetworkIdle({
-            idleTime: 1.5 * 1000,
-        });
+        try {
+            yield Promise.all([
+                page.waitForNetworkIdle({
+                    idleTime: 1.5 * 1000,
+                    timeout,
+                }),
+                (0, utils_1.sleep)(waitTime),
+            ]);
+        }
+        catch (err) {
+            config_1.logger.warn(`Failed to wait for network idle in ${url}:`, err);
+        }
         // get screenshot
         if (includeScreenshot) {
             // Try to find the tallest element and set the browser to the same height
@@ -220,7 +247,7 @@ const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url,
             // check if the page is an error page
             const isErrorPage = ['<h2>Unexpected Application Error!</h2>', 'Current route occurred an error'].some((errorHtml) => data.html.includes(errorHtml));
             if (isErrorPage) {
-                throw new Error('Page is an error page');
+                throw new Error(`${url} is an error page`);
             }
             meta.title = data.title;
             meta.description = data.description;
@@ -255,9 +282,8 @@ exports.getPageContent = getPageContent;
 // eslint-disable-next-line require-await
 function crawlUrl(params, callback) {
     return __awaiter(this, void 0, void 0, function* () {
-        params = Object.assign(Object.assign({}, params), { url: (0, utils_1.formatUrl)(params.url) });
         // skip duplicate job
-        const existsJob = yield job_1.Job.isExists(params);
+        const existsJob = yield store_1.Job.isExists(params);
         if (existsJob) {
             config_1.logger.info(`Crawl job already exists for ${params.url}, skip`);
             return existsJob.id;

package/lib/cjs/cron.js

@@ -20,6 +20,8 @@ let cron = null;
 function initCron() {
     if (cron)
         return;
+    if (!config_1.config.siteCron)
+        return;
     config_1.logger.info('Init cron', { config: config_1.config.siteCron });
     cron = cron_1.default.init({
         context: {},
@@ -29,6 +31,9 @@ function initCron() {
                 time: config_1.config.siteCron.time,
                 options: { runOnInit: config_1.config.siteCron.immediate },
                 fn: () => __awaiter(this, void 0, void 0, function* () {
+                    var _a;
+                    if (!((_a = config_1.config.siteCron) === null || _a === void 0 ? void 0 : _a.enabled))
+                        return;
                     config_1.logger.info('Start cron to crawl site', { sites: config_1.config.siteCron.sites });
                     for (const site of config_1.config.siteCron.sites) {
                         try {

package/lib/cjs/index.d.ts

@@ -3,7 +3,5 @@ export * from './crawler';
 export * from './site';
 export * from './services/snapshot';
 export * as utils from './utils';
-type DeepPartial<T> = T extends object ? {
-    [P in keyof T]?: DeepPartial<T[P]>;
-} : T;
-export declare function initCrawler(params: DeepPartial<Pick<Config, 'puppeteerPath' | 'siteCron'>>): Promise<void>;
+export { migrate } from './store/migrate';
+export declare function initCrawler(params: Pick<Config, 'puppeteerPath' | 'siteCron' | 'cookies' | 'localStorage' | 'concurrency'>): Promise<void>;

package/lib/cjs/index.js

@@ -48,27 +48,27 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.utils = void 0;
+exports.migrate = exports.utils = void 0;
 exports.initCrawler = initCrawler;
+/* eslint-disable @typescript-eslint/indent */
 const merge_1 = __importDefault(require("lodash/merge"));
 const config_1 = require("./config");
-const crawler_1 = require("./crawler");
 const cron_1 = require("./cron");
 const puppeteer_1 = require("./puppeteer");
-const store_1 = require("./store");
 __exportStar(require("./crawler"), exports);
 __exportStar(require("./site"), exports);
 __exportStar(require("./services/snapshot"), exports);
 exports.utils = __importStar(require("./utils"));
+var migrate_1 = require("./store/migrate");
+Object.defineProperty(exports, "migrate", { enumerable: true, get: function () { return migrate_1.migrate; } });
 function initCrawler(params) {
     return __awaiter(this, void 0, void 0, function* () {
+        var _a;
         (0, merge_1.default)(config_1.config, params);
         config_1.logger.info('Init crawler', { params, config: config_1.config });
         try {
-            yield (0, store_1.initDatabase)();
             yield (0, puppeteer_1.ensureBrowser)();
-            yield (0, crawler_1.createCrawlQueue)();
-            if (config_1.config.siteCron.enabled) {
+            if ((_a = config_1.config.siteCron) === null || _a === void 0 ? void 0 : _a.enabled) {
                 yield (0, cron_1.initCron)();
             }
         }

package/lib/cjs/services/snapshot.d.ts

@@ -1,5 +1,5 @@
-import { JobState } from '../store/job';
-import { SnapshotModel } from '../store/snapshot';
+import { Transaction, WhereOptions } from '@sequelize/core';
+import { JobState, SnapshotModel } from '../store';
 export declare function convertJobToSnapshot({ job, snapshot }: {
     job: JobState;
     snapshot?: Partial<SnapshotModel>;
@@ -10,3 +10,6 @@ export declare function formatSnapshot(snapshot: SnapshotModel, columns?: Array<
  */
 export declare function getSnapshot(jobId: string): Promise<SnapshotModel | null>;
 export declare function getLatestSnapshot(url: string): Promise<SnapshotModel | null>;
+export declare function deleteSnapshots(where: WhereOptions<SnapshotModel>, { txn }?: {
+    txn?: Transaction;
+}): Promise<string[]>;

package/lib/cjs/services/snapshot.js

@@ -16,16 +16,17 @@ exports.convertJobToSnapshot = convertJobToSnapshot;
 exports.formatSnapshot = formatSnapshot;
 exports.getSnapshot = getSnapshot;
 exports.getLatestSnapshot = getLatestSnapshot;
+exports.deleteSnapshots = deleteSnapshots;
+const cloneDeep_1 = __importDefault(require("lodash/cloneDeep"));
 const pick_1 = __importDefault(require("lodash/pick"));
 const promises_1 = __importDefault(require("node:fs/promises"));
 const node_path_1 = __importDefault(require("node:path"));
 const ufo_1 = require("ufo");
 const config_1 = require("../config");
-const job_1 = require("../store/job");
-const snapshot_1 = require("../store/snapshot");
+const store_1 = require("../store");
 const utils_1 = require("../utils");
 function convertJobToSnapshot({ job, snapshot }) {
-    return Object.assign({ jobId: job.jobId || job.id, url: job.url, lastModified: job.lastModified || new Date().toISOString(), options: {
+    return Object.assign({ jobId: job.jobId || job.id, url: job.url, lastModified: job.lastModified || new Date().toISOString(), replace: job.replace, options: {
             width: job.width,
             height: job.height,
             includeScreenshot: job.includeScreenshot,
@@ -36,7 +37,7 @@ function convertJobToSnapshot({ job, snapshot }) {
 }
 function formatSnapshot(snapshot, columns) {
     return __awaiter(this, void 0, void 0, function* () {
-        let data = Object.assign({}, snapshot);
+        let data = (0, cloneDeep_1.default)(snapshot);
         // format screenshot path to full url
         if (data.screenshot) {
             data.screenshot = (0, ufo_1.joinURL)(config_1.config.appUrl, data.screenshot);
@@ -46,6 +47,12 @@ function formatSnapshot(snapshot, columns) {
             const html = yield promises_1.default.readFile(node_path_1.default.join(config_1.config.dataDir, data.html));
             data.html = html.toString();
         }
+        // remove sensitive options that should not be returned
+        if (data.options) {
+            delete data.options.cookies;
+            delete data.options.localStorage;
+            delete data.options.headers;
+        }
         if (columns === null || columns === void 0 ? void 0 : columns.length) {
             data = (0, pick_1.default)(data, columns);
         }
@@ -57,11 +64,11 @@ function formatSnapshot(snapshot, columns) {
  */
 function getSnapshot(jobId) {
     return __awaiter(this, void 0, void 0, function* () {
-        const snapshot = yield snapshot_1.Snapshot.findSnapshot({ where: { jobId } });
+        const snapshot = yield store_1.Snapshot.findSnapshot({ where: { jobId } });
         if (snapshot) {
             return formatSnapshot(snapshot);
         }
-        const job = yield job_1.Job.findJob({ id: jobId });
+        const job = yield store_1.Job.findJob({ id: jobId });
         if (job) {
             return {
                 jobId,
@@ -73,12 +80,42 @@ function getSnapshot(jobId) {
 }
 function getLatestSnapshot(url) {
     return __awaiter(this, void 0, void 0, function* () {
-        const snapshot = yield snapshot_1.Snapshot.findSnapshot({
+        const snapshot = yield store_1.Snapshot.findSnapshot({
             where: {
                 url: (0, utils_1.formatUrl)(url),
                 status: 'success',
             },
+            order: [
+                ['lastModified', 'DESC'],
+                ['updatedAt', 'DESC'],
+            ],
         });
         return snapshot ? formatSnapshot(snapshot) : null;
     });
 }
+function deleteSnapshots(where_1) {
+    return __awaiter(this, arguments, void 0, function* (where, { txn } = {}) {
+        const snapshots = yield store_1.Snapshot.findAll({
+            where,
+            order: [
+                ['lastModified', 'DESC'],
+                ['updatedAt', 'DESC'],
+            ],
+        });
+        const jobIds = yield Promise.all(snapshots.map((snapshot) => __awaiter(this, void 0, void 0, function* () {
+            try {
+                yield Promise.all([
+                    snapshot.html && promises_1.default.unlink(node_path_1.default.join(config_1.config.dataDir, snapshot.html)),
+                    snapshot.screenshot && promises_1.default.unlink(node_path_1.default.join(config_1.config.dataDir, snapshot.screenshot)),
+                ]);
+                yield snapshot.destroy({ transaction: txn });
+                return snapshot.jobId;
+            }
+            catch (error) {
+                config_1.logger.error('Failed to delete snapshot', { error, snapshot });
+                throw error;
+            }
+        })));
+        return jobIds.filter(Boolean);
+    });
+}

package/lib/cjs/site.d.ts

@@ -1,2 +1,2 @@
 import { Site } from './config';
-export declare const crawlSite: ({ url, pathname, interval }: Site) => Promise<(string | null)[]>;
+export declare const crawlSite: ({ url, pathname, interval }: Site) => Promise<(`${string}-${string}-${string}-${string}-${string}` | null)[]>;

package/lib/cjs/site.js

@@ -14,12 +14,14 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.crawlSite = void 0;
 const uniq_1 = __importDefault(require("lodash/uniq"));
+const node_crypto_1 = require("node:crypto");
 const p_map_1 = __importDefault(require("p-map"));
 const config_1 = require("./config");
 const crawler_1 = require("./crawler");
-const snapshot_1 = require("./store/snapshot");
+const store_1 = require("./store");
 const utils_1 = require("./utils");
 const crawlBlockletRunningMap = new Map();
+const crawlQueue = (0, crawler_1.createCrawlQueue)('cronJobs');
 function parseSitemapUrl(sitemapItem) {
     var _a;
     const links = ((_a = sitemapItem.links) === null || _a === void 0 ? void 0 : _a.map((item) => item.url)) || [];
@@ -27,6 +29,7 @@ function parseSitemapUrl(sitemapItem) {
     return urls.map((url) => ({ url, sitemapItem }));
 }
 const crawlSite = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, pathname, interval = 0 }) {
+    var _b;
     config_1.logger.info(`Start crawl from sitemap ${url}`, { pathname });
     const key = `${url}-${pathname}`;
     if (crawlBlockletRunningMap.has(key)) {
@@ -47,7 +50,7 @@ const crawlSite = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, path
     try {
         const jobIds = yield (0, p_map_1.default)(sitemapItems, (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, sitemapItem }) {
             processCount++;
-            const snapshot = yield snapshot_1.Snapshot.findOne({ where: { url: (0, utils_1.formatUrl)(url) } });
+            const snapshot = yield store_1.Snapshot.findOne({ where: { url: (0, utils_1.formatUrl)(url) } });
             if (snapshot === null || snapshot === void 0 ? void 0 : snapshot.lastModified) {
                 const lastModified = new Date(snapshot.lastModified);
                 // skip if snapshot lastModified is greater than sitemap lastmod
@@ -66,13 +69,17 @@ const crawlSite = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, path
                 url,
             });
             crawlCount++;
-            return (0, crawler_1.crawlUrl)({
+            const jobId = (0, node_crypto_1.randomUUID)();
+            crawlQueue.push({
+                id: jobId,
                 url,
                 lastModified: sitemapItem.lastmod,
                 includeScreenshot: false,
                 includeHtml: true,
+                replace: true,
             });
-        }), { concurrency: config_1.config.siteCron.sitemapConcurrency });
+            return jobId;
+        }), { concurrency: ((_b = config_1.config.siteCron) === null || _b === void 0 ? void 0 : _b.concurrency) || 30 });
         config_1.logger.info('Enqueued jobs from sitemap finished', {
             url,
             pathname,

package/lib/cjs/store/index.d.ts

@@ -1,3 +1,6 @@
 import { Sequelize } from '@sequelize/core';
 import { SqliteDialect } from '@sequelize/sqlite3';
-export declare function initDatabase(): Promise<Sequelize<SqliteDialect>>;
+declare const sequelize: Sequelize<SqliteDialect>;
+export { sequelize };
+export * from './job';
+export * from './snapshot';

package/lib/cjs/store/index.js

@@ -1,57 +1,49 @@
 "use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.initDatabase = initDatabase;
+exports.sequelize = void 0;
 const core_1 = require("@sequelize/core");
 const sqlite3_1 = require("@sequelize/sqlite3");
 const path_1 = __importDefault(require("path"));
 const config_1 = require("../config");
 const job_1 = require("./job");
 const snapshot_1 = require("./snapshot");
-function initDatabase() {
-    return __awaiter(this, void 0, void 0, function* () {
-        const sequelize = new core_1.Sequelize({
-            dialect: sqlite3_1.SqliteDialect,
-            storage: path_1.default.join(config_1.config.dataDir, 'snap-kit.db'),
-            logging: (msg) => process.env.SQLITE_LOG && config_1.logger.debug(msg),
-            pool: {
-                min: 0,
-                max: 10,
-                idle: 10000,
-            },
-            retry: {
-                match: [/SQLITE_BUSY/],
-                name: 'query',
-                max: 10,
-            },
-        });
-        job_1.Job.initModel(sequelize);
-        snapshot_1.Snapshot.initModel(sequelize);
-        try {
-            yield Promise.all([
-                sequelize.query('pragma journal_mode = WAL;'),
-                sequelize.query('pragma synchronous = normal;'),
-                sequelize.query('pragma journal_size_limit = 67108864;'),
-            ]);
-            yield sequelize.authenticate();
-            yield sequelize.sync({ alter: process.env.ALTER_SQLITE === 'true' });
-            config_1.logger.info('Successfully connected to database');
-        }
-        catch (error) {
-            config_1.logger.error('Failed to connect to database:', error);
-            throw error;
-        }
-        return sequelize;
-    });
-}
+const sequelize = new core_1.Sequelize({
+    dialect: sqlite3_1.SqliteDialect,
+    storage: path_1.default.join(config_1.config.dataDir, 'snap-kit.db'),
+    logging: (msg) => process.env.SQLITE_LOG && config_1.logger.debug(msg),
+    pool: {
+        min: 0,
+        max: 10,
+        idle: 10000,
+    },
+    retry: {
+        match: [/SQLITE_BUSY/],
+        name: 'query',
+        max: 10,
+    },
+});
+exports.sequelize = sequelize;
+sequelize.query('pragma journal_mode = WAL;');
+sequelize.query('pragma synchronous = normal;');
+sequelize.query('pragma journal_size_limit = 67108864;');
+job_1.Job.initModel(sequelize);
+snapshot_1.Snapshot.initModel(sequelize);
+__exportStar(require("./job"), exports);
+__exportStar(require("./snapshot"), exports);

package/lib/cjs/store/job.d.ts

@@ -12,9 +12,14 @@ export interface JobState {
     timeout?: number;
     fullPage?: boolean;
     lastModified?: string;
+    waitTime?: number;
+    replace?: boolean;
     headers?: Record<string, string>;
     cookies?: CookieParam[];
-    localStorage?: Record<string, string>;
+    localStorage?: {
+        key: string;
+        value: string;
+    }[];
 }
 export interface JobModel {
     id: string;

package/lib/cjs/store/migrate.d.ts

@@ -0,0 +1,4 @@
+import { Umzug } from 'umzug';
+declare const umzug: Umzug<import("@sequelize/sqlite3").SqliteQueryInterface<import("@sequelize/sqlite3").SqliteDialect>>;
+export declare function migrate(): Promise<import("umzug").MigrationMeta[]>;
+export { umzug };