@arcblock/crawler 1.1.5 → 1.2.0

package/lib/cjs/store/migrate.js

@@ -0,0 +1,63 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.umzug = void 0;
+exports.migrate = migrate;
+/* eslint-disable global-require */
+const umzug_1 = require("umzug");
+const index_1 = require("./index");
+const migration20250615 = __importStar(require("./migrations/20250615-genesis"));
+const migration20250616Replace = __importStar(require("./migrations/20250616-replace"));
+const umzug = new umzug_1.Umzug({
+    migrations: [
+        {
+            name: '20250615-genesis',
+            up: ({ context }) => migration20250615.up({ context }),
+            down: ({ context }) => migration20250615.down({ context }),
+        },
+        {
+            name: '20250616-replace',
+            up: ({ context }) => migration20250616Replace.up({ context }),
+            down: ({ context }) => migration20250616Replace.down({ context }),
+        },
+    ],
+    context: index_1.sequelize.getQueryInterface(),
+    storage: new umzug_1.SequelizeStorage({ sequelize: index_1.sequelize }),
+    logger: console,
+});
+exports.umzug = umzug;
+function migrate() {
+    return umzug.up();
+}

package/lib/cjs/store/migrations/20250615-genesis.d.ts

@@ -0,0 +1,6 @@
+export declare function up({ context }: {
+    context: any;
+}): Promise<void>;
+export declare function down({ context }: {
+    context: any;
+}): Promise<void>;

package/lib/cjs/store/migrations/20250615-genesis.js

@@ -0,0 +1,114 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.up = up;
+exports.down = down;
+/* eslint-disable no-console */
+const core_1 = require("@sequelize/core");
+function up(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ context }) {
+        console.log('[20250615-genesis:up] Migrating...');
+        yield context.createTable('snap', {
+            jobId: {
+                type: core_1.DataTypes.STRING,
+                primaryKey: true,
+                allowNull: false,
+            },
+            url: {
+                type: core_1.DataTypes.STRING,
+                allowNull: false,
+                index: true,
+            },
+            status: {
+                type: core_1.DataTypes.ENUM('success', 'failed', 'pending'),
+                allowNull: false,
+            },
+            html: {
+                type: core_1.DataTypes.TEXT,
+                allowNull: true,
+            },
+            screenshot: {
+                type: core_1.DataTypes.STRING,
+                allowNull: true,
+            },
+            error: {
+                type: core_1.DataTypes.STRING,
+                allowNull: true,
+            },
+            lastModified: {
+                type: core_1.DataTypes.STRING,
+                allowNull: true,
+            },
+            meta: {
+                type: core_1.DataTypes.JSON,
+                allowNull: true,
+            },
+            options: {
+                type: core_1.DataTypes.JSON,
+                allowNull: true,
+            },
+            createdAt: {
+                type: core_1.DataTypes.DATE,
+                defaultValue: core_1.DataTypes.NOW,
+            },
+            updatedAt: {
+                type: core_1.DataTypes.DATE,
+                defaultValue: core_1.DataTypes.NOW,
+            },
+        });
+        yield context.createTable('jobs', {
+            id: {
+                type: core_1.DataTypes.STRING(40),
+                primaryKey: true,
+            },
+            queue: {
+                type: core_1.DataTypes.STRING(32),
+                allowNull: false,
+            },
+            job: {
+                type: core_1.DataTypes.JSON,
+                allowNull: false,
+            },
+            retryCount: {
+                type: core_1.DataTypes.INTEGER,
+            },
+            delay: {
+                type: core_1.DataTypes.INTEGER,
+            },
+            willRunAt: {
+                type: core_1.DataTypes.INTEGER,
+            },
+            cancelled: {
+                type: core_1.DataTypes.BOOLEAN,
+                defaultValue: false,
+            },
+            createdAt: {
+                type: core_1.DataTypes.DATE,
+                defaultValue: core_1.DataTypes.NOW,
+                index: true,
+            },
+            updatedAt: {
+                type: core_1.DataTypes.DATE,
+                defaultValue: core_1.DataTypes.NOW,
+                index: true,
+            },
+        });
+        console.log('[20250615-genesis:up] Migrated successfully!');
+    });
+}
+function down(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ context }) {
+        console.log('[20250615-genesis:down] Migrating...');
+        yield context.dropTable('snap');
+        yield context.dropTable('jobs');
+        console.log('[20250615-genesis:down] Migrated successfully!');
+    });
+}

package/lib/cjs/store/migrations/20250616-replace.d.ts

@@ -0,0 +1,6 @@
+export declare function up({ context }: {
+    context: any;
+}): Promise<void>;
+export declare function down({ context }: {
+    context: any;
+}): Promise<void>;

package/lib/cjs/store/migrations/20250616-replace.js

@@ -0,0 +1,40 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.up = up;
+exports.down = down;
+/* eslint-disable no-console */
+const core_1 = require("@sequelize/core");
+function up(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ context }) {
+        console.log('[20250616-replace:up] Migrating...');
+        yield context.addColumn('snap', 'replace', {
+            type: core_1.DataTypes.BOOLEAN,
+            allowNull: false,
+            defaultValue: false,
+            index: true,
+        });
+        yield context.addIndex('snap', ['createdAt']);
+        yield context.addIndex('snap', ['updatedAt']);
+        yield context.addIndex('snap', ['status']);
+        console.log('[20250616-replace:up] Migrated successfully!');
+    });
+}
+function down(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ context }) {
+        console.log('[20250616-replace:down] Migrating...');
+        yield context.removeColumn('snap', 'replace');
+        yield context.removeIndex('snap', ['createdAt']);
+        yield context.removeIndex('snap', ['updatedAt']);
+        yield context.removeIndex('snap', ['status']);
+        console.log('[20250616-replace:down] Migrated successfully!');
+    });
+}

package/lib/cjs/store/snapshot.d.ts

@@ -1,3 +1,4 @@
+import { CookieParam } from '@blocklet/puppeteer';
 import { FindOptions, Model, Sequelize } from '@sequelize/core';
 export interface SnapshotModel {
     jobId: string;
@@ -7,6 +8,7 @@ export interface SnapshotModel {
     screenshot?: string | null;
     error?: string;
     lastModified?: string;
+    replace?: boolean;
     meta?: {
         title?: string;
         description?: string;
@@ -19,6 +21,11 @@ export interface SnapshotModel {
         quality?: number;
         fullPage?: boolean;
         headers?: Record<string, string>;
+        cookies?: CookieParam[];
+        localStorage?: {
+            key: string;
+            value: string;
+        }[];
     };
 }
 export declare class Snapshot extends Model<SnapshotModel> implements SnapshotModel {
@@ -29,6 +36,7 @@ export declare class Snapshot extends Model<SnapshotModel> implements SnapshotMo
     screenshot?: SnapshotModel['screenshot'];
     error?: SnapshotModel['error'];
     lastModified?: SnapshotModel['lastModified'];
+    replace?: SnapshotModel['replace'];
     meta?: SnapshotModel['meta'];
     options: SnapshotModel['options'];
     static initModel(sequelize: Sequelize): typeof Snapshot;

package/lib/cjs/store/snapshot.js

@@ -27,6 +27,7 @@ class Snapshot extends core_1.Model {
             status: {
                 type: core_1.DataTypes.ENUM('success', 'failed', 'pending'),
                 allowNull: false,
+                index: true,
             },
             html: {
                 type: core_1.DataTypes.TEXT,
@@ -44,6 +45,12 @@ class Snapshot extends core_1.Model {
                 type: core_1.DataTypes.STRING,
                 allowNull: true,
             },
+            replace: {
+                type: core_1.DataTypes.BOOLEAN,
+                allowNull: false,
+                defaultValue: false,
+                index: true,
+            },
             meta: {
                 type: core_1.DataTypes.JSON,
                 allowNull: true,

package/lib/esm/config.d.ts

@@ -1,3 +1,4 @@
+import { CookieParam } from '@blocklet/puppeteer';
 export type Site = {
     url: string;
     pathname: string;
@@ -11,14 +12,19 @@ export type Config = {
     appUrl: string;
     cacheDir: string;
     puppeteerPath?: string;
-    siteCron: {
+    concurrency: number;
+    siteCron?: {
         sites: Site[];
         time: string;
         enabled: boolean;
         immediate: boolean;
-        crawlConcurrency: number;
-        sitemapConcurrency: number;
+        concurrency: number;
     };
+    cookies?: CookieParam[];
+    localStorage?: {
+        key: string;
+        value: string;
+    }[];
 };
 export declare const logger: any;
 export declare const config: Config;

package/lib/esm/config.js

@@ -3,17 +3,9 @@ export const logger = createLogger('@arcblock/crawler', { level: process.env.LOG
 export const config = {
     isProd: process.env.NODE_ENV === 'production',
     dataDir: process.env.BLOCKLET_DATA_DIR,
-    appDir: process.env.BLOCKLET_APP_DIR || process.cwd(),
     cacheDir: process.env.BLOCKLET_CACHE_DIR || process.cwd(),
+    appDir: process.env.BLOCKLET_APP_DIR || process.cwd(),
     appUrl: process.env.BLOCKLET_APP_URL || '/',
     puppeteerPath: process.env.PUPPETEER_EXECUTABLE_PATH,
-    // cron
-    siteCron: {
-        sites: [],
-        enabled: true,
-        time: '0 0 0 * * *',
-        immediate: false,
-        crawlConcurrency: 2,
-        sitemapConcurrency: 30,
-    },
+    concurrency: 2,
 };

package/lib/esm/crawler.d.ts

@@ -1,11 +1,10 @@
-import { JobState } from './store/job';
-import { SnapshotModel } from './store/snapshot';
-export declare function createCrawlQueue(): void;
+import { JobState, SnapshotModel } from './store';
+export declare function createCrawlQueue(queue: string): any;
 export declare function getDataDir(): Promise<{
     htmlDir: string;
     screenshotDir: string;
 }>;
-export declare const getPageContent: ({ url, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage, headers, cookies, localStorage, }: JobState) => Promise<{
+export declare const getPageContent: ({ url, includeScreenshot, includeHtml, width, height, quality, timeout, waitTime, fullPage, headers, cookies, localStorage, }: JobState) => Promise<{
     html: string | null;
     screenshot: Uint8Array<ArrayBufferLike> | null;
     meta: {

package/lib/esm/crawler.js

@@ -14,17 +14,17 @@ import fs from 'fs-extra';
 import path from 'path';
 import { config, logger } from './config';
 import { initPage } from './puppeteer';
-import { convertJobToSnapshot, formatSnapshot } from './services/snapshot';
-import { Job } from './store/job';
-import { Snapshot } from './store/snapshot';
-import { findMaxScrollHeight, formatUrl, isAcceptCrawler, md5 } from './utils';
+import { convertJobToSnapshot, deleteSnapshots, formatSnapshot } from './services/snapshot';
+import { Job, Snapshot, sequelize } from './store';
+import { findMaxScrollHeight, formatUrl, isAcceptCrawler, md5, sleep } from './utils';
 const { BaseState } = require('@abtnode/models');
-let crawlQueue;
-export function createCrawlQueue() {
+// eslint-disable-next-line import/no-mutable-exports
+const crawlQueue = createCrawlQueue('urlCrawler');
+export function createCrawlQueue(queue) {
     const db = new BaseState(Job);
-    crawlQueue = createQueue({
-        store: new SequelizeStore(db, 'crawler'),
-        concurrency: config.siteCron.crawlConcurrency,
+    return createQueue({
+        store: new SequelizeStore(db, queue),
+        concurrency: config.concurrency,
         onJob: (job) => __awaiter(this, void 0, void 0, function* () {
             logger.info('Starting to execute crawl job', job);
             const canCrawl = yield isAcceptCrawler(job.url);
@@ -48,18 +48,14 @@ export function createCrawlQueue() {
             // } catch (error) {
             //   logger.error('failed to close browser when queue index reached autoCloseBrowserCount:', error);
             // }
+            const formattedJob = Object.assign(Object.assign({}, job), { cookies: (config.cookies || []).concat(job.cookies || []), localStorage: (config.localStorage || []).concat(job.localStorage || []), url: formatUrl(job.url) });
             try {
                 // get page content later
-                const result = yield getPageContent(Object.assign({ localStorage: {
-                        // for blocklet theme
-                        blocklet_theme_prefer: 'light',
-                        // for blocklet domain warning
-                        'domain-warning-skip': Date.now().toString(),
-                    } }, job));
+                const result = yield getPageContent(formattedJob);
                 if (!result || (!result.html && !result.screenshot)) {
-                    logger.error(`failed to crawl ${job.url}, empty content`, job);
+                    logger.error(`failed to crawl ${formattedJob.url}, empty content`, formattedJob);
                     const snapshot = convertJobToSnapshot({
-                        job,
+                        job: formattedJob,
                         snapshot: {
                             status: 'failed',
                             error: 'Failed to crawl content',
@@ -68,28 +64,45 @@ export function createCrawlQueue() {
                     yield Snapshot.upsert(snapshot);
                     return snapshot;
                 }
-                // save html and screenshot to data dir
-                const { screenshotPath, htmlPath } = yield saveSnapshotToLocal({
-                    screenshot: result.screenshot,
-                    html: result.html,
-                });
-                // const lastModified = job.lastmodMap?.get(url) || new Date().toISOString();
-                const snapshot = convertJobToSnapshot({
-                    job,
-                    snapshot: {
-                        status: 'success',
-                        screenshot: screenshotPath === null || screenshotPath === void 0 ? void 0 : screenshotPath.replace(config.dataDir, ''),
-                        html: htmlPath === null || htmlPath === void 0 ? void 0 : htmlPath.replace(config.dataDir, ''),
-                        meta: result.meta,
-                    },
-                });
-                yield Snapshot.upsert(snapshot);
+                const snapshot = yield sequelize.transaction((txn) => __awaiter(this, void 0, void 0, function* () {
+                    // delete old snapshot
+                    if (formattedJob.replace) {
+                        try {
+                            const deletedJobIds = yield deleteSnapshots({
+                                url: formattedJob.url,
+                                replace: true,
+                            }, { txn });
+                            if (deletedJobIds) {
+                                logger.info('Deleted old snapshot', { deletedJobIds });
+                            }
+                        }
+                        catch (error) {
+                            logger.error('Failed to delete old snapshot', { error, formattedJob });
+                        }
+                    }
+                    // save html and screenshot to data dir
+                    const { screenshotPath, htmlPath } = yield saveSnapshotToLocal({
+                        screenshot: result.screenshot,
+                        html: result.html,
+                    });
+                    const snapshot = convertJobToSnapshot({
+                        job: formattedJob,
+                        snapshot: {
+                            status: 'success',
+                            screenshot: screenshotPath === null || screenshotPath === void 0 ? void 0 : screenshotPath.replace(config.dataDir, ''),
+                            html: htmlPath === null || htmlPath === void 0 ? void 0 : htmlPath.replace(config.dataDir, ''),
+                            meta: result.meta,
+                        },
+                    });
+                    yield Snapshot.upsert(snapshot, { transaction: txn });
+                    return snapshot;
+                }));
                 return snapshot;
             }
             catch (error) {
-                logger.error(`Failed to crawl ${job.url}`, { error, job });
+                logger.error(`Failed to crawl ${formattedJob.url}`, { error, formattedJob });
                 const snapshot = convertJobToSnapshot({
-                    job,
+                    job: formattedJob,
                     snapshot: {
                         status: 'failed',
                         error: 'Internal error',
@@ -133,7 +146,7 @@ function saveSnapshotToLocal(_a) {
         };
     });
 }
-export const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, timeout = 90 * 1000, fullPage = false, headers, cookies = [], localStorage, }) {
+export const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, timeout = 90 * 1000, waitTime = 0, fullPage = false, headers, cookies, localStorage, }) {
     const page = yield initPage();
     if (width && height) {
         yield page.setViewport({ width, height, deviceScaleFactor: 2 });
@@ -141,13 +154,18 @@ export const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function*
     if (headers) {
         yield page.setExtraHTTPHeaders(headers);
     }
-    if (cookies === null || cookies === void 0 ? void 0 : cookies.length) {
-        yield page.setCookie(...cookies);
+    // handle cookies
+    if (cookies) {
+        const { hostname } = new URL(url);
+        const cookieParams = cookies.map((item) => (Object.assign(Object.assign({}, item), { expires: item.expires ? new Date(item.expires).getTime() : undefined, domain: item.domain || hostname, path: item.path || '/' })));
+        yield page.setCookie(...cookieParams);
     }
+    // handle localStorage
     if (localStorage) {
         yield page.evaluateOnNewDocument((items) => {
-            Object.entries(items).forEach(([key, value]) => {
-                window.localStorage.setItem(key, value);
+            items.forEach((item) => {
+                const value = item.value === 'now()' ? new Date().toISOString() : item.value;
+                window.localStorage.setItem(item.key, value);
             });
         }, localStorage);
     }
@@ -165,9 +183,18 @@ export const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function*
         }
         // await for networkidle0
         // https://pptr.dev/api/puppeteer.page.waitfornetworkidle
-        yield page.waitForNetworkIdle({
-            idleTime: 1.5 * 1000,
-        });
+        try {
+            yield Promise.all([
+                page.waitForNetworkIdle({
+                    idleTime: 1.5 * 1000,
+                    timeout,
+                }),
+                sleep(waitTime),
+            ]);
+        }
+        catch (err) {
+            logger.warn(`Failed to wait for network idle in ${url}:`, err);
+        }
         // get screenshot
         if (includeScreenshot) {
             // Try to find the tallest element and set the browser to the same height
@@ -211,7 +238,7 @@ export const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function*
             // check if the page is an error page
             const isErrorPage = ['<h2>Unexpected Application Error!</h2>', 'Current route occurred an error'].some((errorHtml) => data.html.includes(errorHtml));
             if (isErrorPage) {
-                throw new Error('Page is an error page');
+                throw new Error(`${url} is an error page`);
             }
             meta.title = data.title;
             meta.description = data.description;
@@ -245,7 +272,6 @@ export const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function*
 // eslint-disable-next-line require-await
 export function crawlUrl(params, callback) {
     return __awaiter(this, void 0, void 0, function* () {
-        params = Object.assign(Object.assign({}, params), { url: formatUrl(params.url) });
         // skip duplicate job
         const existsJob = yield Job.isExists(params);
         if (existsJob) {

package/lib/esm/cron.js

@@ -14,6 +14,8 @@ let cron = null;
 export function initCron() {
     if (cron)
         return;
+    if (!config.siteCron)
+        return;
     logger.info('Init cron', { config: config.siteCron });
     cron = Cron.init({
         context: {},
@@ -23,6 +25,9 @@ export function initCron() {
                 time: config.siteCron.time,
                 options: { runOnInit: config.siteCron.immediate },
                 fn: () => __awaiter(this, void 0, void 0, function* () {
+                    var _a;
+                    if (!((_a = config.siteCron) === null || _a === void 0 ? void 0 : _a.enabled))
+                        return;
                     logger.info('Start cron to crawl site', { sites: config.siteCron.sites });
                     for (const site of config.siteCron.sites) {
                         try {

package/lib/esm/index.d.ts

@@ -3,7 +3,5 @@ export * from './crawler';
 export * from './site';
 export * from './services/snapshot';
 export * as utils from './utils';
-type DeepPartial<T> = T extends object ? {
-    [P in keyof T]?: DeepPartial<T[P]>;
-} : T;
-export declare function initCrawler(params: DeepPartial<Pick<Config, 'puppeteerPath' | 'siteCron'>>): Promise<void>;
+export { migrate } from './store/migrate';
+export declare function initCrawler(params: Pick<Config, 'puppeteerPath' | 'siteCron' | 'cookies' | 'localStorage' | 'concurrency'>): Promise<void>;

package/lib/esm/index.js

@@ -7,25 +7,24 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+/* eslint-disable @typescript-eslint/indent */
 import merge from 'lodash/merge';
 import { config, logger } from './config';
-import { createCrawlQueue } from './crawler';
 import { initCron } from './cron';
 import { ensureBrowser } from './puppeteer';
-import { initDatabase } from './store';
 export * from './crawler';
 export * from './site';
 export * from './services/snapshot';
 export * as utils from './utils';
+export { migrate } from './store/migrate';
 export function initCrawler(params) {
     return __awaiter(this, void 0, void 0, function* () {
+        var _a;
         merge(config, params);
         logger.info('Init crawler', { params, config });
         try {
-            yield initDatabase();
             yield ensureBrowser();
-            yield createCrawlQueue();
-            if (config.siteCron.enabled) {
+            if ((_a = config.siteCron) === null || _a === void 0 ? void 0 : _a.enabled) {
                 yield initCron();
             }
         }

package/lib/esm/services/snapshot.d.ts

@@ -1,5 +1,5 @@
-import { JobState } from '../store/job';
-import { SnapshotModel } from '../store/snapshot';
+import { Transaction, WhereOptions } from '@sequelize/core';
+import { JobState, SnapshotModel } from '../store';
 export declare function convertJobToSnapshot({ job, snapshot }: {
     job: JobState;
     snapshot?: Partial<SnapshotModel>;
@@ -10,3 +10,6 @@ export declare function formatSnapshot(snapshot: SnapshotModel, columns?: Array<
  */
 export declare function getSnapshot(jobId: string): Promise<SnapshotModel | null>;
 export declare function getLatestSnapshot(url: string): Promise<SnapshotModel | null>;
+export declare function deleteSnapshots(where: WhereOptions<SnapshotModel>, { txn }?: {
+    txn?: Transaction;
+}): Promise<string[]>;