@arcblock/crawler 1.0.5 → 1.1.1

package/{dist → lib/cjs}/puppeteer.js

@@ -51,33 +51,28 @@ exports.ensureBrowser = ensureBrowser;
 exports.connectBrowser = connectBrowser;
 exports.launchBrowser = launchBrowser;
 exports.initPage = initPage;
-// import fs from 'fs-extra';
-// import path from 'path';
 const puppeteer_1 = __importDefault(require("@blocklet/puppeteer"));
 exports.puppeteer = puppeteer_1.default;
-const config_1 = require("@blocklet/sdk/lib/config");
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const path_1 = __importDefault(require("path"));
 const timers_1 = require("timers");
-const cache_1 = require("./cache");
-const config_2 = require("./config");
+const config_1 = require("./config");
 const utils_1 = require("./utils");
-// let puppeteerConfig: {
-//   cacheDirectory: string;
-//   temporaryDirectory: string;
-// };
-const BROWSER_WS_ENDPOINT_KEY = `browserWSEndpoint-${config_1.env.appId || 'unknown'}`;
 const BrowserStatus = {
+    None: 'None',
     Launching: 'Launching',
     Ready: 'Ready',
 };
+let browserStatus = BrowserStatus.None;
+/** Chromium WebSocket endpoint that allows puppeteer browser instance to connect to the browser */
+let browserEndpoint = '';
 let browser;
 let browserActivatedTimer;
 function ensurePuppeteerrc() {
     return __awaiter(this, void 0, void 0, function* () {
-        const cacheDirectory = path_1.default.join(config_2.config.cacheDir, 'puppeteer', 'cache');
-        const temporaryDirectory = path_1.default.join(config_2.config.cacheDir, 'puppeteer', 'tmp');
-        const puppeteerrcPath = path_1.default.join(config_2.config.appDir, '.puppeteerrc.js');
+        const cacheDirectory = path_1.default.join(config_1.config.cacheDir, 'puppeteer', 'cache');
+        const temporaryDirectory = path_1.default.join(config_1.config.cacheDir, 'puppeteer', 'tmp');
+        const puppeteerrcPath = path_1.default.join(config_1.config.appDir, '.puppeteerrc.js');
         // ensure directory exists
         yield Promise.all([fs_extra_1.default.ensureDir(cacheDirectory), fs_extra_1.default.ensureDir(temporaryDirectory), fs_extra_1.default.ensureFile(puppeteerrcPath)]);
         const puppeteerConfig = {
@@ -86,17 +81,17 @@ function ensurePuppeteerrc() {
         };
         const fileContent = `module.exports = ${JSON.stringify(puppeteerConfig, null, 2)}`;
         yield fs_extra_1.default.writeFile(puppeteerrcPath, fileContent);
-        config_2.logger.debug(`Puppeteerrc file created at ${puppeteerrcPath}`, puppeteerConfig);
+        config_1.logger.debug(`Puppeteerrc file created at ${puppeteerrcPath}`, puppeteerConfig);
         return puppeteerConfig;
     });
 }
 function ensureBrowser() {
     return __awaiter(this, void 0, void 0, function* () {
         const puppeteerConfig = yield ensurePuppeteerrc();
-        const executablePath = process.env.PUPPETEER_EXECUTABLE_PATH || '/usr/bin/chromium';
-        config_2.logger.info('executablePath', executablePath);
-        if (!fs_extra_1.default.existsSync(executablePath)) {
-            config_2.logger.info('start download browser', puppeteerConfig);
+        const executablePath = config_1.config.puppeteerPath;
+        config_1.logger.debug('executablePath', executablePath);
+        if (!executablePath || !fs_extra_1.default.existsSync(executablePath)) {
+            config_1.logger.info('start download browser', puppeteerConfig);
             const { downloadBrowser } = yield (() => __awaiter(this, void 0, void 0, function* () {
                 try {
                     // @ts-ignore
@@ -104,45 +99,44 @@ function ensureBrowser() {
                     return yield Promise.resolve().then(() => __importStar(require('@blocklet/puppeteer/internal/node/install.js')));
                 }
                 catch (err) {
-                    config_2.logger.warn('Skipping browser installation because the Puppeteer build is not available. Run `npm install` again after you have re-built Puppeteer.');
+                    config_1.logger.warn('Skipping browser installation because the Puppeteer build is not available. Run `npm install` again after you have re-built Puppeteer.');
                 }
             }))();
             if (downloadBrowser) {
                 yield downloadBrowser();
-                config_2.logger.info('Browser download completed successfully');
+                config_1.logger.info('Browser download completed successfully');
             }
         }
         // try to launch browser
-        if (config_2.config.testOnInitialize) {
+        if (config_1.config.isProd) {
             const browser = yield launchBrowser();
             if (!browser) {
                 throw new Error('Failed to launch browser');
             }
             yield (0, exports.closeBrowser)();
         }
-        config_2.logger.info('Puppeteer is ready');
+        config_1.logger.info('Puppeteer is ready');
     });
 }
 function connectBrowser() {
     return __awaiter(this, void 0, void 0, function* () {
-        const browserWSEndpoint = yield cache_1.useCache.get(BROWSER_WS_ENDPOINT_KEY);
-        if (!browserWSEndpoint) {
+        if (!browserEndpoint) {
             return null;
         }
         // retry if browser is launching
-        if (browserWSEndpoint.status === BrowserStatus.Launching) {
+        if (browserStatus === BrowserStatus.Launching) {
             yield (0, utils_1.sleep)(Math.floor(Math.random() * 1000));
             return connectBrowser();
         }
         try {
             browser = yield puppeteer_1.default.connect({
-                browserWSEndpoint: browserWSEndpoint.endpoint,
+                browserWSEndpoint: browserEndpoint,
             });
-            config_2.logger.info('Connect browser success');
+            config_1.logger.info('Connect browser success');
         }
         catch (err) {
-            config_2.logger.warn('Connect browser failed, clear endpoint', err);
-            yield cache_1.useCache.remove(BROWSER_WS_ENDPOINT_KEY);
+            config_1.logger.warn('Connect browser failed, clear endpoint', err);
+            browserEndpoint = '';
             return null;
         }
         return browser;
@@ -150,12 +144,9 @@ function connectBrowser() {
 }
 function launchBrowser() {
     return __awaiter(this, void 0, void 0, function* () {
-        yield cache_1.useCache.set(BROWSER_WS_ENDPOINT_KEY, {
-            endpoint: null,
-            status: BrowserStatus.Launching,
-        });
+        browserEndpoint = '';
+        browserStatus = BrowserStatus.Launching;
         try {
-            // @ts-ignore
             browser = yield puppeteer_1.default.launch({
                 headless: true,
                 args: [
@@ -182,20 +173,17 @@ function launchBrowser() {
                     '--font-render-hinting=none',
                 ],
             });
-            config_2.logger.info('Launch browser success');
+            config_1.logger.info('Launch browser');
         }
         catch (error) {
-            config_2.logger.error('launch browser failed: ', error);
-            // cleanup browser endpoint
-            yield cache_1.useCache.remove(BROWSER_WS_ENDPOINT_KEY);
+            config_1.logger.error('launch browser failed: ', error);
+            browserStatus = BrowserStatus.None;
+            browserEndpoint = '';
             throw error;
         }
         // save browserWSEndpoint to cache
-        const endpoint = yield browser.wsEndpoint();
-        yield cache_1.useCache.set(BROWSER_WS_ENDPOINT_KEY, {
-            endpoint,
-            status: BrowserStatus.Ready,
-        });
+        browserEndpoint = yield browser.wsEndpoint();
+        browserStatus = BrowserStatus.Ready;
         return browser;
     });
 }
@@ -208,13 +196,13 @@ function checkBrowserActivated() {
             const pages = yield browser.pages().catch(() => []);
             if (pages.length === 1 && ((_a = pages[0]) === null || _a === void 0 ? void 0 : _a.url()) === 'about:blank') {
                 count++;
-                config_2.logger.debug(`Browser inactive count: ${count}/3`);
+                config_1.logger.debug(`Browser inactive count: ${count}/3`);
             }
             else {
                 count = 0; // 重置计数器！
             }
             if (count >= 3) {
-                config_2.logger.info('Browser inactive for 3 minutes, closing...');
+                config_1.logger.info('Browser inactive for 3 minutes, closing...');
                 yield (0, exports.closeBrowser)({
                     trimCache: true,
                 });
@@ -236,14 +224,15 @@ const getBrowser = () => __awaiter(void 0, void 0, void 0, function* () {
     // try to connect browser
     const connectedBrowser = yield connectBrowser();
     if (connectedBrowser) {
-        config_2.logger.debug('getBrowser.connectedBrowser');
+        config_1.logger.debug('getBrowser.connectedBrowser');
         browser = connectedBrowser;
+        checkBrowserActivated();
         return browser;
     }
     // try to launch browser
     const launchedBrowser = yield launchBrowser();
     if (launchedBrowser) {
-        config_2.logger.debug('getBrowser.launchedBrowser');
+        config_1.logger.debug('getBrowser.launchedBrowser');
         browser = launchedBrowser;
         checkBrowserActivated();
         return browser;
@@ -260,20 +249,20 @@ const closeBrowser = (...args_1) => __awaiter(void 0, [...args_1], void 0, funct
         yield Promise.all(pages.map((page) => page.close()));
     }
     catch (err) {
-        config_2.logger.error('Failed to close all pages:', err);
+        config_1.logger.warn('Failed to close all pages:', err);
     }
     // close browser
     try {
         yield browser.close();
     }
     catch (err) {
-        config_2.logger.error('Failed to close browser:', err);
+        config_1.logger.warn('Failed to close browser:', err);
     }
     // clear cache
     try {
         if (trimCache) {
             yield puppeteer_1.default.trimCache();
-            config_2.logger.info('Trim cache success');
+            config_1.logger.debug('Trim cache success');
         }
         // try to clear temporary directory
         // if (puppeteerConfig) {
@@ -284,12 +273,13 @@ const closeBrowser = (...args_1) => __awaiter(void 0, [...args_1], void 0, funct
         }
     }
     catch (err) {
-        config_2.logger.error('Failed to clear browser cache:', err);
+        config_1.logger.warn('Failed to clear browser cache:', err);
     }
     browser = null;
     clearBrowserActivatedTimer();
-    yield cache_1.useCache.remove(BROWSER_WS_ENDPOINT_KEY);
-    config_2.logger.info('Close browser success');
+    browserEndpoint = '';
+    browserStatus = BrowserStatus.None;
+    config_1.logger.info('Close browser success');
 });
 exports.closeBrowser = closeBrowser;
 function initPage() {
@@ -306,7 +296,6 @@ function initPage() {
         if (abortResourceTypes.length > 0) {
             yield page.setRequestInterception(true);
             page.on('request', (req) => {
-                // @ts-ignore
                 if (abortResourceTypes.includes(req.resourceType())) {
                     return req.abort();
                 }

package/lib/cjs/services/snapshot.d.ts

@@ -0,0 +1,12 @@
+import { JobState } from '../store/job';
+import { SnapshotModel } from '../store/snapshot';
+export declare function convertJobToSnapshot({ job, snapshot }: {
+    job: JobState;
+    snapshot?: Partial<SnapshotModel>;
+}): SnapshotModel;
+export declare function formatSnapshot(snapshot: SnapshotModel, columns?: Array<keyof SnapshotModel>): Promise<SnapshotModel>;
+/**
+ * get snapshot from db or crawl queue
+ */
+export declare function getSnapshot(jobId: string): Promise<SnapshotModel | null>;
+export declare function getLatestSnapshot(url: string): Promise<SnapshotModel | null>;

package/lib/cjs/services/snapshot.js

@@ -0,0 +1,84 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.convertJobToSnapshot = convertJobToSnapshot;
+exports.formatSnapshot = formatSnapshot;
+exports.getSnapshot = getSnapshot;
+exports.getLatestSnapshot = getLatestSnapshot;
+const pick_1 = __importDefault(require("lodash/pick"));
+const promises_1 = __importDefault(require("node:fs/promises"));
+const node_path_1 = __importDefault(require("node:path"));
+const ufo_1 = require("ufo");
+const config_1 = require("../config");
+const job_1 = require("../store/job");
+const snapshot_1 = require("../store/snapshot");
+const utils_1 = require("../utils");
+function convertJobToSnapshot({ job, snapshot }) {
+    return Object.assign({ jobId: job.jobId || job.id, url: job.url, lastModified: job.lastModified || new Date().toISOString(), options: {
+            width: job.width,
+            height: job.height,
+            includeScreenshot: job.includeScreenshot,
+            includeHtml: job.includeHtml,
+            quality: job.quality,
+            fullPage: job.fullPage,
+        } }, snapshot);
+}
+function formatSnapshot(snapshot, columns) {
+    return __awaiter(this, void 0, void 0, function* () {
+        let data = Object.assign({}, snapshot);
+        // format screenshot path to full url
+        if (data.screenshot) {
+            data.screenshot = (0, ufo_1.joinURL)(config_1.config.appUrl, data.screenshot);
+        }
+        // format html path to string
+        if (data.html) {
+            const html = yield promises_1.default.readFile(node_path_1.default.join(config_1.config.dataDir, data.html));
+            data.html = html.toString();
+        }
+        if (columns === null || columns === void 0 ? void 0 : columns.length) {
+            data = (0, pick_1.default)(data, columns);
+        }
+        return data;
+    });
+}
+/**
+ * get snapshot from db or crawl queue
+ */
+function getSnapshot(jobId) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const snapshot = yield snapshot_1.Snapshot.findSnapshot({ where: { jobId } });
+        if (snapshot) {
+            return formatSnapshot(snapshot);
+        }
+        const job = yield job_1.Job.findJob({ id: jobId });
+        if (job) {
+            return {
+                jobId,
+                status: 'pending',
+            };
+        }
+        return null;
+    });
+}
+function getLatestSnapshot(url) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const snapshot = yield snapshot_1.Snapshot.findSnapshot({
+            where: {
+                url: (0, utils_1.formatUrl)(url),
+                status: 'success',
+            },
+        });
+        return snapshot ? formatSnapshot(snapshot) : null;
+    });
+}

package/lib/cjs/site.d.ts

@@ -0,0 +1,2 @@
+import { Site } from './config';
+export declare const crawlSite: ({ url, pathname, interval }: Site) => Promise<(string | undefined)[]>;

package/lib/cjs/site.js

@@ -0,0 +1,76 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.crawlSite = void 0;
+const uniq_1 = __importDefault(require("lodash/uniq"));
+const p_map_1 = __importDefault(require("p-map"));
+const config_1 = require("./config");
+const crawler_1 = require("./crawler");
+const snapshot_1 = require("./store/snapshot");
+const utils_1 = require("./utils");
+const crawlBlockletRunningMap = new Map();
+function parseSitemapUrl(sitemapItem) {
+    var _a;
+    const links = ((_a = sitemapItem.links) === null || _a === void 0 ? void 0 : _a.map((item) => item.url)) || [];
+    const urls = (0, uniq_1.default)([...links, sitemapItem.url]).filter(Boolean);
+    return urls.map((url) => ({ url, sitemapItem }));
+}
+const crawlSite = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, pathname, interval = 0 }) {
+    config_1.logger.info(`Start crawl from sitemap ${url}`, { pathname });
+    const sitemapList = yield (0, utils_1.getSitemapList)(url);
+    const pathnameRegex = new RegExp(pathname);
+    const sitemapItems = sitemapList
+        .filter((item) => new URL(item.url).pathname.match(pathnameRegex))
+        .flatMap((sitemapItem) => {
+        return parseSitemapUrl(sitemapItem);
+    });
+    config_1.logger.info(`Found ${sitemapItems.length} sitemap items which match ${pathname} from ${url}`);
+    const crawlableItems = (yield Promise.all(sitemapItems.map((_a) => __awaiter(void 0, [_a], void 0, function* ({ url, sitemapItem }) {
+        const snapshot = yield snapshot_1.Snapshot.findOne({ where: { url: (0, utils_1.formatUrl)(url) } });
+        if (snapshot === null || snapshot === void 0 ? void 0 : snapshot.lastModified) {
+            const lastModified = new Date(snapshot.lastModified);
+            // skip if snapshot lastModified is greater than sitemap lastmod
+            if (sitemapItem.lastmod && lastModified >= new Date(sitemapItem.lastmod)) {
+                return null;
+            }
+            // skip if interval time has not been reached
+            if (Date.now() - lastModified.getTime() < interval * 24 * 60 * 60 * 1000) {
+                return null;
+            }
+        }
+        return { url, sitemapItem };
+    })))).filter(Boolean);
+    config_1.logger.info(`Found ${crawlableItems.length} pages to crawl from sitemap ${url}`, { pathname });
+    const key = `${url}-${pathname}`;
+    crawlBlockletRunningMap.set(key, crawlableItems);
+    try {
+        const jobIds = yield (0, p_map_1.default)(crawlableItems, ({ url, sitemapItem }) => {
+            return (0, crawler_1.crawlUrl)({
+                url,
+                lastModified: sitemapItem.lastmod,
+                includeScreenshot: false,
+                includeHtml: true,
+            });
+        }, { concurrency: config_1.config.siteCron.concurrency });
+        return jobIds;
+    }
+    catch (error) {
+        config_1.logger.error(`Failed to crawl from sitemap ${url} ${pathname}`, error);
+        throw new Error(error);
+    }
+    finally {
+        crawlBlockletRunningMap.delete(key);
+    }
+});
+exports.crawlSite = crawlSite;

package/lib/cjs/store/index.d.ts

@@ -0,0 +1,3 @@
+import { Sequelize } from '@sequelize/core';
+import { SqliteDialect } from '@sequelize/sqlite3';
+export declare function initDatabase(): Promise<Sequelize<SqliteDialect>>;

package/{dist/db → lib/cjs/store}/index.js

@@ -12,23 +12,38 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ensureDatabase = ensureDatabase;
+exports.initDatabase = initDatabase;
 const core_1 = require("@sequelize/core");
 const sqlite3_1 = require("@sequelize/sqlite3");
 const path_1 = __importDefault(require("path"));
 const config_1 = require("../config");
 const job_1 = require("./job");
 const snapshot_1 = require("./snapshot");
-function ensureDatabase() {
+function initDatabase() {
     return __awaiter(this, void 0, void 0, function* () {
         const sequelize = new core_1.Sequelize({
             dialect: sqlite3_1.SqliteDialect,
             storage: path_1.default.join(config_1.config.dataDir, 'snap-kit.db'),
-            logging: (msg) => config_1.logger.debug(msg),
+            logging: (msg) => process.env.SQLITE_LOG && config_1.logger.debug(msg),
+            pool: {
+                min: 0,
+                max: 10,
+                idle: 10000,
+            },
+            retry: {
+                match: [/SQLITE_BUSY/],
+                name: 'query',
+                max: 10,
+            },
         });
-        yield (0, snapshot_1.initSnapshotModel)(sequelize);
-        yield (0, job_1.initJobModel)(sequelize);
+        job_1.Job.initModel(sequelize);
+        snapshot_1.Snapshot.initModel(sequelize);
         try {
+            yield Promise.all([
+                sequelize.query('pragma journal_mode = WAL;'),
+                sequelize.query('pragma synchronous = normal;'),
+                sequelize.query('pragma journal_size_limit = 67108864;'),
+            ]);
             yield sequelize.authenticate();
             yield sequelize.sync();
             config_1.logger.info('Successfully connected to database');
@@ -37,5 +52,6 @@ function ensureDatabase() {
             config_1.logger.error('Failed to connect to database:', error);
             throw error;
         }
+        return sequelize;
     });
 }

package/{dist/db → lib/cjs/store}/job.d.ts

@@ -10,6 +10,7 @@ export interface JobState {
     quality?: number;
     timeout?: number;
     fullPage?: boolean;
+    lastModified?: string;
 }
 export interface JobModel {
     id: string;
@@ -20,7 +21,7 @@ export interface JobModel {
     delay: number;
     cancelled: boolean;
 }
-declare class Job extends Model<JobModel> implements JobModel {
+export declare class Job extends Model<JobModel> implements JobModel {
     id: JobModel['id'];
     queue: JobModel['queue'];
     job: JobModel['job'];
@@ -28,6 +29,6 @@ declare class Job extends Model<JobModel> implements JobModel {
     willRunAt: JobModel['willRunAt'];
     delay: JobModel['delay'];
     cancelled: JobModel['cancelled'];
+    static initModel(sequelize: Sequelize): typeof Job;
+    static findJob(condition: Partial<JobState>): Promise<JobModel | null>;
 }
-export { Job };
-export declare function initJobModel(sequelize: Sequelize): typeof Job;

package/lib/cjs/store/job.js

@@ -0,0 +1,110 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Job = void 0;
+const core_1 = __importStar(require("@sequelize/core"));
+class Job extends core_1.Model {
+    static initModel(sequelize) {
+        return Job.init({
+            id: {
+                type: core_1.DataTypes.STRING(40),
+                primaryKey: true,
+            },
+            queue: {
+                type: core_1.DataTypes.STRING(32),
+                allowNull: false,
+            },
+            job: {
+                type: core_1.DataTypes.JSON,
+                allowNull: false,
+            },
+            retryCount: {
+                type: core_1.DataTypes.INTEGER,
+            },
+            delay: {
+                type: core_1.DataTypes.INTEGER,
+            },
+            willRunAt: {
+                type: core_1.DataTypes.INTEGER,
+            },
+            cancelled: {
+                type: core_1.DataTypes.BOOLEAN,
+                defaultValue: false,
+            },
+            createdAt: {
+                type: core_1.DataTypes.DATE,
+                defaultValue: core_1.DataTypes.NOW,
+                index: true,
+            },
+            updatedAt: {
+                type: core_1.DataTypes.DATE,
+                defaultValue: core_1.DataTypes.NOW,
+                index: true,
+            },
+        }, {
+            sequelize,
+            indexes: [{ fields: ['queue'] }],
+            modelName: 'job',
+            tableName: 'jobs',
+            timestamps: true,
+        });
+    }
+    static findJob(condition) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const where = Object.keys(condition)
+                .filter((key) => condition[key] !== undefined)
+                .map((key) => {
+                return core_1.default.where(core_1.default.fn('json_extract', core_1.default.col('job'), `$.${key}`), condition[key]);
+            });
+            const job = yield Job.findOne({
+                where: {
+                    [core_1.default.Op.and]: where,
+                },
+                order: [['createdAt', 'DESC']],
+            });
+            return (job === null || job === void 0 ? void 0 : job.toJSON()) || null;
+        });
+    }
+}
+exports.Job = Job;

package/{dist/db → lib/cjs/store}/snapshot.d.ts

@@ -1,5 +1,5 @@
-import { Model, Sequelize } from '@sequelize/core';
-interface SnapshotModel {
+import { FindOptions, Model, Sequelize } from '@sequelize/core';
+export interface SnapshotModel {
     jobId: string;
     url: string;
     status: 'success' | 'failed' | 'pending';
@@ -16,7 +16,7 @@ interface SnapshotModel {
         fullPage?: boolean;
     };
 }
-declare class Snapshot extends Model<SnapshotModel> implements SnapshotModel {
+export declare class Snapshot extends Model<SnapshotModel> implements SnapshotModel {
     jobId: SnapshotModel['jobId'];
     url: SnapshotModel['url'];
     status: SnapshotModel['status'];
@@ -25,7 +25,6 @@ declare class Snapshot extends Model<SnapshotModel> implements SnapshotModel {
     error?: SnapshotModel['error'];
     lastModified?: SnapshotModel['lastModified'];
     options: SnapshotModel['options'];
+    static initModel(sequelize: Sequelize): typeof Snapshot;
+    static findSnapshot(condition: FindOptions<SnapshotModel>): Promise<SnapshotModel | null>;
 }
-export { Snapshot };
-export type { SnapshotModel };
-export declare function initSnapshotModel(sequelize: Sequelize): typeof Snapshot;