@arcblock/crawler 1.4.7 → 1.5.0

package/lib/cjs/crawler.d.ts

@@ -1,10 +1,11 @@
 import { Page } from '@blocklet/puppeteer';
 import { JobState, SnapshotModel } from './store';
-declare let crawlQueue: any;
-declare let syncQueue: any;
-declare let codeQueue: any;
-declare let cronQueue: any;
-export { crawlQueue, syncQueue, codeQueue, cronQueue };
+export declare const queueMap: {
+    urlCrawler: any;
+    syncCrawler: any;
+    codeCrawler: any;
+    cronJobs: any;
+};
 export declare function initQueue(): void;
 type PageHandler = {
     handleScreenshot?: (page: Page, params?: JobState) => Promise<Buffer | null>;
@@ -28,6 +29,7 @@ export declare const getPageContent: ({ url, includeScreenshot, includeHtml, wid
  * @param params
  * @param callback callback when job finished
  */
-export declare function enqueue(queue: any, params: Omit<JobState, 'jobId'>, callback?: (snapshot: SnapshotModel | null) => void): Promise<string>;
+export declare function enqueue(queueName: keyof typeof queueMap, params: Omit<JobState, 'jobId'>, callback?: (snapshot: SnapshotModel | null) => void): Promise<string>;
 export declare function crawlUrl(params: Omit<JobState, 'jobId'>, callback?: (snapshot: SnapshotModel | null) => void): Promise<string>;
 export declare function crawlCode(params: Omit<JobState, 'jobId'>, callback?: (snapshot: SnapshotModel | null) => void): Promise<string>;
+export {};

package/lib/cjs/crawler.js

@@ -12,7 +12,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getPageContent = exports.cronQueue = exports.codeQueue = exports.syncQueue = exports.crawlQueue = void 0;
+exports.getPageContent = exports.queueMap = void 0;
 exports.initQueue = initQueue;
 exports.createCrawlQueue = createCrawlQueue;
 exports.getDataDir = getDataDir;
@@ -26,49 +26,57 @@ const crypto_1 = require("crypto");
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const path_1 = __importDefault(require("path"));
 const config_1 = require("./config");
+const metrics_1 = require("./metrics");
 const puppeteer_1 = require("./puppeteer");
 const carbon_1 = require("./services/carbon");
 const snapshot_1 = require("./services/snapshot");
 const store_1 = require("./store");
 const utils_1 = require("./utils");
 const { BaseState } = require('@abtnode/models');
-let crawlQueue;
-let syncQueue;
-let codeQueue;
-let cronQueue;
+exports.queueMap = {
+    urlCrawler: null,
+    syncCrawler: null,
+    codeCrawler: null,
+    cronJobs: null,
+};
 function initQueue() {
-    exports.crawlQueue = crawlQueue = createCrawlQueue('urlCrawler');
-    exports.syncQueue = syncQueue = createCrawlQueue('syncCrawler');
-    exports.codeQueue = codeQueue = createCrawlQueue('codeCrawler', {
+    exports.queueMap.urlCrawler = createCrawlQueue('urlCrawler');
+    exports.queueMap.syncCrawler = createCrawlQueue('syncCrawler');
+    exports.queueMap.codeCrawler = createCrawlQueue('codeCrawler', {
         handleScreenshot: carbon_1.createCarbonImage,
     });
-    exports.cronQueue = cronQueue = createCrawlQueue('cronJobs');
+    exports.queueMap.cronJobs = createCrawlQueue('cronJobs');
 }
 function createCrawlQueue(queue, handler) {
     const db = new BaseState(store_1.Job);
     return (0, queue_1.default)({
         store: new sequelize_1.default(db, queue),
-        concurrency: config_1.config.concurrency,
+        options: {
+            concurrency: config_1.config.concurrency,
+            enableScheduledJob: true,
+        },
         onJob: (job) => __awaiter(this, void 0, void 0, function* () {
-            config_1.logger.info('Starting to execute crawl job', job);
-            // check robots.txt
-            if (!job.ignoreRobots) {
-                const canCrawl = yield (0, utils_1.isAcceptCrawler)(job.url);
-                if (!canCrawl) {
-                    config_1.logger.error(`failed to crawl ${job.url}, denied by robots.txt`, job);
-                    const snapshot = (0, snapshot_1.convertJobToSnapshot)({
-                        job,
-                        snapshot: {
-                            status: 'failed',
-                            error: 'Denied by robots.txt',
-                        },
-                    });
-                    yield store_1.Snapshot.upsert(snapshot);
-                    return snapshot;
-                }
-            }
+            const startTime = Date.now();
+            let status = 'failed';
             const formattedJob = Object.assign(Object.assign({}, job), { cookies: (config_1.config.cookies || []).concat(job.cookies || []), localStorage: (config_1.config.localStorage || []).concat(job.localStorage || []), url: (0, utils_1.formatUrl)(job.url) });
             try {
+                config_1.logger.info(`Starting to execute ${queue} job`, Object.assign(Object.assign({}, job), { queueSize: yield store_1.Job.count() }));
+                // check robots.txt
+                if (!job.ignoreRobots) {
+                    const canCrawl = yield (0, utils_1.isAcceptCrawler)(job.url);
+                    if (!canCrawl) {
+                        config_1.logger.error(`failed to crawl ${job.url}, denied by robots.txt`, job);
+                        const snapshot = (0, snapshot_1.convertJobToSnapshot)({
+                            job,
+                            snapshot: {
+                                status: 'failed',
+                                error: 'Denied by robots.txt',
+                            },
+                        });
+                        yield store_1.Snapshot.upsert(snapshot);
+                        return snapshot;
+                    }
+                }
                 // get page content later
                 const result = yield (0, exports.getPageContent)(formattedJob, handler);
                 if (!result || (!result.html && !result.screenshot)) {
@@ -86,18 +94,13 @@ function createCrawlQueue(queue, handler) {
                 const snapshot = yield store_1.sequelize.transaction((txn) => __awaiter(this, void 0, void 0, function* () {
                     // delete old snapshot
                     if (formattedJob.replace) {
-                        try {
-                            const deletedJobIds = yield (0, snapshot_1.deleteSnapshots)({
-                                url: formattedJob.url,
-                                replace: true,
-                            }, { txn });
-                            if (deletedJobIds) {
-                                config_1.logger.info('Deleted old snapshot', { deletedJobIds });
-                            }
-                        }
-                        catch (error) {
+                        const deletedJobIds = yield (0, snapshot_1.deleteSnapshots)({
+                            url: formattedJob.url,
+                            replace: true,
+                        }, { txn }).catch((error) => {
                             config_1.logger.error('Failed to delete old snapshot', { error, formattedJob });
-                        }
+                        });
+                        config_1.logger.info('Deleted old snapshot', { deletedJobIds });
                     }
                     // save html and screenshot to data dir
                     const { screenshotPath, htmlPath } = yield saveSnapshotToLocal({
@@ -117,10 +120,12 @@ function createCrawlQueue(queue, handler) {
                     yield store_1.Snapshot.upsert(snapshot, { transaction: txn });
                     return snapshot;
                 }));
+                status = 'success';
                 return snapshot;
             }
             catch (error) {
                 config_1.logger.error(`Failed to crawl ${formattedJob.url}`, { error, formattedJob });
+                status = 'failed';
                 const snapshot = (0, snapshot_1.convertJobToSnapshot)({
                     job: formattedJob,
                     snapshot: {
@@ -131,6 +136,14 @@ function createCrawlQueue(queue, handler) {
                 yield store_1.Snapshot.upsert(snapshot);
                 return snapshot;
             }
+            finally {
+                const now = Date.now();
+                metrics_1.jobsTotal.inc({ queue, status });
+                metrics_1.jobDurationSeconds.observe({ queue, status }, (now - startTime) / 1000);
+                if (job.enqueuedAt) {
+                    metrics_1.jobTotalLatencySeconds.observe({ queue, status }, (now - job.enqueuedAt) / 1000);
+                }
+            }
         }),
     });
 }
@@ -166,7 +179,7 @@ function saveSnapshotToLocal(_a) {
         };
     });
 }
-const getPageContent = (_a, handler_1) => __awaiter(void 0, [_a, handler_1], void 0, function* ({ url, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, format = 'webp', timeout = 90 * 1000, waitTime = 0, fullPage = false, headers, cookies, localStorage, }, handler) {
+const getPageContent = (_a, handler_1) => __awaiter(void 0, [_a, handler_1], void 0, function* ({ url, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, format = 'webp', timeout = 60 * 1000, waitTime = 0, fullPage = false, headers, cookies, localStorage, }, handler) {
     const page = yield (0, puppeteer_1.initPage)();
     if (width && height) {
         yield page.setViewport({ width, height, deviceScaleFactor: 2 });
@@ -299,31 +312,59 @@ exports.getPageContent = getPageContent;
  * @param callback callback when job finished
  */
 // eslint-disable-next-line require-await
-function enqueue(queue, params, callback) {
+function enqueue(queueName, params, callback) {
     return __awaiter(this, void 0, void 0, function* () {
+        const queue = exports.queueMap[queueName];
+        if (!queue) {
+            throw new Error(`Queue ${queueName} not found`);
+        }
         // skip duplicate job
         const existsJob = yield store_1.Job.isExists(params);
         if (existsJob && !params.sync) {
             config_1.logger.info(`Crawl job already exists for ${params.url}, skip`);
             return existsJob.id;
         }
-        config_1.logger.info('enqueue crawl job', params);
         const jobId = (0, crypto_1.randomUUID)();
-        const job = queue.push(Object.assign(Object.assign({}, params), { id: jobId }));
+        const enqueuedAt = Date.now();
+        const job = queue.push({ job: Object.assign(Object.assign({}, params), { id: jobId, enqueuedAt }), jobId });
+        metrics_1.jobsEnqueuedTotal.inc({ queue: queueName });
+        // Get current queue size for logging
+        const queueSize = yield store_1.Job.count();
+        config_1.logger.info('enqueue crawl job', Object.assign(Object.assign({}, params), { jobId, queueSize }));
         job.on('finished', (_a) => __awaiter(this, [_a], void 0, function* ({ result }) {
-            config_1.logger.info(`Crawl completed ${params.url}, status: ${result ? 'success' : 'failed'}`, { job: params, result });
-            callback === null || callback === void 0 ? void 0 : callback(result ? yield (0, snapshot_1.formatSnapshot)(result) : null);
+            try {
+                const isSuccess = (result === null || result === void 0 ? void 0 : result.status) === 'success';
+                const queueSize = yield store_1.Job.count();
+                config_1.logger.info(`Crawl completed ${params.url}, status: ${isSuccess ? 'success' : 'failed'}`, {
+                    job: params,
+                    result,
+                    queueSize,
+                });
+                callback === null || callback === void 0 ? void 0 : callback(result ? yield (0, snapshot_1.formatSnapshot)(result) : null);
+            }
+            catch (error) {
+                config_1.logger.error(`Error in finished event handler for ${params.url}`, { error });
+                callback === null || callback === void 0 ? void 0 : callback(null);
+            }
+        }));
+        job.on('failed', (_a) => __awaiter(this, [_a], void 0, function* ({ error }) {
+            try {
+                const queueSize = yield store_1.Job.count();
+                config_1.logger.error(`Failed to execute job for ${params.url}`, { error, job: params, queueSize });
+            }
+            catch (err) {
+                config_1.logger.error(`Error in failed event handler for ${params.url}`, { error: err });
+            }
+            finally {
+                callback === null || callback === void 0 ? void 0 : callback(null);
+            }
         }));
-        job.on('failed', ({ error }) => {
-            config_1.logger.error(`Failed to execute job for ${params.url}`, { error, job: params });
-            callback === null || callback === void 0 ? void 0 : callback(null);
-        });
         return jobId;
     });
 }
 function crawlUrl(params, callback) {
-    return enqueue(params.sync ? syncQueue : crawlQueue, params, callback);
+    return enqueue(params.sync ? 'syncCrawler' : 'urlCrawler', params, callback);
 }
 function crawlCode(params, callback) {
-    return enqueue(codeQueue, Object.assign({ ignoreRobots: true, includeHtml: false, includeScreenshot: true }, params), callback);
+    return enqueue('codeCrawler', Object.assign({ ignoreRobots: true, includeHtml: false, includeScreenshot: true }, params), callback);
 }

package/lib/cjs/index.d.ts

@@ -1,5 +1,7 @@
 import { Config } from './config';
 export * from './crawler';
 export * from './services/snapshot';
+export * from './store/job';
 export * as utils from './utils';
+export * from './metrics';
 export declare function initCrawler(params: Pick<Config, 'puppeteerPath' | 'siteCron' | 'cookies' | 'localStorage' | 'concurrency'>): Promise<void>;

package/lib/cjs/index.js

@@ -59,7 +59,9 @@ const puppeteer_1 = require("./puppeteer");
 const migrate_1 = require("./store/migrate");
 __exportStar(require("./crawler"), exports);
 __exportStar(require("./services/snapshot"), exports);
+__exportStar(require("./store/job"), exports);
 exports.utils = __importStar(require("./utils"));
+__exportStar(require("./metrics"), exports);
 function initCrawler(params) {
     return __awaiter(this, void 0, void 0, function* () {
         var _a;

package/lib/cjs/metrics.d.ts

@@ -0,0 +1,18 @@
+import { Counter, Gauge, Histogram } from 'prom-client';
+export declare const jobsTotal: Counter<"queue" | "status">;
+export declare const jobsEnqueuedTotal: Counter<"queue">;
+export declare const jobDurationSeconds: Histogram<"queue" | "status">;
+export declare const jobTotalLatencySeconds: Histogram<"queue" | "status">;
+export declare const queueSize: Gauge<"queue">;
+/**
+ * Collect all metrics by querying database
+ */
+export declare function collectMetrics(): Promise<void>;
+/**
+ * Get metrics in Prometheus format
+ */
+export declare function getMetrics(): Promise<string>;
+/**
+ * Get content type for metrics endpoint
+ */
+export declare function getContentType(): "text/plain; version=0.0.4; charset=utf-8";

package/lib/cjs/metrics.js

@@ -0,0 +1,88 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.queueSize = exports.jobTotalLatencySeconds = exports.jobDurationSeconds = exports.jobsEnqueuedTotal = exports.jobsTotal = void 0;
+exports.collectMetrics = collectMetrics;
+exports.getMetrics = getMetrics;
+exports.getContentType = getContentType;
+const prom_client_1 = require("prom-client");
+const store_1 = require("./store");
+// Create a new registry
+const register = new prom_client_1.Registry();
+// ========== Counter - 爬取任务计数 ==========
+exports.jobsTotal = new prom_client_1.Counter({
+    name: 'crawler_jobs_total',
+    help: 'Total number of crawl jobs processed',
+    labelNames: ['queue', 'status'],
+    registers: [register],
+});
+// ========== Counter - 入队任务数 ==========
+exports.jobsEnqueuedTotal = new prom_client_1.Counter({
+    name: 'crawler_jobs_enqueued_total',
+    help: 'Total number of crawl jobs enqueued',
+    labelNames: ['queue'],
+    registers: [register],
+});
+// ========== Histogram - 任务执行耗时 ==========
+exports.jobDurationSeconds = new prom_client_1.Histogram({
+    name: 'crawler_job_duration_seconds',
+    help: 'Duration of crawl job execution in seconds',
+    labelNames: ['queue', 'status'],
+    buckets: [10, 30, 60, 120, 300, 600, 900, 1800, 3600],
+    registers: [register],
+});
+// ========== Histogram - 入队到完成总耗时 ==========
+exports.jobTotalLatencySeconds = new prom_client_1.Histogram({
+    name: 'crawler_job_total_latency_seconds',
+    help: 'Total latency from enqueue to completion in seconds',
+    labelNames: ['queue', 'status'],
+    buckets: [10, 30, 60, 120, 300, 600, 900, 1800, 3600],
+    registers: [register],
+});
+// ========== Gauge - 队列大小 ==========
+exports.queueSize = new prom_client_1.Gauge({
+    name: 'crawler_queue_size',
+    help: 'Current number of jobs in queue',
+    labelNames: ['queue'],
+    registers: [register],
+});
+/**
+ * Collect all metrics by querying database
+ */
+function collectMetrics() {
+    return __awaiter(this, void 0, void 0, function* () {
+        try {
+            // 收集队列大小
+            const jobStats = yield store_1.Job.stats();
+            jobStats.queues.forEach((q) => {
+                exports.queueSize.set({ queue: q.queue }, q.count);
+            });
+        }
+        catch (error) {
+            console.error('Error collecting metrics:', error);
+        }
+    });
+}
+/**
+ * Get metrics in Prometheus format
+ */
+function getMetrics() {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield collectMetrics();
+        return register.metrics();
+    });
+}
+/**
+ * Get content type for metrics endpoint
+ */
+function getContentType() {
+    return register.contentType;
+}

package/lib/cjs/puppeteer.js

@@ -24,6 +24,7 @@ const fs_extra_1 = __importDefault(require("fs-extra"));
 const path_1 = __importDefault(require("path"));
 const timers_1 = require("timers");
 const config_1 = require("./config");
+const store_1 = require("./store");
 const utils_1 = require("./utils");
 const BrowserStatus = {
     None: 'None',
@@ -124,21 +125,25 @@ function launchBrowser() {
                     '--no-sandbox',
                     '--no-zygote',
                     '--disable-setuid-sandbox',
-                    '--disable-gpu',
                     '--disable-dev-shm-usage',
                     '--disable-site-isolation-trials',
-                    '--disable-accelerated-2d-canvas',
                     '--disable-extensions',
-                    '--js-flags=--max_old_space_size=512', // 限制V8内存
+                    '--js-flags=--max_old_space_size=768', // 限制V8内存
                     '--disable-background-networking',
                     '--disable-default-apps',
                     // '--disable-web-security', // 允许跨域请求
-                    '--disable-software-rasterizer',
                     '--disable-crash-reporter',
                     '--disable-service-workers',
                     '--disable-notifications',
                     '--disable-infobars',
                     '--font-render-hinting=none',
+                    // WebGL: use software GL fallback for servers without GPU
+                    '--enable-webgl',
+                    '--ignore-gpu-blocklist',
+                    '--use-gl=swiftshader',
+                    '--use-angle=swiftshader',
+                    '--enable-unsafe-swiftshader',
+                    '--disable-gpu-sandbox',
                 ],
             });
             config_1.logger.info('Launch browser');
@@ -162,12 +167,18 @@ function checkBrowserActivated() {
         var _a;
         if (browser) {
             const pages = yield browser.pages().catch(() => []);
-            if (pages.length === 1 && ((_a = pages[0]) === null || _a === void 0 ? void 0 : _a.url()) === 'about:blank') {
+            const jobCount = yield store_1.Job.count().catch(() => 0);
+            // Check if browser is inactive: only blank page AND no pending jobs
+            const isInactive = pages.length === 1 && ((_a = pages[0]) === null || _a === void 0 ? void 0 : _a.url()) === 'about:blank' && jobCount === 0;
+            if (isInactive) {
                 count++;
                 config_1.logger.debug(`Browser inactive count: ${count}/3`);
             }
             else {
-                count = 0; // 重置计数器！
+                count = 0;
+                if (jobCount > 0) {
+                    config_1.logger.debug(`Browser has ${jobCount} pending jobs, keeping active`);
+                }
             }
             if (count >= 3) {
                 config_1.logger.info('Browser inactive for 3 minutes, closing...');

package/lib/cjs/site.js

@@ -17,6 +17,7 @@ const uniq_1 = __importDefault(require("lodash/uniq"));
 const node_crypto_1 = require("node:crypto");
 const config_1 = require("./config");
 const crawler_1 = require("./crawler");
+const metrics_1 = require("./metrics");
 const store_1 = require("./store");
 const utils_1 = require("./utils");
 const crawlBlockletRunningMap = new Map();
@@ -69,21 +70,30 @@ const crawlSite = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, path
             });
             crawlCount++;
             const jobId = (0, node_crypto_1.randomUUID)();
-            crawler_1.cronQueue.push({
-                id: jobId,
-                url,
-                lastModified: sitemapItem.lastmod,
-                includeScreenshot: false,
-                includeHtml: true,
-                replace: true,
+            crawler_1.queueMap.cronJobs.push({
+                job: {
+                    id: jobId,
+                    url,
+                    lastModified: sitemapItem.lastmod,
+                    includeScreenshot: false,
+                    includeHtml: true,
+                    replace: true,
+                    enqueuedAt: Date.now(),
+                },
+                jobId,
+                delay: 5,
             });
+            metrics_1.jobsEnqueuedTotal.inc({ queue: 'cronJobs' });
             return jobId;
         }), { concurrency: ((_b = config_1.config.siteCron) === null || _b === void 0 ? void 0 : _b.concurrency) || 30 });
+        // Get current queue size for logging
+        const queueSize = yield store_1.Job.count();
         config_1.logger.info('Enqueued jobs from sitemap finished', {
             url,
             pathname,
             processCount,
             crawlCount,
+            queueSize,
         });
         return jobIds;
     }

package/lib/cjs/store/job.d.ts

@@ -17,6 +17,7 @@ export interface JobState {
     replace?: boolean;
     sync?: boolean;
     ignoreRobots?: boolean;
+    enqueuedAt?: number;
     headers?: Record<string, string>;
     cookies?: CookieParam[];
     localStorage?: {
@@ -32,18 +33,46 @@ export interface JobModel {
     willRunAt: number;
     delay: number;
     cancelled: boolean;
+    processingBy: string | null;
+    processingAt: number | null;
 }
 export declare class Job extends Model<JobModel> implements JobModel {
     id: JobModel['id'];
     queue: JobModel['queue'];
     job: JobModel['job'];
     retryCount: JobModel['retryCount'];
-    willRunAt: JobModel['willRunAt'];
     delay: JobModel['delay'];
+    willRunAt: JobModel['willRunAt'];
     cancelled: JobModel['cancelled'];
+    processingBy: JobModel['processingBy'];
+    processingAt: JobModel['processingAt'];
     static initModel(sequelize: Sequelize): typeof Job;
     static findJob(condition: Partial<JobState>): Promise<JobModel | null>;
     static isExists(condition: Partial<JobState> & {
         url: string;
     }): Promise<JobModel | null | undefined>;
+    static paginate({ page, pageSize, queue, }?: {
+        page?: number;
+        pageSize?: number;
+        queue?: string;
+    }): Promise<{
+        total: number;
+        page: number;
+        pageSize: number;
+        totalPages: number;
+        data: JobModel[];
+    }>;
+    static stats(): Promise<{
+        total: number;
+        queues: {
+            queue: string;
+            count: number;
+        }[];
+    }>;
+    static deleteByQueue(queue: string): Promise<{
+        deleted: number;
+    }>;
+    static deleteByIds(ids: string[]): Promise<{
+        deleted: number;
+    }>;
 }

package/lib/cjs/store/job.js

@@ -76,6 +76,14 @@ class Job extends core_1.Model {
                 type: core_1.DataTypes.BOOLEAN,
                 defaultValue: false,
             },
+            processingBy: {
+                type: core_1.DataTypes.STRING(32),
+                allowNull: true,
+            },
+            processingAt: {
+                type: core_1.DataTypes.INTEGER,
+                allowNull: true,
+            },
             createdAt: {
                 type: core_1.DataTypes.DATE,
                 defaultValue: core_1.DataTypes.NOW,
@@ -127,5 +135,53 @@ class Job extends core_1.Model {
             return existsJob === null || existsJob === void 0 ? void 0 : existsJob.get();
         });
     }
+    static paginate() {
+        return __awaiter(this, arguments, void 0, function* ({ page = 1, pageSize = 20, queue, } = {}) {
+            const where = queue ? { queue } : {};
+            const offset = (page - 1) * pageSize;
+            const { count, rows } = yield Job.findAndCountAll({
+                where,
+                order: [['createdAt', 'DESC']],
+                limit: pageSize,
+                offset,
+            });
+            return {
+                total: count,
+                page,
+                pageSize,
+                totalPages: Math.ceil(count / pageSize),
+                data: rows.map((row) => row.toJSON()),
+            };
+        });
+    }
+    static stats() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const results = (yield Job.findAll({
+                attributes: ['queue', [core_1.default.fn('COUNT', core_1.default.col('id')), 'count']],
+                group: ['queue'],
+                raw: true,
+            }));
+            const total = results.reduce((sum, item) => sum + Number(item.count), 0);
+            return {
+                total,
+                queues: results.map((item) => ({
+                    queue: item.queue,
+                    count: Number(item.count),
+                })),
+            };
+        });
+    }
+    static deleteByQueue(queue) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const count = yield Job.destroy({ where: { queue } });
+            return { deleted: count };
+        });
+    }
+    static deleteByIds(ids) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const count = yield Job.destroy({ where: { id: ids } });
+            return { deleted: count };
+        });
+    }
 }
 exports.Job = Job;

package/lib/cjs/store/migrate.js

@@ -40,6 +40,7 @@ const umzug_1 = require("umzug");
 const index_1 = require("./index");
 const migration20250615 = __importStar(require("./migrations/20250615-genesis"));
 const migration20250616Replace = __importStar(require("./migrations/20250616-replace"));
+const migration20251226JobProcessing = __importStar(require("./migrations/20251226-job-processing"));
 const umzug = new umzug_1.Umzug({
     migrations: [
         {
@@ -52,6 +53,11 @@ const umzug = new umzug_1.Umzug({
             up: ({ context }) => migration20250616Replace.up({ context }),
             down: ({ context }) => migration20250616Replace.down({ context }),
         },
+        {
+            name: '20251226-job-processing',
+            up: ({ context }) => migration20251226JobProcessing.up({ context }),
+            down: ({ context }) => migration20251226JobProcessing.down({ context }),
+        },
     ],
     context: index_1.sequelize.getQueryInterface(),
     storage: new umzug_1.SequelizeStorage({ sequelize: index_1.sequelize }),

package/lib/cjs/store/migrations/20251226-job-processing.d.ts

@@ -0,0 +1,6 @@
+export declare function up({ context }: {
+    context: any;
+}): Promise<void>;
+export declare function down({ context }: {
+    context: any;
+}): Promise<void>;