@arcblock/crawler 1.4.6 → 1.5.0

package/lib/cjs/store/migrations/20251226-job-processing.js

@@ -0,0 +1,37 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.up = up;
+exports.down = down;
+/* eslint-disable no-console */
+const core_1 = require("@sequelize/core");
+function up(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ context }) {
+        console.log('[20251226-job-processing:up] Migrating...');
+        yield context.addColumn('jobs', 'processingBy', {
+            type: core_1.DataTypes.STRING(32),
+            allowNull: true,
+        });
+        yield context.addColumn('jobs', 'processingAt', {
+            type: core_1.DataTypes.INTEGER,
+            allowNull: true,
+        });
+        console.log('[20251226-job-processing:up] Migrated successfully!');
+    });
+}
+function down(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ context }) {
+        console.log('[20251226-job-processing:down] Migrating...');
+        yield context.removeColumn('jobs', 'processingBy');
+        yield context.removeColumn('jobs', 'processingAt');
+        console.log('[20251226-job-processing:down] Migrated successfully!');
+    });
+}

package/lib/esm/crawler.d.ts

@@ -1,10 +1,11 @@
 import { Page } from '@blocklet/puppeteer';
 import { JobState, SnapshotModel } from './store';
-declare let crawlQueue: any;
-declare let syncQueue: any;
-declare let codeQueue: any;
-declare let cronQueue: any;
-export { crawlQueue, syncQueue, codeQueue, cronQueue };
+export declare const queueMap: {
+    urlCrawler: any;
+    syncCrawler: any;
+    codeCrawler: any;
+    cronJobs: any;
+};
 export declare function initQueue(): void;
 type PageHandler = {
     handleScreenshot?: (page: Page, params?: JobState) => Promise<Buffer | null>;
@@ -28,6 +29,7 @@ export declare const getPageContent: ({ url, includeScreenshot, includeHtml, wid
  * @param params
  * @param callback callback when job finished
  */
-export declare function enqueue(queue: any, params: Omit<JobState, 'jobId'>, callback?: (snapshot: SnapshotModel | null) => void): Promise<string>;
+export declare function enqueue(queueName: keyof typeof queueMap, params: Omit<JobState, 'jobId'>, callback?: (snapshot: SnapshotModel | null) => void): Promise<string>;
 export declare function crawlUrl(params: Omit<JobState, 'jobId'>, callback?: (snapshot: SnapshotModel | null) => void): Promise<string>;
 export declare function crawlCode(params: Omit<JobState, 'jobId'>, callback?: (snapshot: SnapshotModel | null) => void): Promise<string>;
+export {};

package/lib/esm/crawler.js

@@ -14,50 +14,57 @@ import { randomUUID } from 'crypto';
 import fs from 'fs-extra';
 import path from 'path';
 import { config, logger } from './config';
+import { jobDurationSeconds, jobTotalLatencySeconds, jobsEnqueuedTotal, jobsTotal } from './metrics';
 import { initPage } from './puppeteer';
 import { createCarbonImage } from './services/carbon';
 import { convertJobToSnapshot, deleteSnapshots, formatSnapshot } from './services/snapshot';
 import { Job, Snapshot, sequelize } from './store';
 import { findMaxScrollHeight, formatUrl, isAcceptCrawler, md5, sleep } from './utils';
 const { BaseState } = require('@abtnode/models');
-let crawlQueue;
-let syncQueue;
-let codeQueue;
-let cronQueue;
-export { crawlQueue, syncQueue, codeQueue, cronQueue };
+export const queueMap = {
+    urlCrawler: null,
+    syncCrawler: null,
+    codeCrawler: null,
+    cronJobs: null,
+};
 export function initQueue() {
-    crawlQueue = createCrawlQueue('urlCrawler');
-    syncQueue = createCrawlQueue('syncCrawler');
-    codeQueue = createCrawlQueue('codeCrawler', {
+    queueMap.urlCrawler = createCrawlQueue('urlCrawler');
+    queueMap.syncCrawler = createCrawlQueue('syncCrawler');
+    queueMap.codeCrawler = createCrawlQueue('codeCrawler', {
         handleScreenshot: createCarbonImage,
     });
-    cronQueue = createCrawlQueue('cronJobs');
+    queueMap.cronJobs = createCrawlQueue('cronJobs');
 }
 export function createCrawlQueue(queue, handler) {
     const db = new BaseState(Job);
     return createQueue({
         store: new SequelizeStore(db, queue),
-        concurrency: config.concurrency,
+        options: {
+            concurrency: config.concurrency,
+            enableScheduledJob: true,
+        },
         onJob: (job) => __awaiter(this, void 0, void 0, function* () {
-            logger.info('Starting to execute crawl job', job);
-            // check robots.txt
-            if (!job.ignoreRobots) {
-                const canCrawl = yield isAcceptCrawler(job.url);
-                if (!canCrawl) {
-                    logger.error(`failed to crawl ${job.url}, denied by robots.txt`, job);
-                    const snapshot = convertJobToSnapshot({
-                        job,
-                        snapshot: {
-                            status: 'failed',
-                            error: 'Denied by robots.txt',
-                        },
-                    });
-                    yield Snapshot.upsert(snapshot);
-                    return snapshot;
-                }
-            }
+            const startTime = Date.now();
+            let status = 'failed';
             const formattedJob = Object.assign(Object.assign({}, job), { cookies: (config.cookies || []).concat(job.cookies || []), localStorage: (config.localStorage || []).concat(job.localStorage || []), url: formatUrl(job.url) });
             try {
+                logger.info(`Starting to execute ${queue} job`, Object.assign(Object.assign({}, job), { queueSize: yield Job.count() }));
+                // check robots.txt
+                if (!job.ignoreRobots) {
+                    const canCrawl = yield isAcceptCrawler(job.url);
+                    if (!canCrawl) {
+                        logger.error(`failed to crawl ${job.url}, denied by robots.txt`, job);
+                        const snapshot = convertJobToSnapshot({
+                            job,
+                            snapshot: {
+                                status: 'failed',
+                                error: 'Denied by robots.txt',
+                            },
+                        });
+                        yield Snapshot.upsert(snapshot);
+                        return snapshot;
+                    }
+                }
                 // get page content later
                 const result = yield getPageContent(formattedJob, handler);
                 if (!result || (!result.html && !result.screenshot)) {
@@ -75,18 +82,13 @@ export function createCrawlQueue(queue, handler) {
                 const snapshot = yield sequelize.transaction((txn) => __awaiter(this, void 0, void 0, function* () {
                     // delete old snapshot
                     if (formattedJob.replace) {
-                        try {
-                            const deletedJobIds = yield deleteSnapshots({
-                                url: formattedJob.url,
-                                replace: true,
-                            }, { txn });
-                            if (deletedJobIds) {
-                                logger.info('Deleted old snapshot', { deletedJobIds });
-                            }
-                        }
-                        catch (error) {
+                        const deletedJobIds = yield deleteSnapshots({
+                            url: formattedJob.url,
+                            replace: true,
+                        }, { txn }).catch((error) => {
                             logger.error('Failed to delete old snapshot', { error, formattedJob });
-                        }
+                        });
+                        logger.info('Deleted old snapshot', { deletedJobIds });
                     }
                     // save html and screenshot to data dir
                     const { screenshotPath, htmlPath } = yield saveSnapshotToLocal({
@@ -106,10 +108,12 @@ export function createCrawlQueue(queue, handler) {
                     yield Snapshot.upsert(snapshot, { transaction: txn });
                     return snapshot;
                 }));
+                status = 'success';
                 return snapshot;
             }
             catch (error) {
                 logger.error(`Failed to crawl ${formattedJob.url}`, { error, formattedJob });
+                status = 'failed';
                 const snapshot = convertJobToSnapshot({
                     job: formattedJob,
                     snapshot: {
@@ -120,6 +124,14 @@ export function createCrawlQueue(queue, handler) {
                 yield Snapshot.upsert(snapshot);
                 return snapshot;
             }
+            finally {
+                const now = Date.now();
+                jobsTotal.inc({ queue, status });
+                jobDurationSeconds.observe({ queue, status }, (now - startTime) / 1000);
+                if (job.enqueuedAt) {
+                    jobTotalLatencySeconds.observe({ queue, status }, (now - job.enqueuedAt) / 1000);
+                }
+            }
         }),
     });
 }
@@ -155,7 +167,7 @@ function saveSnapshotToLocal(_a) {
         };
     });
 }
-export const getPageContent = (_a, handler_1) => __awaiter(void 0, [_a, handler_1], void 0, function* ({ url, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, format = 'webp', timeout = 90 * 1000, waitTime = 0, fullPage = false, headers, cookies, localStorage, }, handler) {
+export const getPageContent = (_a, handler_1) => __awaiter(void 0, [_a, handler_1], void 0, function* ({ url, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, format = 'webp', timeout = 60 * 1000, waitTime = 0, fullPage = false, headers, cookies, localStorage, }, handler) {
     const page = yield initPage();
     if (width && height) {
         yield page.setViewport({ width, height, deviceScaleFactor: 2 });
@@ -287,31 +299,59 @@ export const getPageContent = (_a, handler_1) => __awaiter(void 0, [_a, handler_
  * @param callback callback when job finished
  */
 // eslint-disable-next-line require-await
-export function enqueue(queue, params, callback) {
+export function enqueue(queueName, params, callback) {
     return __awaiter(this, void 0, void 0, function* () {
+        const queue = queueMap[queueName];
+        if (!queue) {
+            throw new Error(`Queue ${queueName} not found`);
+        }
         // skip duplicate job
         const existsJob = yield Job.isExists(params);
         if (existsJob && !params.sync) {
             logger.info(`Crawl job already exists for ${params.url}, skip`);
             return existsJob.id;
         }
-        logger.info('enqueue crawl job', params);
         const jobId = randomUUID();
-        const job = queue.push(Object.assign(Object.assign({}, params), { id: jobId }));
+        const enqueuedAt = Date.now();
+        const job = queue.push({ job: Object.assign(Object.assign({}, params), { id: jobId, enqueuedAt }), jobId });
+        jobsEnqueuedTotal.inc({ queue: queueName });
+        // Get current queue size for logging
+        const queueSize = yield Job.count();
+        logger.info('enqueue crawl job', Object.assign(Object.assign({}, params), { jobId, queueSize }));
         job.on('finished', (_a) => __awaiter(this, [_a], void 0, function* ({ result }) {
-            logger.info(`Crawl completed ${params.url}, status: ${result ? 'success' : 'failed'}`, { job: params, result });
-            callback === null || callback === void 0 ? void 0 : callback(result ? yield formatSnapshot(result) : null);
+            try {
+                const isSuccess = (result === null || result === void 0 ? void 0 : result.status) === 'success';
+                const queueSize = yield Job.count();
+                logger.info(`Crawl completed ${params.url}, status: ${isSuccess ? 'success' : 'failed'}`, {
+                    job: params,
+                    result,
+                    queueSize,
+                });
+                callback === null || callback === void 0 ? void 0 : callback(result ? yield formatSnapshot(result) : null);
+            }
+            catch (error) {
+                logger.error(`Error in finished event handler for ${params.url}`, { error });
+                callback === null || callback === void 0 ? void 0 : callback(null);
+            }
+        }));
+        job.on('failed', (_a) => __awaiter(this, [_a], void 0, function* ({ error }) {
+            try {
+                const queueSize = yield Job.count();
+                logger.error(`Failed to execute job for ${params.url}`, { error, job: params, queueSize });
+            }
+            catch (err) {
+                logger.error(`Error in failed event handler for ${params.url}`, { error: err });
+            }
+            finally {
+                callback === null || callback === void 0 ? void 0 : callback(null);
+            }
         }));
-        job.on('failed', ({ error }) => {
-            logger.error(`Failed to execute job for ${params.url}`, { error, job: params });
-            callback === null || callback === void 0 ? void 0 : callback(null);
-        });
         return jobId;
     });
 }
 export function crawlUrl(params, callback) {
-    return enqueue(params.sync ? syncQueue : crawlQueue, params, callback);
+    return enqueue(params.sync ? 'syncCrawler' : 'urlCrawler', params, callback);
 }
 export function crawlCode(params, callback) {
-    return enqueue(codeQueue, Object.assign({ ignoreRobots: true, includeHtml: false, includeScreenshot: true }, params), callback);
+    return enqueue('codeCrawler', Object.assign({ ignoreRobots: true, includeHtml: false, includeScreenshot: true }, params), callback);
 }

package/lib/esm/index.d.ts

@@ -1,5 +1,7 @@
 import { Config } from './config';
 export * from './crawler';
 export * from './services/snapshot';
+export * from './store/job';
 export * as utils from './utils';
+export * from './metrics';
 export declare function initCrawler(params: Pick<Config, 'puppeteerPath' | 'siteCron' | 'cookies' | 'localStorage' | 'concurrency'>): Promise<void>;

package/lib/esm/index.js

@@ -16,7 +16,9 @@ import { ensureBrowser } from './puppeteer';
 import { migrate } from './store/migrate';
 export * from './crawler';
 export * from './services/snapshot';
+export * from './store/job';
 export * as utils from './utils';
+export * from './metrics';
 export function initCrawler(params) {
     return __awaiter(this, void 0, void 0, function* () {
         var _a;

package/lib/esm/metrics.d.ts

@@ -0,0 +1,18 @@
+import { Counter, Gauge, Histogram } from 'prom-client';
+export declare const jobsTotal: Counter<"queue" | "status">;
+export declare const jobsEnqueuedTotal: Counter<"queue">;
+export declare const jobDurationSeconds: Histogram<"queue" | "status">;
+export declare const jobTotalLatencySeconds: Histogram<"queue" | "status">;
+export declare const queueSize: Gauge<"queue">;
+/**
+ * Collect all metrics by querying database
+ */
+export declare function collectMetrics(): Promise<void>;
+/**
+ * Get metrics in Prometheus format
+ */
+export declare function getMetrics(): Promise<string>;
+/**
+ * Get content type for metrics endpoint
+ */
+export declare function getContentType(): "text/plain; version=0.0.4; charset=utf-8";

package/lib/esm/metrics.js

@@ -0,0 +1,82 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { Counter, Gauge, Histogram, Registry } from 'prom-client';
+import { Job } from './store';
+// Create a new registry
+const register = new Registry();
+// ========== Counter - 爬取任务计数 ==========
+export const jobsTotal = new Counter({
+    name: 'crawler_jobs_total',
+    help: 'Total number of crawl jobs processed',
+    labelNames: ['queue', 'status'],
+    registers: [register],
+});
+// ========== Counter - 入队任务数 ==========
+export const jobsEnqueuedTotal = new Counter({
+    name: 'crawler_jobs_enqueued_total',
+    help: 'Total number of crawl jobs enqueued',
+    labelNames: ['queue'],
+    registers: [register],
+});
+// ========== Histogram - 任务执行耗时 ==========
+export const jobDurationSeconds = new Histogram({
+    name: 'crawler_job_duration_seconds',
+    help: 'Duration of crawl job execution in seconds',
+    labelNames: ['queue', 'status'],
+    buckets: [10, 30, 60, 120, 300, 600, 900, 1800, 3600],
+    registers: [register],
+});
+// ========== Histogram - 入队到完成总耗时 ==========
+export const jobTotalLatencySeconds = new Histogram({
+    name: 'crawler_job_total_latency_seconds',
+    help: 'Total latency from enqueue to completion in seconds',
+    labelNames: ['queue', 'status'],
+    buckets: [10, 30, 60, 120, 300, 600, 900, 1800, 3600],
+    registers: [register],
+});
+// ========== Gauge - 队列大小 ==========
+export const queueSize = new Gauge({
+    name: 'crawler_queue_size',
+    help: 'Current number of jobs in queue',
+    labelNames: ['queue'],
+    registers: [register],
+});
+/**
+ * Collect all metrics by querying database
+ */
+export function collectMetrics() {
+    return __awaiter(this, void 0, void 0, function* () {
+        try {
+            // 收集队列大小
+            const jobStats = yield Job.stats();
+            jobStats.queues.forEach((q) => {
+                queueSize.set({ queue: q.queue }, q.count);
+            });
+        }
+        catch (error) {
+            console.error('Error collecting metrics:', error);
+        }
+    });
+}
+/**
+ * Get metrics in Prometheus format
+ */
+export function getMetrics() {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield collectMetrics();
+        return register.metrics();
+    });
+}
+/**
+ * Get content type for metrics endpoint
+ */
+export function getContentType() {
+    return register.contentType;
+}

package/lib/esm/puppeteer.js

@@ -12,6 +12,7 @@ import fs from 'fs-extra';
 import path from 'path';
 import { clearInterval, setInterval } from 'timers';
 import { config, logger } from './config';
+import { Job } from './store';
 import { CRAWLER_FLAG, sleep } from './utils';
 const BrowserStatus = {
     None: 'None',
@@ -113,21 +114,25 @@ export function launchBrowser() {
                     '--no-sandbox',
                     '--no-zygote',
                     '--disable-setuid-sandbox',
-                    '--disable-gpu',
                     '--disable-dev-shm-usage',
                     '--disable-site-isolation-trials',
-                    '--disable-accelerated-2d-canvas',
                     '--disable-extensions',
-                    '--js-flags=--max_old_space_size=512', // 限制V8内存
+                    '--js-flags=--max_old_space_size=768', // 限制V8内存
                     '--disable-background-networking',
                     '--disable-default-apps',
                     // '--disable-web-security', // 允许跨域请求
-                    '--disable-software-rasterizer',
                     '--disable-crash-reporter',
                     '--disable-service-workers',
                     '--disable-notifications',
                     '--disable-infobars',
                     '--font-render-hinting=none',
+                    // WebGL: use software GL fallback for servers without GPU
+                    '--enable-webgl',
+                    '--ignore-gpu-blocklist',
+                    '--use-gl=swiftshader',
+                    '--use-angle=swiftshader',
+                    '--enable-unsafe-swiftshader',
+                    '--disable-gpu-sandbox',
                 ],
             });
             logger.info('Launch browser');
@@ -151,12 +156,18 @@ function checkBrowserActivated() {
         var _a;
         if (browser) {
             const pages = yield browser.pages().catch(() => []);
-            if (pages.length === 1 && ((_a = pages[0]) === null || _a === void 0 ? void 0 : _a.url()) === 'about:blank') {
+            const jobCount = yield Job.count().catch(() => 0);
+            // Check if browser is inactive: only blank page AND no pending jobs
+            const isInactive = pages.length === 1 && ((_a = pages[0]) === null || _a === void 0 ? void 0 : _a.url()) === 'about:blank' && jobCount === 0;
+            if (isInactive) {
                 count++;
                 logger.debug(`Browser inactive count: ${count}/3`);
             }
             else {
-                count = 0; // 重置计数器！
+                count = 0;
+                if (jobCount > 0) {
+                    logger.debug(`Browser has ${jobCount} pending jobs, keeping active`);
+                }
             }
             if (count >= 3) {
                 logger.info('Browser inactive for 3 minutes, closing...');

package/lib/esm/site.js

@@ -10,8 +10,9 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 import uniq from 'lodash/uniq';
 import { randomUUID } from 'node:crypto';
 import { config, logger } from './config';
-import { cronQueue } from './crawler';
-import { Snapshot } from './store';
+import { queueMap } from './crawler';
+import { jobsEnqueuedTotal } from './metrics';
+import { Job, Snapshot } from './store';
 import { formatUrl, getSitemapList } from './utils';
 const crawlBlockletRunningMap = new Map();
 function parseSitemapUrl(sitemapItem) {
@@ -63,21 +64,30 @@ export const crawlSite = (_a) => __awaiter(void 0, [_a], void 0, function* ({ ur
             });
             crawlCount++;
             const jobId = randomUUID();
-            cronQueue.push({
-                id: jobId,
-                url,
-                lastModified: sitemapItem.lastmod,
-                includeScreenshot: false,
-                includeHtml: true,
-                replace: true,
+            queueMap.cronJobs.push({
+                job: {
+                    id: jobId,
+                    url,
+                    lastModified: sitemapItem.lastmod,
+                    includeScreenshot: false,
+                    includeHtml: true,
+                    replace: true,
+                    enqueuedAt: Date.now(),
+                },
+                jobId,
+                delay: 5,
             });
+            jobsEnqueuedTotal.inc({ queue: 'cronJobs' });
             return jobId;
         }), { concurrency: ((_b = config.siteCron) === null || _b === void 0 ? void 0 : _b.concurrency) || 30 });
+        // Get current queue size for logging
+        const queueSize = yield Job.count();
         logger.info('Enqueued jobs from sitemap finished', {
             url,
             pathname,
             processCount,
             crawlCount,
+            queueSize,
         });
         return jobIds;
     }

package/lib/esm/store/job.d.ts

@@ -17,6 +17,7 @@ export interface JobState {
     replace?: boolean;
     sync?: boolean;
     ignoreRobots?: boolean;
+    enqueuedAt?: number;
     headers?: Record<string, string>;
     cookies?: CookieParam[];
     localStorage?: {
@@ -32,18 +33,46 @@ export interface JobModel {
     willRunAt: number;
     delay: number;
     cancelled: boolean;
+    processingBy: string | null;
+    processingAt: number | null;
 }
 export declare class Job extends Model<JobModel> implements JobModel {
     id: JobModel['id'];
     queue: JobModel['queue'];
     job: JobModel['job'];
     retryCount: JobModel['retryCount'];
-    willRunAt: JobModel['willRunAt'];
     delay: JobModel['delay'];
+    willRunAt: JobModel['willRunAt'];
     cancelled: JobModel['cancelled'];
+    processingBy: JobModel['processingBy'];
+    processingAt: JobModel['processingAt'];
     static initModel(sequelize: Sequelize): typeof Job;
     static findJob(condition: Partial<JobState>): Promise<JobModel | null>;
     static isExists(condition: Partial<JobState> & {
         url: string;
     }): Promise<JobModel | null | undefined>;
+    static paginate({ page, pageSize, queue, }?: {
+        page?: number;
+        pageSize?: number;
+        queue?: string;
+    }): Promise<{
+        total: number;
+        page: number;
+        pageSize: number;
+        totalPages: number;
+        data: JobModel[];
+    }>;
+    static stats(): Promise<{
+        total: number;
+        queues: {
+            queue: string;
+            count: number;
+        }[];
+    }>;
+    static deleteByQueue(queue: string): Promise<{
+        deleted: number;
+    }>;
+    static deleteByIds(ids: string[]): Promise<{
+        deleted: number;
+    }>;
 }

package/lib/esm/store/job.js

@@ -37,6 +37,14 @@ export class Job extends Model {
                 type: DataTypes.BOOLEAN,
                 defaultValue: false,
             },
+            processingBy: {
+                type: DataTypes.STRING(32),
+                allowNull: true,
+            },
+            processingAt: {
+                type: DataTypes.INTEGER,
+                allowNull: true,
+            },
             createdAt: {
                 type: DataTypes.DATE,
                 defaultValue: DataTypes.NOW,
@@ -88,4 +96,52 @@ export class Job extends Model {
             return existsJob === null || existsJob === void 0 ? void 0 : existsJob.get();
         });
     }
+    static paginate() {
+        return __awaiter(this, arguments, void 0, function* ({ page = 1, pageSize = 20, queue, } = {}) {
+            const where = queue ? { queue } : {};
+            const offset = (page - 1) * pageSize;
+            const { count, rows } = yield Job.findAndCountAll({
+                where,
+                order: [['createdAt', 'DESC']],
+                limit: pageSize,
+                offset,
+            });
+            return {
+                total: count,
+                page,
+                pageSize,
+                totalPages: Math.ceil(count / pageSize),
+                data: rows.map((row) => row.toJSON()),
+            };
+        });
+    }
+    static stats() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const results = (yield Job.findAll({
+                attributes: ['queue', [sequelize.fn('COUNT', sequelize.col('id')), 'count']],
+                group: ['queue'],
+                raw: true,
+            }));
+            const total = results.reduce((sum, item) => sum + Number(item.count), 0);
+            return {
+                total,
+                queues: results.map((item) => ({
+                    queue: item.queue,
+                    count: Number(item.count),
+                })),
+            };
+        });
+    }
+    static deleteByQueue(queue) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const count = yield Job.destroy({ where: { queue } });
+            return { deleted: count };
+        });
+    }
+    static deleteByIds(ids) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const count = yield Job.destroy({ where: { id: ids } });
+            return { deleted: count };
+        });
+    }
 }

package/lib/esm/store/migrate.js

@@ -3,6 +3,7 @@ import { SequelizeStorage, Umzug } from 'umzug';
 import { sequelize } from './index';
 import * as migration20250615 from './migrations/20250615-genesis';
 import * as migration20250616Replace from './migrations/20250616-replace';
+import * as migration20251226JobProcessing from './migrations/20251226-job-processing';
 const umzug = new Umzug({
     migrations: [
         {
@@ -15,6 +16,11 @@ const umzug = new Umzug({
             up: ({ context }) => migration20250616Replace.up({ context }),
             down: ({ context }) => migration20250616Replace.down({ context }),
         },
+        {
+            name: '20251226-job-processing',
+            up: ({ context }) => migration20251226JobProcessing.up({ context }),
+            down: ({ context }) => migration20251226JobProcessing.down({ context }),
+        },
     ],
     context: sequelize.getQueryInterface(),
     storage: new SequelizeStorage({ sequelize }),

package/lib/esm/store/migrations/20251226-job-processing.d.ts

@@ -0,0 +1,6 @@
+export declare function up({ context }: {
+    context: any;
+}): Promise<void>;
+export declare function down({ context }: {
+    context: any;
+}): Promise<void>;