@arcblock/crawler 1.0.6 → 1.1.1

package/dist/db/job.js

@@ -1,54 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Job = void 0;
-exports.initJobModel = initJobModel;
-const core_1 = require("@sequelize/core");
-class Job extends core_1.Model {
-}
-exports.Job = Job;
-function initJobModel(sequelize) {
-    Job.init({
-        id: {
-            type: core_1.DataTypes.STRING(40),
-            primaryKey: true,
-        },
-        queue: {
-            type: core_1.DataTypes.STRING(32),
-            allowNull: false,
-        },
-        job: {
-            type: core_1.DataTypes.JSON,
-            allowNull: false,
-        },
-        retryCount: {
-            type: core_1.DataTypes.INTEGER,
-        },
-        delay: {
-            type: core_1.DataTypes.INTEGER,
-        },
-        willRunAt: {
-            type: core_1.DataTypes.INTEGER,
-        },
-        cancelled: {
-            type: core_1.DataTypes.BOOLEAN,
-            defaultValue: false,
-        },
-        createdAt: {
-            type: core_1.DataTypes.DATE,
-            defaultValue: core_1.DataTypes.NOW,
-            index: true,
-        },
-        updatedAt: {
-            type: core_1.DataTypes.DATE,
-            defaultValue: core_1.DataTypes.NOW,
-            index: true,
-        },
-    }, {
-        sequelize,
-        indexes: [{ fields: ['queue'] }],
-        modelName: 'job',
-        tableName: 'jobs',
-        timestamps: true,
-    });
-    return Job;
-}

package/dist/db/snapshot.js

@@ -1,52 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Snapshot = void 0;
-exports.initSnapshotModel = initSnapshotModel;
-const core_1 = require("@sequelize/core");
-class Snapshot extends core_1.Model {
-}
-exports.Snapshot = Snapshot;
-function initSnapshotModel(sequelize) {
-    Snapshot.init({
-        jobId: {
-            type: core_1.DataTypes.STRING,
-            primaryKey: true,
-            allowNull: false,
-        },
-        url: {
-            type: core_1.DataTypes.STRING,
-            allowNull: false,
-            index: true,
-        },
-        status: {
-            type: core_1.DataTypes.ENUM('success', 'failed'),
-            allowNull: false,
-        },
-        html: {
-            type: core_1.DataTypes.TEXT,
-            allowNull: true,
-        },
-        screenshot: {
-            type: core_1.DataTypes.STRING,
-            allowNull: true,
-        },
-        error: {
-            type: core_1.DataTypes.STRING,
-            allowNull: true,
-        },
-        lastModified: {
-            type: core_1.DataTypes.STRING,
-            allowNull: true,
-        },
-        options: {
-            type: core_1.DataTypes.JSON,
-            allowNull: true,
-        },
-    }, {
-        sequelize,
-        modelName: 'snapshot',
-        tableName: 'snap',
-        timestamps: true,
-    });
-    return Snapshot;
-}

package/dist/index.d.ts

@@ -1,6 +0,0 @@
-import { config } from './config';
-export * from './blocklet';
-export * from './crawler';
-export * from './middleware';
-export { Snapshot } from './db/snapshot';
-export declare function initCrawler(_config: Partial<typeof config>): Promise<void>;

package/dist/index.js

@@ -1,45 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Snapshot = void 0;
-exports.initCrawler = initCrawler;
-const config_1 = require("./config");
-const crawler_1 = require("./crawler");
-const db_1 = require("./db");
-const puppeteer_1 = require("./puppeteer");
-__exportStar(require("./blocklet"), exports);
-__exportStar(require("./crawler"), exports);
-__exportStar(require("./middleware"), exports);
-var snapshot_1 = require("./db/snapshot");
-Object.defineProperty(exports, "Snapshot", { enumerable: true, get: function () { return snapshot_1.Snapshot; } });
-function initCrawler(_config) {
-    return __awaiter(this, void 0, void 0, function* () {
-        Object.assign(config_1.config, _config);
-        config_1.logger.debug('init crawler', config_1.config);
-        yield (0, db_1.ensureDatabase)();
-        yield (0, crawler_1.createCrawlQueue)();
-        yield (0, puppeteer_1.ensureBrowser)();
-    });
-}

package/dist/middleware.d.ts

@@ -1,4 +0,0 @@
-export declare function initSEOMiddleware({ autoReturnHtml, allowCrawler, }: {
-    autoReturnHtml?: Boolean;
-    allowCrawler?: Boolean | Function;
-}): (req: any, res: any, next: Function) => Promise<any>;

package/dist/middleware.js

@@ -1,44 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.initSEOMiddleware = initSEOMiddleware;
-const cache_1 = require("./cache");
-const utils_1 = require("./utils");
-function initSEOMiddleware({ autoReturnHtml = true, allowCrawler = true, }) {
-    return (req, res, next) => __awaiter(this, void 0, void 0, function* () {
-        const isBot = (0, utils_1.isBotUserAgent)(req);
-        const isSelf = (0, utils_1.isSelfCrawler)(req);
-        if (!isBot || isSelf) {
-            return next();
-        }
-        const fullUrl = (0, utils_1.getFullUrl)(req);
-        const canCrawl = yield (0, utils_1.isAcceptCrawler)(fullUrl);
-        const allowCrawlerResult = typeof allowCrawler === 'function' ? allowCrawler(req) : allowCrawler;
-        // can not crawl, skip
-        if (!canCrawl || !allowCrawlerResult) {
-            return next();
-        }
-        const cacheData = yield cache_1.useCache.get(fullUrl);
-        // add cached html to req
-        req.cachedHtml = (cacheData === null || cacheData === void 0 ? void 0 : cacheData.content) || cacheData || null;
-        // add cached lastModified to req, ISO string to GMT string
-        req.cachedLastmod = (cacheData === null || cacheData === void 0 ? void 0 : cacheData.lastModified) ? new Date(cacheData === null || cacheData === void 0 ? void 0 : cacheData.lastModified).toUTCString() : null;
-        if (req.cachedLastmod) {
-            res.setHeader('Last-Modified', req.cachedLastmod);
-        }
-        if (autoReturnHtml && req.cachedHtml) {
-            res.send(req.cachedHtml);
-            return;
-        }
-        // missing cache
-        next();
-    });
-}

package/dist/utils.d.ts

@@ -1,17 +0,0 @@
-import { Page } from '@blocklet/puppeteer';
-export declare const api: import("axios").AxiosInstance;
-export declare const sleep: (ms: number) => Promise<unknown>;
-export declare const CRAWLER_FLAG = "x-crawler";
-export declare const isSelfCrawler: (req: any) => boolean;
-export declare const getDefaultRobotsUrl: (url: string) => string;
-export declare function getRobots(url: string): Promise<import("robots-parser").Robot | null>;
-export declare const getDefaultSitemapUrl: (url: string) => string;
-export declare const isAcceptCrawler: (url: string) => Promise<boolean | undefined>;
-export declare const getSitemapList: (url: string) => Promise<import("sitemap").SitemapItem[]>;
-export declare const isBotUserAgent: (req: any) => boolean;
-export declare const getComponentInfo: () => {};
-export declare const getFullUrl: (req: any) => string;
-export declare const getRelativePath: (url: string) => string;
-export declare const formatUrl: (url: string) => string;
-export declare function md5(content: string | Uint8Array): string;
-export declare function findMaxScrollHeight(page: Page): Promise<number>;

package/esm/blocklet.d.ts

@@ -1,6 +0,0 @@
-export declare const crawlBlocklet: () => Promise<void>;
-export declare const initCronCrawlBlocklet: ({ time, options, }?: {
-    time: string;
-    options: any;
-}) => any;
-export declare const cancelCronCrawlBlocklet: () => void;

package/esm/blocklet.js

@@ -1,190 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import Cron from '@abtnode/cron';
-import { components } from '@blocklet/sdk/lib/config';
-import debounce from 'lodash/debounce';
-import { joinURL } from 'ufo';
-import { useCache } from './cache';
-import { config, logger } from './config';
-import { createCrawlJob } from './crawler';
-import { closeBrowser, getBrowser } from './puppeteer';
-import { getComponentInfo, getRelativePath, getSitemapList } from './utils';
-// record crawl blocklet running
-const crawlBlockletRunningMap = new Map();
-// crawl blocklet sitemap urls
-export const crawlBlocklet = () => __awaiter(void 0, void 0, void 0, function* () {
-    // @ts-ignore
-    const { mountPoint, did } = getComponentInfo();
-    if (crawlBlockletRunningMap.has(did) && crawlBlockletRunningMap.get(did)) {
-        logger.info(`Crawler blocklet ${did} is running, skip it`);
-        return;
-    }
-    // check has browser can use
-    try {
-        const browser = yield getBrowser();
-        if (!browser) {
-            throw new Error('No Browser can use');
-        }
-        logger.info('Crawler blocklet existing can use browser');
-    }
-    catch (error) {
-        logger.info(`Crawler blocklet abort by error: ${(error === null || error === void 0 ? void 0 : error.message) || (error === null || error === void 0 ? void 0 : error.reason) || error}`);
-        return;
-    }
-    const { appUrl } = config;
-    if (!appUrl) {
-        throw new Error('appUrl not found');
-    }
-    const sitemapList = yield getSitemapList(appUrl);
-    const matchMountPoint = joinURL(appUrl, !mountPoint || mountPoint === '/' ? '' : mountPoint);
-    const otherMountPointList = components
-        .filter((item) => item.mountPoint && item.mountPoint !== mountPoint)
-        .map((item) => item.mountPoint);
-    // get can use loc
-    const blockletLocList = sitemapList.filter((item) => {
-        var _a;
-        if (mountPoint !== '/') {
-            return ((_a = item === null || item === void 0 ? void 0 : item.url) === null || _a === void 0 ? void 0 : _a.indexOf(matchMountPoint)) > -1;
-        }
-        // if mountPoint is /, skip other mountPoint
-        return otherMountPointList.every((mountPoint) => { var _a; return ((_a = item === null || item === void 0 ? void 0 : item.url) === null || _a === void 0 ? void 0 : _a.indexOf(mountPoint)) === -1; });
-    });
-    const canUseBlockletLocList = [];
-    const lastmodMap = new Map();
-    let skipBlockletLocTotal = 0;
-    let blockletLocTotal = 0;
-    yield Promise.all(blockletLocList.map((item) => __awaiter(void 0, void 0, void 0, function* () {
-        var _a;
-        let tempLocList = [];
-        if (item.url) {
-            tempLocList.push(item.url);
-        }
-        if (((_a = item === null || item === void 0 ? void 0 : item.links) === null || _a === void 0 ? void 0 : _a.length) > 0) {
-            tempLocList.push(...item.links.map((ytem) => ytem.url));
-        }
-        blockletLocTotal += tempLocList.length;
-        // @ts-ignore
-        tempLocList = (yield Promise.all(tempLocList.map((loc) => __awaiter(void 0, void 0, void 0, function* () {
-            try {
-                const { lastModified: cacheLastModified } = yield useCache.get(getRelativePath(loc));
-                // sitemap item lastmod is same as cache lastModified, skip it
-                if (item.lastmod &&
-                    cacheLastModified &&
-                    new Date(cacheLastModified).getTime() === new Date(item.lastmod).getTime()) {
-                    skipBlockletLocTotal++;
-                    return false;
-                }
-                return loc;
-            }
-            catch (error) {
-                // ignore error
-            }
-            // if can not get cache, return loc
-            return loc;
-        })))).filter(Boolean);
-        tempLocList.forEach((loc) => {
-            if (item.lastmod)
-                lastmodMap.set(loc, item.lastmod);
-        });
-        canUseBlockletLocList.push(...tempLocList);
-    })));
-    const crawlerLogText = (step = '') => [
-        `Crawler sitemap.xml about ${did} ${step}: `,
-        {
-            blockletLocTotal,
-            canUseBlockletLocTotal: canUseBlockletLocList.length,
-            skipBlockletLocTotal,
-            lastmodMapTotal: lastmodMap.size,
-        },
-    ];
-    logger.info(...crawlerLogText('start'));
-    try {
-        // record crawl blocklet running
-        crawlBlockletRunningMap.set(did, true);
-        yield createCrawlJob({
-            // @ts-ignore
-            urls: canUseBlockletLocList,
-            saveToRedis: true,
-            lastmodMap,
-            // formatPageContent: async ({ page }: { page: any; url: string; lastmod?: string }) => {
-            //   const pageContent = await page.evaluate(() => {
-            //     const removeElements = (tagName: string) => {
-            //       const elements = document.querySelectorAll(tagName);
-            //       for (let i = elements.length - 1; i >= 0; i--) {
-            //         try {
-            //           elements[i]?.parentNode?.removeChild(elements[i] as Node);
-            //         } catch (error) {
-            //           // do noting
-            //         }
-            //       }
-            //     };
-            //     // remove script, style, link, noscript
-            //     // removeElements('script');
-            //     // removeElements('style');
-            //     // removeElements('link');
-            //     // removeElements('noscript');
-            //     // remove uploader
-            //     removeElements('[id="uploader-container"]');
-            //     removeElements('[class^="uppy-"]');
-            //     // remove point up component
-            //     removeElements('[id="point-up-component"]');
-            //     // add meta tag to record crawler
-            //     const meta = document.createElement('meta');
-            //     meta.name = 'blocklet-crawler';
-            //     meta.content = 'true';
-            //     document.head.appendChild(meta);
-            //     return document.documentElement.outerHTML;
-            //   });
-            //   return pageContent;
-            // },
-        });
-        logger.info(...crawlerLogText('success'));
-        yield closeBrowser({
-            trimCache: true,
-        });
-    }
-    catch (error) {
-        logger.info('Crawler blocklet abort by error', error);
-    }
-    finally {
-        // delete crawl blocklet running
-        crawlBlockletRunningMap.delete(did);
-    }
-});
-const CRON_CRAWL_BLOCKLET_KEY = 'cron-crawl-blocklet';
-let cronCrawlBlockletJob = null;
-// init cron crawl blocklet
-export const initCronCrawlBlocklet = ({ time = '0 0 */12 * * *', // every 12 hours
-options, } = {}) => {
-    if (!cronCrawlBlockletJob) {
-        cronCrawlBlockletJob = Cron.init({
-            context: {},
-            jobs: [
-                {
-                    name: CRON_CRAWL_BLOCKLET_KEY,
-                    time,
-                    fn: debounce(crawlBlocklet),
-                    options: Object.assign({ runOnInit: false }, options),
-                },
-            ],
-            onError: (err) => {
-                console.error('run job failed', err);
-            },
-        });
-    }
-    return cronCrawlBlockletJob;
-};
-export const cancelCronCrawlBlocklet = () => {
-    if (cronCrawlBlockletJob) {
-        cronCrawlBlockletJob.jobs[CRON_CRAWL_BLOCKLET_KEY].stop();
-        cronCrawlBlockletJob = null;
-        logger.info('Cron crawl blocklet stop, clear crawl queue');
-    }
-};

package/esm/cache.d.ts

@@ -1,10 +0,0 @@
-export declare const cachePool: import("generic-pool").Pool<any>;
-export declare const memoryPool: import("generic-pool").Pool<Map<string, any>>;
-export declare const withCache: (cb: Function) => Promise<any>;
-export declare const formatKey: (key: string) => string;
-export declare const useCache: {
-    get: (key: string) => Promise<any>;
-    set: (key: string, value: any, options?: any) => Promise<any>;
-    remove: (key: string) => Promise<any>;
-    list: (key?: string) => Promise<any>;
-};

package/esm/cache.js

@@ -1,114 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var _a;
-import { createPool } from 'generic-pool';
-import { createClient } from 'redis';
-import { config, logger } from './config';
-const cacheKeyPrefix = ((_a = process.env) === null || _a === void 0 ? void 0 : _a.BLOCKLET_REAL_DID) ? `${process.env.BLOCKLET_REAL_DID}:` : '';
-const MAX_REDIS_RETRY = 3;
-const ttl = 1000 * 60 * 60 * 24 * 7;
-export const cachePool = createPool({
-    create: () => __awaiter(void 0, void 0, void 0, function* () {
-        try {
-            const { redisUrl } = config;
-            const redisClient = createClient({
-                url: redisUrl,
-                socket: {
-                    // @ts-ignore
-                    reconnectStrategy: (retries) => {
-                        if (retries >= MAX_REDIS_RETRY) {
-                            return new Error('Retry Time Exhausted');
-                        }
-                        return Math.min(retries * 500, 1000 * 3);
-                    },
-                },
-            });
-            redisClient.on('error', (err) => logger.warn('Redis Client Error:', err));
-            yield redisClient.connect();
-            logger.info(`Successfully connected to Redis: ${redisUrl}`);
-            return redisClient;
-        }
-        catch (error) {
-            logger.warn('Redis connection failed', error);
-            return null;
-        }
-    }),
-    destroy: (client) => __awaiter(void 0, void 0, void 0, function* () {
-        // if is redis client
-        if (client.isReady) {
-            yield client.quit();
-        }
-    }),
-}, {
-    max: 2, // 2 clients
-    min: 0,
-    // evictionRunIntervalMillis: 0,
-});
-export const memoryPool = createPool({
-    create: () => {
-        const map = new Map();
-        // @ts-ignore
-        map.del = map.delete;
-        return Promise.resolve(map);
-    },
-    destroy: (client) => {
-        client.clear();
-        return Promise.resolve();
-    },
-}, {
-    max: 10,
-    min: 0,
-});
-export const withCache = (cb) => __awaiter(void 0, void 0, void 0, function* () {
-    const pool = config.redisUrl ? cachePool : memoryPool;
-    const client = yield pool.acquire();
-    if (client) {
-        try {
-            return cb(client);
-        }
-        finally {
-            // release client to pool, let other use
-            yield pool.release(client);
-        }
-    }
-});
-export const formatKey = (key) => {
-    return `${cacheKeyPrefix}${key}`;
-};
-export const useCache = {
-    get: (key) => {
-        return withCache((client) => __awaiter(void 0, void 0, void 0, function* () {
-            const value = yield client.get(formatKey(key));
-            try {
-                return JSON.parse(value);
-            }
-            catch (error) {
-                // ignore error
-            }
-            return value;
-        }));
-    },
-    set: (key, value, options) => {
-        return withCache((client) => {
-            const formatValue = typeof value === 'string' ? value : JSON.stringify(value);
-            return client.set(formatKey(key), formatValue, Object.assign({ PX: ttl }, options));
-        });
-    },
-    remove: (key) => {
-        return withCache((client) => {
-            return client.del(formatKey(key));
-        });
-    },
-    list: (key = '*') => {
-        return withCache((client) => {
-            return client.keys(formatKey(key));
-        });
-    },
-};

package/esm/config.d.ts

@@ -1,10 +0,0 @@
-export declare const logger: any;
-export declare const config: {
-    redisUrl: string;
-    dataDir: string;
-    appDir: string;
-    appUrl: string;
-    puppeteerPath: string;
-    cacheDir: string;
-    testOnInitialize: boolean;
-};

package/esm/crawler.d.ts

@@ -1,28 +0,0 @@
-import { JobState } from './db/job';
-import { SnapshotModel } from './db/snapshot';
-export declare function createCrawlQueue(): void;
-export declare function getDataDir(): Promise<{
-    htmlDir: string;
-    screenshotDir: string;
-}>;
-export declare const getPageContent: ({ url, formatPageContent, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage, }: {
-    url: string;
-    formatPageContent?: Function;
-    includeScreenshot?: boolean;
-    includeHtml?: boolean;
-    width?: number;
-    height?: number;
-    quality?: number;
-    timeout?: number;
-    fullPage?: boolean;
-}) => Promise<{
-    html: string;
-    screenshot: Uint8Array<ArrayBufferLike> | null;
-}>;
-export declare function createCrawlJob(params: JobState, callback?: (snapshot: SnapshotModel | null) => void): Promise<any>;
-export declare function getJob(condition: Partial<JobState>): Promise<any>;
-export declare function formatSnapshot(snapshot: SnapshotModel, columns?: Array<keyof SnapshotModel>): Promise<SnapshotModel>;
-/**
- * get snapshot from db or crawl queue
- */
-export declare function getSnapshot(jobId: string): Promise<SnapshotModel | null>;

package/esm/db/index.d.ts

@@ -1 +0,0 @@
-export declare function ensureDatabase(): Promise<void>;

package/esm/db/job.js

@@ -1,50 +0,0 @@
-import { DataTypes, Model } from '@sequelize/core';
-class Job extends Model {
-}
-export { Job };
-export function initJobModel(sequelize) {
-    Job.init({
-        id: {
-            type: DataTypes.STRING(40),
-            primaryKey: true,
-        },
-        queue: {
-            type: DataTypes.STRING(32),
-            allowNull: false,
-        },
-        job: {
-            type: DataTypes.JSON,
-            allowNull: false,
-        },
-        retryCount: {
-            type: DataTypes.INTEGER,
-        },
-        delay: {
-            type: DataTypes.INTEGER,
-        },
-        willRunAt: {
-            type: DataTypes.INTEGER,
-        },
-        cancelled: {
-            type: DataTypes.BOOLEAN,
-            defaultValue: false,
-        },
-        createdAt: {
-            type: DataTypes.DATE,
-            defaultValue: DataTypes.NOW,
-            index: true,
-        },
-        updatedAt: {
-            type: DataTypes.DATE,
-            defaultValue: DataTypes.NOW,
-            index: true,
-        },
-    }, {
-        sequelize,
-        indexes: [{ fields: ['queue'] }],
-        modelName: 'job',
-        tableName: 'jobs',
-        timestamps: true,
-    });
-    return Job;
-}