@arcblock/crawler 1.0.0 → 1.0.2

package/esm/blocklet.js

@@ -0,0 +1,190 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import Cron from '@abtnode/cron';
+import { components } from '@blocklet/sdk/lib/config';
+import debounce from 'lodash/debounce';
+import { joinURL } from 'ufo';
+import { useCache } from './cache';
+import { config, logger } from './config';
+import { createCrawlJob } from './crawler';
+import { closeBrowser, getBrowser } from './puppeteer';
+import { getComponentInfo, getRelativePath, getSitemapList } from './utils';
+// record crawl blocklet running
+const crawlBlockletRunningMap = new Map();
+// crawl blocklet sitemap urls
+export const crawlBlocklet = () => __awaiter(void 0, void 0, void 0, function* () {
+    // @ts-ignore
+    const { mountPoint, did } = getComponentInfo();
+    if (crawlBlockletRunningMap.has(did) && crawlBlockletRunningMap.get(did)) {
+        logger.info(`Crawler blocklet ${did} is running, skip it`);
+        return;
+    }
+    // check has browser can use
+    try {
+        const browser = yield getBrowser();
+        if (!browser) {
+            throw new Error('No Browser can use');
+        }
+        logger.info('Crawler blocklet existing can use browser');
+    }
+    catch (error) {
+        logger.info(`Crawler blocklet abort by error: ${(error === null || error === void 0 ? void 0 : error.message) || (error === null || error === void 0 ? void 0 : error.reason) || error}`);
+        return;
+    }
+    const { appUrl } = config;
+    if (!appUrl) {
+        throw new Error('appUrl not found');
+    }
+    const sitemapList = yield getSitemapList(appUrl);
+    const matchMountPoint = joinURL(appUrl, !mountPoint || mountPoint === '/' ? '' : mountPoint);
+    const otherMountPointList = components
+        .filter((item) => item.mountPoint && item.mountPoint !== mountPoint)
+        .map((item) => item.mountPoint);
+    // get can use loc
+    const blockletLocList = sitemapList.filter((item) => {
+        var _a;
+        if (mountPoint !== '/') {
+            return ((_a = item === null || item === void 0 ? void 0 : item.url) === null || _a === void 0 ? void 0 : _a.indexOf(matchMountPoint)) > -1;
+        }
+        // if mountPoint is /, skip other mountPoint
+        return otherMountPointList.every((mountPoint) => { var _a; return ((_a = item === null || item === void 0 ? void 0 : item.url) === null || _a === void 0 ? void 0 : _a.indexOf(mountPoint)) === -1; });
+    });
+    const canUseBlockletLocList = [];
+    const lastmodMap = new Map();
+    let skipBlockletLocTotal = 0;
+    let blockletLocTotal = 0;
+    yield Promise.all(blockletLocList.map((item) => __awaiter(void 0, void 0, void 0, function* () {
+        var _a;
+        let tempLocList = [];
+        if (item.url) {
+            tempLocList.push(item.url);
+        }
+        if (((_a = item === null || item === void 0 ? void 0 : item.links) === null || _a === void 0 ? void 0 : _a.length) > 0) {
+            tempLocList.push(...item.links.map((ytem) => ytem.url));
+        }
+        blockletLocTotal += tempLocList.length;
+        // @ts-ignore
+        tempLocList = (yield Promise.all(tempLocList.map((loc) => __awaiter(void 0, void 0, void 0, function* () {
+            try {
+                const { lastModified: cacheLastModified } = yield useCache.get(getRelativePath(loc));
+                // sitemap item lastmod is same as cache lastModified, skip it
+                if (item.lastmod &&
+                    cacheLastModified &&
+                    new Date(cacheLastModified).getTime() === new Date(item.lastmod).getTime()) {
+                    skipBlockletLocTotal++;
+                    return false;
+                }
+                return loc;
+            }
+            catch (error) {
+                // ignore error
+            }
+            // if can not get cache, return loc
+            return loc;
+        })))).filter(Boolean);
+        tempLocList.forEach((loc) => {
+            if (item.lastmod)
+                lastmodMap.set(loc, item.lastmod);
+        });
+        canUseBlockletLocList.push(...tempLocList);
+    })));
+    const crawlerLogText = (step = '') => [
+        `Crawler sitemap.xml about ${did} ${step}: `,
+        {
+            blockletLocTotal,
+            canUseBlockletLocTotal: canUseBlockletLocList.length,
+            skipBlockletLocTotal,
+            lastmodMapTotal: lastmodMap.size,
+        },
+    ];
+    logger.info(...crawlerLogText('start'));
+    try {
+        // record crawl blocklet running
+        crawlBlockletRunningMap.set(did, true);
+        yield createCrawlJob({
+            // @ts-ignore
+            urls: canUseBlockletLocList,
+            saveToRedis: true,
+            lastmodMap,
+            // formatPageContent: async ({ page }: { page: any; url: string; lastmod?: string }) => {
+            //   const pageContent = await page.evaluate(() => {
+            //     const removeElements = (tagName: string) => {
+            //       const elements = document.querySelectorAll(tagName);
+            //       for (let i = elements.length - 1; i >= 0; i--) {
+            //         try {
+            //           elements[i]?.parentNode?.removeChild(elements[i] as Node);
+            //         } catch (error) {
+            //           // do noting
+            //         }
+            //       }
+            //     };
+            //     // remove script, style, link, noscript
+            //     // removeElements('script');
+            //     // removeElements('style');
+            //     // removeElements('link');
+            //     // removeElements('noscript');
+            //     // remove uploader
+            //     removeElements('[id="uploader-container"]');
+            //     removeElements('[class^="uppy-"]');
+            //     // remove point up component
+            //     removeElements('[id="point-up-component"]');
+            //     // add meta tag to record crawler
+            //     const meta = document.createElement('meta');
+            //     meta.name = 'blocklet-crawler';
+            //     meta.content = 'true';
+            //     document.head.appendChild(meta);
+            //     return document.documentElement.outerHTML;
+            //   });
+            //   return pageContent;
+            // },
+        });
+        logger.info(...crawlerLogText('success'));
+        yield closeBrowser({
+            trimCache: true,
+        });
+    }
+    catch (error) {
+        logger.info('Crawler blocklet abort by error', error);
+    }
+    finally {
+        // delete crawl blocklet running
+        crawlBlockletRunningMap.delete(did);
+    }
+});
+const CRON_CRAWL_BLOCKLET_KEY = 'cron-crawl-blocklet';
+let cronCrawlBlockletJob = null;
+// init cron crawl blocklet
+export const initCronCrawlBlocklet = ({ time = '0 0 */12 * * *', // every 12 hours
+options, } = {}) => {
+    if (!cronCrawlBlockletJob) {
+        cronCrawlBlockletJob = Cron.init({
+            context: {},
+            jobs: [
+                {
+                    name: CRON_CRAWL_BLOCKLET_KEY,
+                    time,
+                    fn: debounce(crawlBlocklet),
+                    options: Object.assign({ runOnInit: false }, options),
+                },
+            ],
+            onError: (err) => {
+                console.error('run job failed', err);
+            },
+        });
+    }
+    return cronCrawlBlockletJob;
+};
+export const cancelCronCrawlBlocklet = () => {
+    if (cronCrawlBlockletJob) {
+        cronCrawlBlockletJob.jobs[CRON_CRAWL_BLOCKLET_KEY].stop();
+        cronCrawlBlockletJob = null;
+        logger.info('Cron crawl blocklet stop, clear crawl queue');
+    }
+};

package/esm/cache.d.ts

@@ -0,0 +1,10 @@
+export declare const cachePool: import("generic-pool").Pool<any>;
+export declare const memoryPool: import("generic-pool").Pool<Map<string, any>>;
+export declare const withCache: (cb: Function) => Promise<any>;
+export declare const formatKey: (key: string) => string;
+export declare const useCache: {
+    get: (key: string) => Promise<any>;
+    set: (key: string, value: any, options?: any) => Promise<any>;
+    remove: (key: string) => Promise<any>;
+    list: (key?: string) => Promise<any>;
+};

package/esm/cache.js

@@ -0,0 +1,114 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var _a;
+import { createPool } from 'generic-pool';
+import { createClient } from 'redis';
+import { config, logger } from './config';
+const cacheKeyPrefix = ((_a = process.env) === null || _a === void 0 ? void 0 : _a.BLOCKLET_REAL_DID) ? `${process.env.BLOCKLET_REAL_DID}:` : '';
+const MAX_REDIS_RETRY = 3;
+const ttl = 1000 * 60 * 60 * 24 * 7;
+export const cachePool = createPool({
+    create: () => __awaiter(void 0, void 0, void 0, function* () {
+        try {
+            const { redisUrl } = config;
+            const redisClient = createClient({
+                url: redisUrl,
+                socket: {
+                    // @ts-ignore
+                    reconnectStrategy: (retries) => {
+                        if (retries >= MAX_REDIS_RETRY) {
+                            return new Error('Retry Time Exhausted');
+                        }
+                        return Math.min(retries * 500, 1000 * 3);
+                    },
+                },
+            });
+            redisClient.on('error', (err) => logger.warn('Redis Client Error:', err));
+            yield redisClient.connect();
+            logger.info(`Successfully connected to Redis: ${redisUrl}`);
+            return redisClient;
+        }
+        catch (error) {
+            logger.warn('Redis connection failed', error);
+            return null;
+        }
+    }),
+    destroy: (client) => __awaiter(void 0, void 0, void 0, function* () {
+        // if is redis client
+        if (client.isReady) {
+            yield client.quit();
+        }
+    }),
+}, {
+    max: 2, // 2 clients
+    min: 0,
+    // evictionRunIntervalMillis: 0,
+});
+export const memoryPool = createPool({
+    create: () => {
+        const map = new Map();
+        // @ts-ignore
+        map.del = map.delete;
+        return Promise.resolve(map);
+    },
+    destroy: (client) => {
+        client.clear();
+        return Promise.resolve();
+    },
+}, {
+    max: 10,
+    min: 0,
+});
+export const withCache = (cb) => __awaiter(void 0, void 0, void 0, function* () {
+    const pool = config.redisUrl ? cachePool : memoryPool;
+    const client = yield pool.acquire();
+    if (client) {
+        try {
+            return cb(client);
+        }
+        finally {
+            // release client to pool, let other use
+            yield pool.release(client);
+        }
+    }
+});
+export const formatKey = (key) => {
+    return `${cacheKeyPrefix}${key}`;
+};
+export const useCache = {
+    get: (key) => {
+        return withCache((client) => __awaiter(void 0, void 0, void 0, function* () {
+            const value = yield client.get(formatKey(key));
+            try {
+                return JSON.parse(value);
+            }
+            catch (error) {
+                // ignore error
+            }
+            return value;
+        }));
+    },
+    set: (key, value, options) => {
+        return withCache((client) => {
+            const formatValue = typeof value === 'string' ? value : JSON.stringify(value);
+            return client.set(formatKey(key), formatValue, Object.assign({ PX: ttl }, options));
+        });
+    },
+    remove: (key) => {
+        return withCache((client) => {
+            return client.del(formatKey(key));
+        });
+    },
+    list: (key = '*') => {
+        return withCache((client) => {
+            return client.keys(formatKey(key));
+        });
+    },
+};

package/esm/config.d.ts

@@ -0,0 +1,10 @@
+export declare const logger: any;
+export declare const config: {
+    redisUrl: string;
+    dataDir: string;
+    appDir: string;
+    appUrl: string;
+    puppeteerPath: string;
+    cacheDir: string;
+    testOnInitialize: boolean;
+};

package/esm/config.js

@@ -0,0 +1,11 @@
+import createLogger from '@blocklet/logger';
+export const logger = createLogger('crawler', { level: process.env.LOG_LEVEL || 'info' });
+export const config = {
+    redisUrl: process.env.REDIS_URL,
+    dataDir: process.env.BLOCKLET_DATA_DIR,
+    appDir: process.env.BLOCKLET_APP_DIR || process.cwd(),
+    appUrl: process.env.BLOCKLET_APP_URL,
+    puppeteerPath: process.env.PUPPETEER_EXECUTABLE_PATH,
+    cacheDir: process.env.BLOCKLET_CACHE_DIR,
+    testOnInitialize: process.env.NODE_ENV === 'production',
+};

package/esm/crawler.d.ts

@@ -0,0 +1,28 @@
+import { JobState } from './db/job';
+import { SnapshotModel } from './db/snapshot';
+export declare function createCrawlQueue(): void;
+export declare function getDataDir(): Promise<{
+    htmlDir: string;
+    screenshotDir: string;
+}>;
+export declare const getPageContent: ({ url, formatPageContent, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage, }: {
+    url: string;
+    formatPageContent?: Function;
+    includeScreenshot?: boolean;
+    includeHtml?: boolean;
+    width?: number;
+    height?: number;
+    quality?: number;
+    timeout?: number;
+    fullPage?: boolean;
+}) => Promise<{
+    html: string;
+    screenshot: Uint8Array<ArrayBufferLike> | null;
+}>;
+export declare function createCrawlJob(params: JobState, callback?: (snapshot: SnapshotModel | null) => void): Promise<any>;
+export declare function getJob(condition: Partial<JobState>): Promise<any>;
+export declare function formatSnapshot(snapshot: SnapshotModel, columns?: Array<keyof SnapshotModel>): Promise<SnapshotModel>;
+/**
+ * get snapshot from db or crawl queue
+ */
+export declare function getSnapshot(jobId: string): Promise<SnapshotModel | null>;

package/esm/crawler.js

@@ -0,0 +1,301 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import createQueue from '@abtnode/queue';
+import SequelizeStore from '@abtnode/queue/lib/store/sequelize';
+import sequelize from '@sequelize/core';
+import { randomUUID } from 'crypto';
+import fs from 'fs-extra';
+import pick from 'lodash/pick';
+import path from 'path';
+import { joinURL } from 'ufo';
+import { config, logger } from './config';
+import { Job } from './db/job';
+import { Snapshot } from './db/snapshot';
+import { initPage } from './puppeteer';
+import { formatUrl, isAcceptCrawler, md5 } from './utils';
+const { BaseState } = require('@abtnode/models');
+let crawlQueue;
+export function createCrawlQueue() {
+    const db = new BaseState(Job);
+    crawlQueue = createQueue({
+        store: new SequelizeStore(db, 'crawler'),
+        concurrency: 1,
+        onJob: (job) => __awaiter(this, void 0, void 0, function* () {
+            logger.debug('job start:', job);
+            const canCrawl = yield isAcceptCrawler(job.url);
+            if (!canCrawl) {
+                logger.error(`failed to crawl ${job.url}, denied by robots.txt`, job);
+                const snapshot = convertJobToSnapshot({
+                    job,
+                    snapshot: {
+                        status: 'failed',
+                        error: 'Denied by robots.txt',
+                    },
+                });
+                yield Snapshot.upsert(snapshot);
+                return snapshot;
+            }
+            // if index reach autoCloseBrowserCount, close browser
+            // try {
+            //   if (index >= autoCloseBrowserCount) {
+            //     await closeBrowser({ trimCache: false });
+            //   }
+            // } catch (error) {
+            //   logger.error('failed to close browser when queue index reached autoCloseBrowserCount:', error);
+            // }
+            try {
+                // get page content later
+                const result = yield getPageContent(job);
+                if (!result || (!result.html && !result.screenshot)) {
+                    logger.error(`failed to crawl ${job.url}, empty content`, job);
+                    const snapshot = convertJobToSnapshot({
+                        job,
+                        snapshot: {
+                            status: 'failed',
+                            error: 'Failed to crawl content',
+                        },
+                    });
+                    yield Snapshot.upsert(snapshot);
+                    return snapshot;
+                }
+                // save html and screenshot to data dir
+                const { screenshotPath, htmlPath } = yield saveSnapshotToLocal({
+                    screenshot: result.screenshot,
+                    html: result.html,
+                });
+                // const lastModified = job.lastmodMap?.get(url) || new Date().toISOString();
+                const snapshot = convertJobToSnapshot({
+                    job,
+                    snapshot: {
+                        status: 'success',
+                        screenshot: screenshotPath === null || screenshotPath === void 0 ? void 0 : screenshotPath.replace(config.dataDir, ''),
+                        html: htmlPath === null || htmlPath === void 0 ? void 0 : htmlPath.replace(config.dataDir, ''),
+                    },
+                });
+                yield Snapshot.upsert(snapshot);
+                return snapshot;
+                // save to redis
+                // if (saveToRedis) {
+                //   useCache.set(url, {
+                //     html: result.html || '',
+                //     lastModified,
+                //   });
+                //   logger.info(`success to crawl ${url}`, job);
+                //   return result;
+                // }
+            }
+            catch (error) {
+                logger.error(`Failed to crawl ${job.url}`, { error, job });
+                console.error(error.stack);
+                const snapshot = convertJobToSnapshot({
+                    job,
+                    snapshot: {
+                        status: 'failed',
+                        error: 'Internal error',
+                    },
+                });
+                yield Snapshot.upsert(snapshot);
+                return snapshot;
+            }
+        }),
+    });
+}
+export function getDataDir() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const htmlDir = path.join(config.dataDir, 'data', 'html');
+        const screenshotDir = path.join(config.dataDir, 'data', 'screenshot');
+        yield fs.ensureDir(htmlDir);
+        yield fs.ensureDir(screenshotDir);
+        return { htmlDir, screenshotDir };
+    });
+}
+function saveSnapshotToLocal(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ screenshot, html }) {
+        const { htmlDir, screenshotDir } = yield getDataDir();
+        let screenshotPath = null;
+        let htmlPath = null;
+        if (screenshot) {
+            const hash = md5(screenshot);
+            screenshotPath = path.join(screenshotDir, `${hash}.webp`);
+            logger.debug('saveSnapshotToLocal.screenshot', { screenshotPath });
+            yield fs.writeFile(screenshotPath, screenshot);
+        }
+        if (html) {
+            const hash = md5(html);
+            htmlPath = path.join(htmlDir, `${hash}.html`);
+            logger.debug('saveSnapshotToLocal.html', { htmlPath });
+            yield fs.writeFile(htmlPath, html);
+        }
+        return {
+            screenshotPath,
+            htmlPath,
+        };
+    });
+}
+function formatHtml(htmlString) {
+    if (htmlString.includes('<h2>Unexpected Application Error!</h2>')) {
+        return '';
+    }
+    return htmlString;
+}
+export const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, formatPageContent, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, timeout = 60 * 1000, fullPage = false, }) {
+    logger.debug('getPageContent', { url, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage });
+    const page = yield initPage();
+    if (width && height) {
+        yield page.setViewport({ width, height });
+    }
+    let html = null;
+    let screenshot = null;
+    try {
+        const response = yield page.goto(url, { timeout });
+        if (!response) {
+            throw new Error(`Failed to load page: response is null for ${url}`);
+        }
+        const statusCode = response.status();
+        logger.debug('getPageContent.response', { response, statusCode });
+        if (![200, 304].includes(statusCode)) {
+            throw new Error(`Request failed with status ${statusCode}, in ${url}`);
+        }
+        // await for networkidle0
+        // https://pptr.dev/api/puppeteer.page.goforward/#remarks
+        yield page.waitForNetworkIdle({
+            idleTime: 2 * 1000,
+        });
+        // get screenshot
+        if (includeScreenshot) {
+            try {
+                screenshot = yield page.screenshot({ fullPage, quality, type: 'webp' });
+            }
+            catch (err) {
+                logger.error('Failed to get screenshot:', err);
+            }
+        }
+        // get html
+        if (includeHtml) {
+            if (formatPageContent) {
+                html = yield formatPageContent({ page, url });
+            }
+            else {
+                html = yield page.content();
+            }
+        }
+    }
+    catch (error) {
+        logger.error('Failed to get page content:', error);
+        throw error;
+    }
+    finally {
+        yield page.close();
+    }
+    html = formatHtml(html || '');
+    return {
+        html,
+        screenshot,
+    };
+});
+export function createCrawlJob(params, callback) {
+    return __awaiter(this, void 0, void 0, function* () {
+        params = Object.assign(Object.assign({}, params), { url: formatUrl(params.url) });
+        // skip duplicate job
+        const existsJob = yield getJob({
+            url: params.url,
+            includeScreenshot: params.includeScreenshot,
+            includeHtml: params.includeHtml,
+            quality: params.quality,
+            width: params.width,
+            height: params.height,
+            fullPage: params.fullPage,
+        });
+        logger.info('create crawl job', params);
+        if (existsJob) {
+            logger.warn(`Crawl job already exists for ${params.url}, skip`);
+            return existsJob.id;
+        }
+        const jobId = randomUUID();
+        const job = crawlQueue.push(Object.assign(Object.assign({}, params), { id: jobId }));
+        job.on('finished', ({ result }) => {
+            logger.info(`Crawl completed ${params.url}, status: ${result ? 'success' : 'failed'}`, { job: params, result });
+            callback === null || callback === void 0 ? void 0 : callback(result);
+        });
+        job.on('failed', ({ error }) => {
+            logger.error(`Failed to execute job for ${params.url}`, { error, job: params });
+            callback === null || callback === void 0 ? void 0 : callback(null);
+        });
+        return jobId;
+    });
+}
+// @ts-ignore
+export function getJob(condition) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const where = Object.keys(condition)
+            .filter((key) => condition[key] !== undefined)
+            .map((key) => {
+            return sequelize.where(sequelize.fn('json_extract', sequelize.col('job'), `$.${key}`), condition[key]);
+        });
+        const job = yield crawlQueue.store.db.findOne({
+            where: {
+                [sequelize.Op.and]: where,
+            },
+        });
+        if (job) {
+            return job.job;
+        }
+        return null;
+    });
+}
+function convertJobToSnapshot({ job, snapshot }) {
+    return Object.assign({ 
+        // @ts-ignore
+        jobId: job.jobId || job.id, url: job.url, options: {
+            width: job.width,
+            height: job.height,
+            includeScreenshot: job.includeScreenshot,
+            includeHtml: job.includeHtml,
+            quality: job.quality,
+            fullPage: job.fullPage,
+        } }, snapshot);
+}
+export function formatSnapshot(snapshot, columns) {
+    return __awaiter(this, void 0, void 0, function* () {
+        let data = Object.assign({}, snapshot);
+        // format screenshot path to full url
+        if (data.screenshot) {
+            data.screenshot = joinURL(config.appUrl, data.screenshot);
+        }
+        // format html path to string
+        if (data.html) {
+            const html = yield fs.readFile(path.join(config.dataDir, data.html));
+            data.html = html.toString();
+        }
+        if (columns === null || columns === void 0 ? void 0 : columns.length) {
+            data = pick(data, columns);
+        }
+        return data;
+    });
+}
+/**
+ * get snapshot from db or crawl queue
+ */
+export function getSnapshot(jobId) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const snapshotModel = yield Snapshot.findByPk(jobId);
+        if (snapshotModel) {
+            return snapshotModel.toJSON();
+        }
+        const job = yield getJob({ id: jobId });
+        if (job) {
+            return {
+                jobId,
+                status: 'pending',
+            };
+        }
+        return null;
+    });
+}

package/esm/db/index.d.ts

@@ -0,0 +1 @@
+export declare function ensureDatabase(): Promise<void>;