@arcblock/crawler 1.0.6 → 1.1.1

package/{esm → lib/esm}/utils.js

@@ -7,31 +7,22 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-import { components, env } from '@blocklet/sdk/lib/config';
-import axios from 'axios';
+import Axios from 'axios';
 import flattenDeep from 'lodash/flattenDeep';
 import uniq from 'lodash/uniq';
 import { createHash } from 'node:crypto';
 import robotsParser from 'robots-parser';
 import { parseSitemap } from 'sitemap';
 import { Readable } from 'stream';
-import { joinURL } from 'ufo';
-export const api = axios.create({
-    timeout: 1000 * 10,
+import { joinURL, withQuery } from 'ufo';
+import { logger } from './config';
+export const axios = Axios.create({
+    timeout: 1000 * 30,
     headers: {
         'Content-Type': 'application/json',
     },
 });
-export const sleep = (ms) => {
-    return new Promise((resolve) => {
-        setTimeout(resolve, ms);
-    });
-};
-export const CRAWLER_FLAG = 'x-crawler';
-export const isSelfCrawler = (req) => {
-    const ua = req.get('user-agent') || '';
-    return req.get(CRAWLER_FLAG) === 'true' || `${ua}`.toLowerCase().indexOf('headless') !== -1;
-};
+export const CRAWLER_FLAG = 'x-arcblock-crawler';
 /**
  * A default set of user agent patterns for bots/crawlers that do not perform
  * well with pages that require JavaScript.
@@ -87,12 +78,8 @@ const botUserAgents = [
     /AlibabaGroup/i,
     /adaptive-edge-crawler/i,
 ];
-const isSpider = (ua) => botUserAgents.some((spider) => {
-    return spider.test(ua);
-});
 /**
- * A default set of file extensions for static assets that do not need to be
- * proxied.
+ * A default set of file extensions for static assets that do not need to be proxied.
  */
 const staticFileExtensions = [
     'ai',
@@ -137,81 +124,87 @@ const staticFileExtensions = [
     'xml',
     'zip',
 ];
-export const getDefaultRobotsUrl = (url) => {
-    const { origin } = new URL(url);
-    return joinURL(origin, 'robots.txt?nocache=1');
+export const sleep = (ms) => {
+    return new Promise((resolve) => {
+        setTimeout(resolve, ms);
+    });
+};
+/**
+ * Check if the request is a arcblock crawler
+ */
+export const isSelfCrawler = (req) => {
+    const ua = req.get('user-agent') || '';
+    return req.get(CRAWLER_FLAG) === 'true' || ua.toLowerCase().indexOf('headless') !== -1;
 };
+/**
+ * Check if the request is a static file
+ */
+export function isStaticFile(req) {
+    const excludeUrlPattern = new RegExp(`\\.(${staticFileExtensions.join('|')})$`, 'i');
+    return excludeUrlPattern.test(req.path);
+}
+/**
+ * Check if the request is a spider
+ */
+export function isSpider(req) {
+    const ua = req.get('user-agent') || '';
+    return botUserAgents.some((spider) => spider.test(ua));
+}
+/**
+ * Get and parse the robots.txt by `robots-parser`
+ */
 export function getRobots(url) {
     return __awaiter(this, void 0, void 0, function* () {
         const { origin } = new URL(url);
         const robotsUrl = joinURL(origin, 'robots.txt?nocache=1');
-        const { data } = yield api.get(robotsUrl).catch(() => ({
-            data: '',
-        }));
+        const { data } = yield axios.get(robotsUrl).catch((error) => {
+            logger.warn(`Failed to fetch robots.txt from ${robotsUrl}:`, { error });
+            return { data: null };
+        });
         return data ? robotsParser(robotsUrl, data) : null;
     });
 }
-export const getDefaultSitemapUrl = (url) => {
-    const { origin } = new URL(url);
-    return joinURL(origin, 'sitemap.xml?nocache=1');
-};
+/**
+ * Check if the url is allowed to crawl from robots.txt
+ */
 export const isAcceptCrawler = (url) => __awaiter(void 0, void 0, void 0, function* () {
     const robots = yield getRobots(url);
     const isAllowed = robots ? yield robots.isAllowed(url) : true;
     return isAllowed;
 });
+/**
+ * Get and parse the sitemap.xml by `sitemap` package
+ */
 export const getSitemapList = (url) => __awaiter(void 0, void 0, void 0, function* () {
-    let sitemapUrlList = [getDefaultSitemapUrl(url)];
+    let sitemapUrlList = [];
     const robots = yield getRobots(url);
     if (robots) {
-        const robotsTxtSitemapUrlList = (yield robots.getSitemaps()) || [];
-        if (robotsTxtSitemapUrlList.length > 0) {
-            sitemapUrlList = robotsTxtSitemapUrlList;
-        }
+        sitemapUrlList = (yield robots.getSitemaps()) || [];
+    }
+    if (!sitemapUrlList.length) {
+        const { origin } = new URL(url);
+        sitemapUrlList.push(joinURL(origin, 'sitemap.xml?nocache=1'));
     }
     // loop site map url list
     const sitemapList = yield Promise.all(sitemapUrlList.map((sitemapUrl) => __awaiter(void 0, void 0, void 0, function* () {
-        const newUrl = new URL(sitemapUrl);
-        newUrl.searchParams.set('nocache', '1');
-        sitemapUrl = newUrl.toString();
-        const { data: sitemapTxt } = yield api.get(sitemapUrl).catch(() => ({
-            data: '',
-        }));
-        if (sitemapTxt) {
-            const stream = Readable.from([sitemapTxt]);
-            const sitemapJson = yield parseSitemap(stream);
-            return sitemapJson;
+        sitemapUrl = withQuery(sitemapUrl, { nocache: '1' });
+        try {
+            const { data: sitemapTxt } = yield axios.get(sitemapUrl).catch(() => ({
+                data: '',
+            }));
+            if (sitemapTxt) {
+                const stream = Readable.from([sitemapTxt]);
+                const sitemapJson = yield parseSitemap(stream);
+                return sitemapJson;
+            }
+        }
+        catch (error) {
+            logger.error(`Could not get sitemap from ${sitemapUrl}`, { error });
         }
         return [];
     })));
     return uniq(flattenDeep(sitemapList.filter(Boolean)));
 });
-export const isBotUserAgent = (req) => {
-    const ua = req.get('user-agent');
-    const excludeUrlPattern = new RegExp(`\\.(${staticFileExtensions.join('|')})$`, 'i');
-    if (ua === undefined || !isSpider(ua) || excludeUrlPattern.test(req.path)) {
-        return false;
-    }
-    return true;
-};
-export const getComponentInfo = () => {
-    return components.find((item) => item.did === env.componentDid) || {};
-};
-export const getFullUrl = (req) => {
-    const blockletPathname = req.headers['x-path-prefix']
-        ? joinURL(req.headers['x-path-prefix'], req.originalUrl)
-        : req.originalUrl;
-    return joinURL(env.appUrl, blockletPathname);
-};
-export const getRelativePath = (url) => {
-    try {
-        return new URL(url).pathname;
-    }
-    catch (error) {
-        // ignore error
-    }
-    return url;
-};
 export const formatUrl = (url) => {
     return url.replace(/\/$/, '').trim();
 };

package/package.json

@@ -1,16 +1,28 @@
 {
   "name": "@arcblock/crawler",
-  "version": "1.0.6",
-  "main": "dist/index.js",
-  "module": "esm/index.js",
-  "types": "dist/index.d.ts",
+  "version": "1.1.1",
+  "main": "lib/cjs/index.js",
+  "module": "lib/esm/index.js",
+  "types": "lib/cjs/index.d.ts",
   "publishConfig": {
     "access": "public"
   },
   "files": [
-    "dist",
-    "esm"
+    "lib",
+    "*.d.ts"
   ],
+  "exports": {
+    ".": {
+      "require": "./lib/cjs/index.js",
+      "import": "./lib/esm/index.js",
+      "types": "./lib/cjs/index.d.ts"
+    },
+    "./utils": {
+      "require": "./lib/cjs/utils.js",
+      "import": "./lib/esm/utils.js",
+      "types": "./lib/cjs/utils.d.ts"
+    }
+  },
   "lint-staged": {
     "*.{mjs,js,jsx,ts,tsx}": [
       "prettier --write",
@@ -36,61 +48,37 @@
     "@abtnode/cron": "^1.16.43",
     "@abtnode/models": "^1.16.43",
     "@abtnode/queue": "^1.16.43",
-    "@arcblock/did-auth": "^1.19.15",
-    "@arcblock/did-auth-storage-nedb": "^1.7.1",
     "@blocklet/logger": "^1.16.43",
     "@blocklet/puppeteer": "^22.11.3",
     "@blocklet/sdk": "^1.16.43",
-    "@ocap/client": "^1.19.15",
-    "@ocap/mcrypto": "^1.19.15",
-    "@ocap/util": "^1.20.11",
-    "@ocap/wallet": "^1.19.15",
     "@sequelize/core": "7.0.0-alpha.46",
     "@sequelize/sqlite3": "7.0.0-alpha.46",
     "axios": "^1.7.9",
-    "cookie-parser": "^1.4.7",
-    "cors": "^2.8.5",
-    "dotenv-flow": "^4.1.0",
-    "express": "^4.21.2",
-    "express-async-errors": "^3.1.1",
     "fs-extra": "^11.2.0",
     "generic-pool": "^3.9.0",
     "lodash": "^4.17.21",
     "lru-cache": "^10.4.3",
-    "p-queue": "6.6.2",
-    "p-wait-for": "^5.0.2",
     "redis": "^4.7.0",
     "robots-parser": "^3.0.1",
     "sequelize": "^6.37.7",
     "sitemap": "^7.1.2",
     "sqlite3": "^5.1.7",
     "ufo": "^1.5.4",
-    "url-join": "^4.0.1"
+    "p-map": "^7.0.3"
   },
   "devDependencies": {
     "@blocklet/js-sdk": "^1.16.39",
-    "@types/cookie-parser": "^1.4.8",
-    "@types/cors": "^2.8.17",
     "@types/dotenv-flow": "^3.3.3",
     "@types/express": "^4.17.21",
     "@types/fs-extra": "^11.0.4",
     "@types/lodash": "^4.17.16",
     "@types/node": "^20.17.19",
-    "@types/react": "^18.3.18",
-    "@types/react-dom": "^18.3.5",
-    "@vitejs/plugin-react": "^4.3.4",
+    "express": "^4.21.2",
     "bumpp": "^9.11.1",
     "nodemon": "^3.1.9",
     "npm-run-all": "^4.1.5",
     "puppeteer": "^24.8.2",
-    "react": "~18.2.0",
-    "react-dom": "~18.2.0",
-    "react-router-dom": "^6.29.0",
-    "rimraf": "^5.0.10",
     "tsx": "^4.19.3",
-    "vite": "^5.4.14",
-    "vite-plugin-blocklet": "^0.9.32",
-    "vite-plugin-svgr": "^4.3.0",
     "zx": "^8.3.2"
   },
   "importSort": {

package/dist/blocklet.d.ts

@@ -1,6 +0,0 @@
-export declare const crawlBlocklet: () => Promise<void>;
-export declare const initCronCrawlBlocklet: ({ time, options, }?: {
-    time: string;
-    options: any;
-}) => any;
-export declare const cancelCronCrawlBlocklet: () => void;

package/dist/blocklet.js

@@ -1,199 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.cancelCronCrawlBlocklet = exports.initCronCrawlBlocklet = exports.crawlBlocklet = void 0;
-const cron_1 = __importDefault(require("@abtnode/cron"));
-const config_1 = require("@blocklet/sdk/lib/config");
-const debounce_1 = __importDefault(require("lodash/debounce"));
-const ufo_1 = require("ufo");
-const cache_1 = require("./cache");
-const config_2 = require("./config");
-const crawler_1 = require("./crawler");
-const puppeteer_1 = require("./puppeteer");
-const utils_1 = require("./utils");
-// record crawl blocklet running
-const crawlBlockletRunningMap = new Map();
-// crawl blocklet sitemap urls
-const crawlBlocklet = () => __awaiter(void 0, void 0, void 0, function* () {
-    // @ts-ignore
-    const { mountPoint, did } = (0, utils_1.getComponentInfo)();
-    if (crawlBlockletRunningMap.has(did) && crawlBlockletRunningMap.get(did)) {
-        config_2.logger.info(`Crawler blocklet ${did} is running, skip it`);
-        return;
-    }
-    // check has browser can use
-    try {
-        const browser = yield (0, puppeteer_1.getBrowser)();
-        if (!browser) {
-            throw new Error('No Browser can use');
-        }
-        config_2.logger.info('Crawler blocklet existing can use browser');
-    }
-    catch (error) {
-        config_2.logger.info(`Crawler blocklet abort by error: ${(error === null || error === void 0 ? void 0 : error.message) || (error === null || error === void 0 ? void 0 : error.reason) || error}`);
-        return;
-    }
-    const { appUrl } = config_2.config;
-    if (!appUrl) {
-        throw new Error('appUrl not found');
-    }
-    const sitemapList = yield (0, utils_1.getSitemapList)(appUrl);
-    const matchMountPoint = (0, ufo_1.joinURL)(appUrl, !mountPoint || mountPoint === '/' ? '' : mountPoint);
-    const otherMountPointList = config_1.components
-        .filter((item) => item.mountPoint && item.mountPoint !== mountPoint)
-        .map((item) => item.mountPoint);
-    // get can use loc
-    const blockletLocList = sitemapList.filter((item) => {
-        var _a;
-        if (mountPoint !== '/') {
-            return ((_a = item === null || item === void 0 ? void 0 : item.url) === null || _a === void 0 ? void 0 : _a.indexOf(matchMountPoint)) > -1;
-        }
-        // if mountPoint is /, skip other mountPoint
-        return otherMountPointList.every((mountPoint) => { var _a; return ((_a = item === null || item === void 0 ? void 0 : item.url) === null || _a === void 0 ? void 0 : _a.indexOf(mountPoint)) === -1; });
-    });
-    const canUseBlockletLocList = [];
-    const lastmodMap = new Map();
-    let skipBlockletLocTotal = 0;
-    let blockletLocTotal = 0;
-    yield Promise.all(blockletLocList.map((item) => __awaiter(void 0, void 0, void 0, function* () {
-        var _a;
-        let tempLocList = [];
-        if (item.url) {
-            tempLocList.push(item.url);
-        }
-        if (((_a = item === null || item === void 0 ? void 0 : item.links) === null || _a === void 0 ? void 0 : _a.length) > 0) {
-            tempLocList.push(...item.links.map((ytem) => ytem.url));
-        }
-        blockletLocTotal += tempLocList.length;
-        // @ts-ignore
-        tempLocList = (yield Promise.all(tempLocList.map((loc) => __awaiter(void 0, void 0, void 0, function* () {
-            try {
-                const { lastModified: cacheLastModified } = yield cache_1.useCache.get((0, utils_1.getRelativePath)(loc));
-                // sitemap item lastmod is same as cache lastModified, skip it
-                if (item.lastmod &&
-                    cacheLastModified &&
-                    new Date(cacheLastModified).getTime() === new Date(item.lastmod).getTime()) {
-                    skipBlockletLocTotal++;
-                    return false;
-                }
-                return loc;
-            }
-            catch (error) {
-                // ignore error
-            }
-            // if can not get cache, return loc
-            return loc;
-        })))).filter(Boolean);
-        tempLocList.forEach((loc) => {
-            if (item.lastmod)
-                lastmodMap.set(loc, item.lastmod);
-        });
-        canUseBlockletLocList.push(...tempLocList);
-    })));
-    const crawlerLogText = (step = '') => [
-        `Crawler sitemap.xml about ${did} ${step}: `,
-        {
-            blockletLocTotal,
-            canUseBlockletLocTotal: canUseBlockletLocList.length,
-            skipBlockletLocTotal,
-            lastmodMapTotal: lastmodMap.size,
-        },
-    ];
-    config_2.logger.info(...crawlerLogText('start'));
-    try {
-        // record crawl blocklet running
-        crawlBlockletRunningMap.set(did, true);
-        yield (0, crawler_1.createCrawlJob)({
-            // @ts-ignore
-            urls: canUseBlockletLocList,
-            saveToRedis: true,
-            lastmodMap,
-            // formatPageContent: async ({ page }: { page: any; url: string; lastmod?: string }) => {
-            //   const pageContent = await page.evaluate(() => {
-            //     const removeElements = (tagName: string) => {
-            //       const elements = document.querySelectorAll(tagName);
-            //       for (let i = elements.length - 1; i >= 0; i--) {
-            //         try {
-            //           elements[i]?.parentNode?.removeChild(elements[i] as Node);
-            //         } catch (error) {
-            //           // do noting
-            //         }
-            //       }
-            //     };
-            //     // remove script, style, link, noscript
-            //     // removeElements('script');
-            //     // removeElements('style');
-            //     // removeElements('link');
-            //     // removeElements('noscript');
-            //     // remove uploader
-            //     removeElements('[id="uploader-container"]');
-            //     removeElements('[class^="uppy-"]');
-            //     // remove point up component
-            //     removeElements('[id="point-up-component"]');
-            //     // add meta tag to record crawler
-            //     const meta = document.createElement('meta');
-            //     meta.name = 'blocklet-crawler';
-            //     meta.content = 'true';
-            //     document.head.appendChild(meta);
-            //     return document.documentElement.outerHTML;
-            //   });
-            //   return pageContent;
-            // },
-        });
-        config_2.logger.info(...crawlerLogText('success'));
-        yield (0, puppeteer_1.closeBrowser)({
-            trimCache: true,
-        });
-    }
-    catch (error) {
-        config_2.logger.info('Crawler blocklet abort by error', error);
-    }
-    finally {
-        // delete crawl blocklet running
-        crawlBlockletRunningMap.delete(did);
-    }
-});
-exports.crawlBlocklet = crawlBlocklet;
-const CRON_CRAWL_BLOCKLET_KEY = 'cron-crawl-blocklet';
-let cronCrawlBlockletJob = null;
-// init cron crawl blocklet
-const initCronCrawlBlocklet = ({ time = '0 0 */12 * * *', // every 12 hours
-options, } = {}) => {
-    if (!cronCrawlBlockletJob) {
-        cronCrawlBlockletJob = cron_1.default.init({
-            context: {},
-            jobs: [
-                {
-                    name: CRON_CRAWL_BLOCKLET_KEY,
-                    time,
-                    fn: (0, debounce_1.default)(exports.crawlBlocklet),
-                    options: Object.assign({ runOnInit: false }, options),
-                },
-            ],
-            onError: (err) => {
-                console.error('run job failed', err);
-            },
-        });
-    }
-    return cronCrawlBlockletJob;
-};
-exports.initCronCrawlBlocklet = initCronCrawlBlocklet;
-const cancelCronCrawlBlocklet = () => {
-    if (cronCrawlBlockletJob) {
-        cronCrawlBlockletJob.jobs[CRON_CRAWL_BLOCKLET_KEY].stop();
-        cronCrawlBlockletJob = null;
-        config_2.logger.info('Cron crawl blocklet stop, clear crawl queue');
-    }
-};
-exports.cancelCronCrawlBlocklet = cancelCronCrawlBlocklet;

package/dist/cache.d.ts

@@ -1,10 +0,0 @@
-export declare const cachePool: import("generic-pool").Pool<any>;
-export declare const memoryPool: import("generic-pool").Pool<Map<string, any>>;
-export declare const withCache: (cb: Function) => Promise<any>;
-export declare const formatKey: (key: string) => string;
-export declare const useCache: {
-    get: (key: string) => Promise<any>;
-    set: (key: string, value: any, options?: any) => Promise<any>;
-    remove: (key: string) => Promise<any>;
-    list: (key?: string) => Promise<any>;
-};

package/dist/cache.js

@@ -1,119 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var _a;
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.useCache = exports.formatKey = exports.withCache = exports.memoryPool = exports.cachePool = void 0;
-const generic_pool_1 = require("generic-pool");
-const redis_1 = require("redis");
-const config_1 = require("./config");
-const cacheKeyPrefix = ((_a = process.env) === null || _a === void 0 ? void 0 : _a.BLOCKLET_REAL_DID) ? `${process.env.BLOCKLET_REAL_DID}:` : '';
-const MAX_REDIS_RETRY = 3;
-const ttl = 1000 * 60 * 60 * 24 * 7;
-exports.cachePool = (0, generic_pool_1.createPool)({
-    create: () => __awaiter(void 0, void 0, void 0, function* () {
-        try {
-            const { redisUrl } = config_1.config;
-            const redisClient = (0, redis_1.createClient)({
-                url: redisUrl,
-                socket: {
-                    // @ts-ignore
-                    reconnectStrategy: (retries) => {
-                        if (retries >= MAX_REDIS_RETRY) {
-                            return new Error('Retry Time Exhausted');
-                        }
-                        return Math.min(retries * 500, 1000 * 3);
-                    },
-                },
-            });
-            redisClient.on('error', (err) => config_1.logger.warn('Redis Client Error:', err));
-            yield redisClient.connect();
-            config_1.logger.info(`Successfully connected to Redis: ${redisUrl}`);
-            return redisClient;
-        }
-        catch (error) {
-            config_1.logger.warn('Redis connection failed', error);
-            return null;
-        }
-    }),
-    destroy: (client) => __awaiter(void 0, void 0, void 0, function* () {
-        // if is redis client
-        if (client.isReady) {
-            yield client.quit();
-        }
-    }),
-}, {
-    max: 2, // 2 clients
-    min: 0,
-    // evictionRunIntervalMillis: 0,
-});
-exports.memoryPool = (0, generic_pool_1.createPool)({
-    create: () => {
-        const map = new Map();
-        // @ts-ignore
-        map.del = map.delete;
-        return Promise.resolve(map);
-    },
-    destroy: (client) => {
-        client.clear();
-        return Promise.resolve();
-    },
-}, {
-    max: 10,
-    min: 0,
-});
-const withCache = (cb) => __awaiter(void 0, void 0, void 0, function* () {
-    const pool = config_1.config.redisUrl ? exports.cachePool : exports.memoryPool;
-    const client = yield pool.acquire();
-    if (client) {
-        try {
-            return cb(client);
-        }
-        finally {
-            // release client to pool, let other use
-            yield pool.release(client);
-        }
-    }
-});
-exports.withCache = withCache;
-const formatKey = (key) => {
-    return `${cacheKeyPrefix}${key}`;
-};
-exports.formatKey = formatKey;
-exports.useCache = {
-    get: (key) => {
-        return (0, exports.withCache)((client) => __awaiter(void 0, void 0, void 0, function* () {
-            const value = yield client.get((0, exports.formatKey)(key));
-            try {
-                return JSON.parse(value);
-            }
-            catch (error) {
-                // ignore error
-            }
-            return value;
-        }));
-    },
-    set: (key, value, options) => {
-        return (0, exports.withCache)((client) => {
-            const formatValue = typeof value === 'string' ? value : JSON.stringify(value);
-            return client.set((0, exports.formatKey)(key), formatValue, Object.assign({ PX: ttl }, options));
-        });
-    },
-    remove: (key) => {
-        return (0, exports.withCache)((client) => {
-            return client.del((0, exports.formatKey)(key));
-        });
-    },
-    list: (key = '*') => {
-        return (0, exports.withCache)((client) => {
-            return client.keys((0, exports.formatKey)(key));
-        });
-    },
-};

package/dist/config.d.ts

@@ -1,10 +0,0 @@
-export declare const logger: any;
-export declare const config: {
-    redisUrl: string;
-    dataDir: string;
-    appDir: string;
-    appUrl: string;
-    puppeteerPath: string;
-    cacheDir: string;
-    testOnInitialize: boolean;
-};

package/dist/crawler.d.ts

@@ -1,28 +0,0 @@
-import { JobState } from './db/job';
-import { SnapshotModel } from './db/snapshot';
-export declare function createCrawlQueue(): void;
-export declare function getDataDir(): Promise<{
-    htmlDir: string;
-    screenshotDir: string;
-}>;
-export declare const getPageContent: ({ url, formatPageContent, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage, }: {
-    url: string;
-    formatPageContent?: Function;
-    includeScreenshot?: boolean;
-    includeHtml?: boolean;
-    width?: number;
-    height?: number;
-    quality?: number;
-    timeout?: number;
-    fullPage?: boolean;
-}) => Promise<{
-    html: string;
-    screenshot: Uint8Array<ArrayBufferLike> | null;
-}>;
-export declare function createCrawlJob(params: JobState, callback?: (snapshot: SnapshotModel | null) => void): Promise<any>;
-export declare function getJob(condition: Partial<JobState>): Promise<any>;
-export declare function formatSnapshot(snapshot: SnapshotModel, columns?: Array<keyof SnapshotModel>): Promise<SnapshotModel>;
-/**
- * get snapshot from db or crawl queue
- */
-export declare function getSnapshot(jobId: string): Promise<SnapshotModel | null>;

package/dist/db/index.d.ts

@@ -1 +0,0 @@
-export declare function ensureDatabase(): Promise<void>;