@arcblock/crawler 1.0.6 → 1.1.2

package/esm/cache.js

@@ -1,114 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var _a;
-import { createPool } from 'generic-pool';
-import { createClient } from 'redis';
-import { config, logger } from './config';
-const cacheKeyPrefix = ((_a = process.env) === null || _a === void 0 ? void 0 : _a.BLOCKLET_REAL_DID) ? `${process.env.BLOCKLET_REAL_DID}:` : '';
-const MAX_REDIS_RETRY = 3;
-const ttl = 1000 * 60 * 60 * 24 * 7;
-export const cachePool = createPool({
-    create: () => __awaiter(void 0, void 0, void 0, function* () {
-        try {
-            const { redisUrl } = config;
-            const redisClient = createClient({
-                url: redisUrl,
-                socket: {
-                    // @ts-ignore
-                    reconnectStrategy: (retries) => {
-                        if (retries >= MAX_REDIS_RETRY) {
-                            return new Error('Retry Time Exhausted');
-                        }
-                        return Math.min(retries * 500, 1000 * 3);
-                    },
-                },
-            });
-            redisClient.on('error', (err) => logger.warn('Redis Client Error:', err));
-            yield redisClient.connect();
-            logger.info(`Successfully connected to Redis: ${redisUrl}`);
-            return redisClient;
-        }
-        catch (error) {
-            logger.warn('Redis connection failed', error);
-            return null;
-        }
-    }),
-    destroy: (client) => __awaiter(void 0, void 0, void 0, function* () {
-        // if is redis client
-        if (client.isReady) {
-            yield client.quit();
-        }
-    }),
-}, {
-    max: 2, // 2 clients
-    min: 0,
-    // evictionRunIntervalMillis: 0,
-});
-export const memoryPool = createPool({
-    create: () => {
-        const map = new Map();
-        // @ts-ignore
-        map.del = map.delete;
-        return Promise.resolve(map);
-    },
-    destroy: (client) => {
-        client.clear();
-        return Promise.resolve();
-    },
-}, {
-    max: 10,
-    min: 0,
-});
-export const withCache = (cb) => __awaiter(void 0, void 0, void 0, function* () {
-    const pool = config.redisUrl ? cachePool : memoryPool;
-    const client = yield pool.acquire();
-    if (client) {
-        try {
-            return cb(client);
-        }
-        finally {
-            // release client to pool, let other use
-            yield pool.release(client);
-        }
-    }
-});
-export const formatKey = (key) => {
-    return `${cacheKeyPrefix}${key}`;
-};
-export const useCache = {
-    get: (key) => {
-        return withCache((client) => __awaiter(void 0, void 0, void 0, function* () {
-            const value = yield client.get(formatKey(key));
-            try {
-                return JSON.parse(value);
-            }
-            catch (error) {
-                // ignore error
-            }
-            return value;
-        }));
-    },
-    set: (key, value, options) => {
-        return withCache((client) => {
-            const formatValue = typeof value === 'string' ? value : JSON.stringify(value);
-            return client.set(formatKey(key), formatValue, Object.assign({ PX: ttl }, options));
-        });
-    },
-    remove: (key) => {
-        return withCache((client) => {
-            return client.del(formatKey(key));
-        });
-    },
-    list: (key = '*') => {
-        return withCache((client) => {
-            return client.keys(formatKey(key));
-        });
-    },
-};

package/esm/config.d.ts

@@ -1,10 +0,0 @@
-export declare const logger: any;
-export declare const config: {
-    redisUrl: string;
-    dataDir: string;
-    appDir: string;
-    appUrl: string;
-    puppeteerPath: string;
-    cacheDir: string;
-    testOnInitialize: boolean;
-};

package/esm/config.js

@@ -1,11 +0,0 @@
-import createLogger from '@blocklet/logger';
-export const logger = createLogger('crawler', { level: process.env.LOG_LEVEL || 'info' });
-export const config = {
-    redisUrl: process.env.REDIS_URL,
-    dataDir: process.env.BLOCKLET_DATA_DIR,
-    appDir: process.env.BLOCKLET_APP_DIR || process.cwd(),
-    appUrl: process.env.BLOCKLET_APP_URL,
-    puppeteerPath: process.env.PUPPETEER_EXECUTABLE_PATH,
-    cacheDir: process.env.BLOCKLET_CACHE_DIR,
-    testOnInitialize: process.env.NODE_ENV === 'production',
-};

package/esm/crawler.d.ts

@@ -1,28 +0,0 @@
-import { JobState } from './db/job';
-import { SnapshotModel } from './db/snapshot';
-export declare function createCrawlQueue(): void;
-export declare function getDataDir(): Promise<{
-    htmlDir: string;
-    screenshotDir: string;
-}>;
-export declare const getPageContent: ({ url, formatPageContent, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage, }: {
-    url: string;
-    formatPageContent?: Function;
-    includeScreenshot?: boolean;
-    includeHtml?: boolean;
-    width?: number;
-    height?: number;
-    quality?: number;
-    timeout?: number;
-    fullPage?: boolean;
-}) => Promise<{
-    html: string;
-    screenshot: Uint8Array<ArrayBufferLike> | null;
-}>;
-export declare function createCrawlJob(params: JobState, callback?: (snapshot: SnapshotModel | null) => void): Promise<any>;
-export declare function getJob(condition: Partial<JobState>): Promise<any>;
-export declare function formatSnapshot(snapshot: SnapshotModel, columns?: Array<keyof SnapshotModel>): Promise<SnapshotModel>;
-/**
- * get snapshot from db or crawl queue
- */
-export declare function getSnapshot(jobId: string): Promise<SnapshotModel | null>;

package/esm/db/index.d.ts

@@ -1 +0,0 @@
-export declare function ensureDatabase(): Promise<void>;

package/esm/db/job.js

@@ -1,50 +0,0 @@
-import { DataTypes, Model } from '@sequelize/core';
-class Job extends Model {
-}
-export { Job };
-export function initJobModel(sequelize) {
-    Job.init({
-        id: {
-            type: DataTypes.STRING(40),
-            primaryKey: true,
-        },
-        queue: {
-            type: DataTypes.STRING(32),
-            allowNull: false,
-        },
-        job: {
-            type: DataTypes.JSON,
-            allowNull: false,
-        },
-        retryCount: {
-            type: DataTypes.INTEGER,
-        },
-        delay: {
-            type: DataTypes.INTEGER,
-        },
-        willRunAt: {
-            type: DataTypes.INTEGER,
-        },
-        cancelled: {
-            type: DataTypes.BOOLEAN,
-            defaultValue: false,
-        },
-        createdAt: {
-            type: DataTypes.DATE,
-            defaultValue: DataTypes.NOW,
-            index: true,
-        },
-        updatedAt: {
-            type: DataTypes.DATE,
-            defaultValue: DataTypes.NOW,
-            index: true,
-        },
-    }, {
-        sequelize,
-        indexes: [{ fields: ['queue'] }],
-        modelName: 'job',
-        tableName: 'jobs',
-        timestamps: true,
-    });
-    return Job;
-}

package/esm/db/snapshot.js

@@ -1,48 +0,0 @@
-import { DataTypes, Model } from '@sequelize/core';
-class Snapshot extends Model {
-}
-export { Snapshot };
-export function initSnapshotModel(sequelize) {
-    Snapshot.init({
-        jobId: {
-            type: DataTypes.STRING,
-            primaryKey: true,
-            allowNull: false,
-        },
-        url: {
-            type: DataTypes.STRING,
-            allowNull: false,
-            index: true,
-        },
-        status: {
-            type: DataTypes.ENUM('success', 'failed'),
-            allowNull: false,
-        },
-        html: {
-            type: DataTypes.TEXT,
-            allowNull: true,
-        },
-        screenshot: {
-            type: DataTypes.STRING,
-            allowNull: true,
-        },
-        error: {
-            type: DataTypes.STRING,
-            allowNull: true,
-        },
-        lastModified: {
-            type: DataTypes.STRING,
-            allowNull: true,
-        },
-        options: {
-            type: DataTypes.JSON,
-            allowNull: true,
-        },
-    }, {
-        sequelize,
-        modelName: 'snapshot',
-        tableName: 'snap',
-        timestamps: true,
-    });
-    return Snapshot;
-}

package/esm/index.d.ts

@@ -1,6 +0,0 @@
-import { config } from './config';
-export * from './blocklet';
-export * from './crawler';
-export * from './middleware';
-export { Snapshot } from './db/snapshot';
-export declare function initCrawler(_config: Partial<typeof config>): Promise<void>;

package/esm/middleware.d.ts

@@ -1,4 +0,0 @@
-export declare function initSEOMiddleware({ autoReturnHtml, allowCrawler, }: {
-    autoReturnHtml?: Boolean;
-    allowCrawler?: Boolean | Function;
-}): (req: any, res: any, next: Function) => Promise<any>;

package/esm/middleware.js

@@ -1,41 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { useCache } from './cache';
-import { getFullUrl, isAcceptCrawler, isBotUserAgent, isSelfCrawler } from './utils';
-export function initSEOMiddleware({ autoReturnHtml = true, allowCrawler = true, }) {
-    return (req, res, next) => __awaiter(this, void 0, void 0, function* () {
-        const isBot = isBotUserAgent(req);
-        const isSelf = isSelfCrawler(req);
-        if (!isBot || isSelf) {
-            return next();
-        }
-        const fullUrl = getFullUrl(req);
-        const canCrawl = yield isAcceptCrawler(fullUrl);
-        const allowCrawlerResult = typeof allowCrawler === 'function' ? allowCrawler(req) : allowCrawler;
-        // can not crawl, skip
-        if (!canCrawl || !allowCrawlerResult) {
-            return next();
-        }
-        const cacheData = yield useCache.get(fullUrl);
-        // add cached html to req
-        req.cachedHtml = (cacheData === null || cacheData === void 0 ? void 0 : cacheData.content) || cacheData || null;
-        // add cached lastModified to req, ISO string to GMT string
-        req.cachedLastmod = (cacheData === null || cacheData === void 0 ? void 0 : cacheData.lastModified) ? new Date(cacheData === null || cacheData === void 0 ? void 0 : cacheData.lastModified).toUTCString() : null;
-        if (req.cachedLastmod) {
-            res.setHeader('Last-Modified', req.cachedLastmod);
-        }
-        if (autoReturnHtml && req.cachedHtml) {
-            res.send(req.cachedHtml);
-            return;
-        }
-        // missing cache
-        next();
-    });
-}

package/esm/utils.d.ts

@@ -1,17 +0,0 @@
-import { Page } from '@blocklet/puppeteer';
-export declare const api: import("axios").AxiosInstance;
-export declare const sleep: (ms: number) => Promise<unknown>;
-export declare const CRAWLER_FLAG = "x-crawler";
-export declare const isSelfCrawler: (req: any) => boolean;
-export declare const getDefaultRobotsUrl: (url: string) => string;
-export declare function getRobots(url: string): Promise<import("robots-parser").Robot | null>;
-export declare const getDefaultSitemapUrl: (url: string) => string;
-export declare const isAcceptCrawler: (url: string) => Promise<boolean | undefined>;
-export declare const getSitemapList: (url: string) => Promise<import("sitemap").SitemapItem[]>;
-export declare const isBotUserAgent: (req: any) => boolean;
-export declare const getComponentInfo: () => {};
-export declare const getFullUrl: (req: any) => string;
-export declare const getRelativePath: (url: string) => string;
-export declare const formatUrl: (url: string) => string;
-export declare function md5(content: string | Uint8Array): string;
-export declare function findMaxScrollHeight(page: Page): Promise<number>;