@arcblock/crawler 1.0.6 → 1.1.1

package/esm/db/snapshot.js

@@ -1,48 +0,0 @@
-import { DataTypes, Model } from '@sequelize/core';
-class Snapshot extends Model {
-}
-export { Snapshot };
-export function initSnapshotModel(sequelize) {
-    Snapshot.init({
-        jobId: {
-            type: DataTypes.STRING,
-            primaryKey: true,
-            allowNull: false,
-        },
-        url: {
-            type: DataTypes.STRING,
-            allowNull: false,
-            index: true,
-        },
-        status: {
-            type: DataTypes.ENUM('success', 'failed'),
-            allowNull: false,
-        },
-        html: {
-            type: DataTypes.TEXT,
-            allowNull: true,
-        },
-        screenshot: {
-            type: DataTypes.STRING,
-            allowNull: true,
-        },
-        error: {
-            type: DataTypes.STRING,
-            allowNull: true,
-        },
-        lastModified: {
-            type: DataTypes.STRING,
-            allowNull: true,
-        },
-        options: {
-            type: DataTypes.JSON,
-            allowNull: true,
-        },
-    }, {
-        sequelize,
-        modelName: 'snapshot',
-        tableName: 'snap',
-        timestamps: true,
-    });
-    return Snapshot;
-}

package/esm/index.d.ts

@@ -1,6 +0,0 @@
-import { config } from './config';
-export * from './blocklet';
-export * from './crawler';
-export * from './middleware';
-export { Snapshot } from './db/snapshot';
-export declare function initCrawler(_config: Partial<typeof config>): Promise<void>;

package/esm/middleware.d.ts

@@ -1,4 +0,0 @@
-export declare function initSEOMiddleware({ autoReturnHtml, allowCrawler, }: {
-    autoReturnHtml?: Boolean;
-    allowCrawler?: Boolean | Function;
-}): (req: any, res: any, next: Function) => Promise<any>;

package/esm/middleware.js

@@ -1,41 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { useCache } from './cache';
-import { getFullUrl, isAcceptCrawler, isBotUserAgent, isSelfCrawler } from './utils';
-export function initSEOMiddleware({ autoReturnHtml = true, allowCrawler = true, }) {
-    return (req, res, next) => __awaiter(this, void 0, void 0, function* () {
-        const isBot = isBotUserAgent(req);
-        const isSelf = isSelfCrawler(req);
-        if (!isBot || isSelf) {
-            return next();
-        }
-        const fullUrl = getFullUrl(req);
-        const canCrawl = yield isAcceptCrawler(fullUrl);
-        const allowCrawlerResult = typeof allowCrawler === 'function' ? allowCrawler(req) : allowCrawler;
-        // can not crawl, skip
-        if (!canCrawl || !allowCrawlerResult) {
-            return next();
-        }
-        const cacheData = yield useCache.get(fullUrl);
-        // add cached html to req
-        req.cachedHtml = (cacheData === null || cacheData === void 0 ? void 0 : cacheData.content) || cacheData || null;
-        // add cached lastModified to req, ISO string to GMT string
-        req.cachedLastmod = (cacheData === null || cacheData === void 0 ? void 0 : cacheData.lastModified) ? new Date(cacheData === null || cacheData === void 0 ? void 0 : cacheData.lastModified).toUTCString() : null;
-        if (req.cachedLastmod) {
-            res.setHeader('Last-Modified', req.cachedLastmod);
-        }
-        if (autoReturnHtml && req.cachedHtml) {
-            res.send(req.cachedHtml);
-            return;
-        }
-        // missing cache
-        next();
-    });
-}

package/esm/utils.d.ts

@@ -1,17 +0,0 @@
-import { Page } from '@blocklet/puppeteer';
-export declare const api: import("axios").AxiosInstance;
-export declare const sleep: (ms: number) => Promise<unknown>;
-export declare const CRAWLER_FLAG = "x-crawler";
-export declare const isSelfCrawler: (req: any) => boolean;
-export declare const getDefaultRobotsUrl: (url: string) => string;
-export declare function getRobots(url: string): Promise<import("robots-parser").Robot | null>;
-export declare const getDefaultSitemapUrl: (url: string) => string;
-export declare const isAcceptCrawler: (url: string) => Promise<boolean | undefined>;
-export declare const getSitemapList: (url: string) => Promise<import("sitemap").SitemapItem[]>;
-export declare const isBotUserAgent: (req: any) => boolean;
-export declare const getComponentInfo: () => {};
-export declare const getFullUrl: (req: any) => string;
-export declare const getRelativePath: (url: string) => string;
-export declare const formatUrl: (url: string) => string;
-export declare function md5(content: string | Uint8Array): string;
-export declare function findMaxScrollHeight(page: Page): Promise<number>;