@arcblock/crawler 1.1.4 → 1.1.5

package/lib/cjs/crawler.d.ts

@@ -5,7 +5,7 @@ export declare function getDataDir(): Promise<{
     htmlDir: string;
     screenshotDir: string;
 }>;
-export declare const getPageContent: ({ url, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage, headers, }: JobState) => Promise<{
+export declare const getPageContent: ({ url, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage, headers, cookies, localStorage, }: JobState) => Promise<{
     html: string | null;
     screenshot: Uint8Array<ArrayBufferLike> | null;
     meta: {

package/lib/cjs/crawler.js

@@ -59,7 +59,12 @@ function createCrawlQueue() {
             // }
             try {
                 // get page content later
-                const result = yield (0, exports.getPageContent)(job);
+                const result = yield (0, exports.getPageContent)(Object.assign({ localStorage: {
+                        // for blocklet theme
+                        blocklet_theme_prefer: 'light',
+                        // for blocklet domain warning
+                        'domain-warning-skip': Date.now().toString(),
+                    } }, job));
                 if (!result || (!result.html && !result.screenshot)) {
                     config_1.logger.error(`failed to crawl ${job.url}, empty content`, job);
                     const snapshot = (0, snapshot_1.convertJobToSnapshot)({
@@ -137,7 +142,7 @@ function saveSnapshotToLocal(_a) {
         };
     });
 }
-const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, timeout = 90 * 1000, fullPage = false, headers, }) {
+const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, timeout = 90 * 1000, fullPage = false, headers, cookies = [], localStorage, }) {
     const page = yield (0, puppeteer_1.initPage)();
     if (width && height) {
         yield page.setViewport({ width, height, deviceScaleFactor: 2 });
@@ -145,6 +150,16 @@ const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url,
     if (headers) {
         yield page.setExtraHTTPHeaders(headers);
     }
+    if (cookies === null || cookies === void 0 ? void 0 : cookies.length) {
+        yield page.setCookie(...cookies);
+    }
+    if (localStorage) {
+        yield page.evaluateOnNewDocument((items) => {
+            Object.entries(items).forEach(([key, value]) => {
+                window.localStorage.setItem(key, value);
+            });
+        }, localStorage);
+    }
     let html = null;
     let screenshot = null;
     const meta = {};

package/lib/cjs/store/job.d.ts

@@ -1,3 +1,4 @@
+import { CookieParam } from '@blocklet/puppeteer';
 import { Model, Sequelize } from '@sequelize/core';
 export interface JobState {
     id?: string;
@@ -12,6 +13,8 @@ export interface JobState {
     fullPage?: boolean;
     lastModified?: string;
     headers?: Record<string, string>;
+    cookies?: CookieParam[];
+    localStorage?: Record<string, string>;
 }
 export interface JobModel {
     id: string;

package/lib/cjs/utils.js

@@ -134,6 +134,11 @@ const staticFileExtensions = [
     'xls',
     'xml',
     'zip',
+    'ts',
+    'json',
+    'md',
+    'yml',
+    'yaml',
 ];
 const sleep = (ms) => {
     return new Promise((resolve) => {

package/lib/esm/crawler.d.ts

@@ -5,7 +5,7 @@ export declare function getDataDir(): Promise<{
     htmlDir: string;
     screenshotDir: string;
 }>;
-export declare const getPageContent: ({ url, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage, headers, }: JobState) => Promise<{
+export declare const getPageContent: ({ url, includeScreenshot, includeHtml, width, height, quality, timeout, fullPage, headers, cookies, localStorage, }: JobState) => Promise<{
     html: string | null;
     screenshot: Uint8Array<ArrayBufferLike> | null;
     meta: {

package/lib/esm/crawler.js

@@ -50,7 +50,12 @@ export function createCrawlQueue() {
             // }
             try {
                 // get page content later
-                const result = yield getPageContent(job);
+                const result = yield getPageContent(Object.assign({ localStorage: {
+                        // for blocklet theme
+                        blocklet_theme_prefer: 'light',
+                        // for blocklet domain warning
+                        'domain-warning-skip': Date.now().toString(),
+                    } }, job));
                 if (!result || (!result.html && !result.screenshot)) {
                     logger.error(`failed to crawl ${job.url}, empty content`, job);
                     const snapshot = convertJobToSnapshot({
@@ -128,7 +133,7 @@ function saveSnapshotToLocal(_a) {
         };
     });
 }
-export const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, timeout = 90 * 1000, fullPage = false, headers, }) {
+export const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function* ({ url, includeScreenshot = true, includeHtml = true, width = 1440, height = 900, quality = 80, timeout = 90 * 1000, fullPage = false, headers, cookies = [], localStorage, }) {
     const page = yield initPage();
     if (width && height) {
         yield page.setViewport({ width, height, deviceScaleFactor: 2 });
@@ -136,6 +141,16 @@ export const getPageContent = (_a) => __awaiter(void 0, [_a], void 0, function*
     if (headers) {
         yield page.setExtraHTTPHeaders(headers);
     }
+    if (cookies === null || cookies === void 0 ? void 0 : cookies.length) {
+        yield page.setCookie(...cookies);
+    }
+    if (localStorage) {
+        yield page.evaluateOnNewDocument((items) => {
+            Object.entries(items).forEach(([key, value]) => {
+                window.localStorage.setItem(key, value);
+            });
+        }, localStorage);
+    }
     let html = null;
     let screenshot = null;
     const meta = {};

package/lib/esm/store/job.d.ts

@@ -1,3 +1,4 @@
+import { CookieParam } from '@blocklet/puppeteer';
 import { Model, Sequelize } from '@sequelize/core';
 export interface JobState {
     id?: string;
@@ -12,6 +13,8 @@ export interface JobState {
     fullPage?: boolean;
     lastModified?: string;
     headers?: Record<string, string>;
+    cookies?: CookieParam[];
+    localStorage?: Record<string, string>;
 }
 export interface JobModel {
     id: string;

package/lib/esm/utils.js

@@ -123,6 +123,11 @@ const staticFileExtensions = [
     'xls',
     'xml',
     'zip',
+    'ts',
+    'json',
+    'md',
+    'yml',
+    'yaml',
 ];
 export const sleep = (ms) => {
     return new Promise((resolve) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arcblock/crawler",
-  "version": "1.1.4",
+  "version": "1.1.5",
   "main": "lib/cjs/index.js",
   "module": "lib/esm/index.js",
   "types": "lib/cjs/index.d.ts",
@@ -45,33 +45,32 @@
     ]
   },
   "dependencies": {
-    "@abtnode/cron": "^1.16.43",
-    "@abtnode/models": "^1.16.43",
-    "@abtnode/queue": "^1.16.43",
-    "@blocklet/logger": "^1.16.43",
+    "@abtnode/cron": "^1.16.44",
+    "@abtnode/models": "^1.16.44",
+    "@abtnode/queue": "^1.16.44",
+    "@blocklet/logger": "^1.16.44",
     "@blocklet/puppeteer": "^22.11.3",
-    "@blocklet/sdk": "^1.16.43",
+    "@blocklet/sdk": "^1.16.44",
     "@sequelize/core": "7.0.0-alpha.46",
     "@sequelize/sqlite3": "7.0.0-alpha.46",
     "axios": "^1.7.9",
     "fs-extra": "^11.2.0",
     "lodash": "^4.17.21",
     "lru-cache": "^10.4.3",
+    "p-map": "^7.0.3",
     "robots-parser": "^3.0.1",
     "sitemap": "^7.1.2",
     "sqlite3": "^5.1.7",
-    "ufo": "^1.5.4",
-    "p-map": "^7.0.3"
+    "ufo": "^1.5.4"
   },
   "devDependencies": {
-    "@blocklet/js-sdk": "^1.16.39",
     "@types/dotenv-flow": "^3.3.3",
     "@types/express": "^4.17.21",
     "@types/fs-extra": "^11.0.4",
     "@types/lodash": "^4.17.16",
     "@types/node": "^20.17.19",
-    "express": "^4.21.2",
     "bumpp": "^9.11.1",
+    "express": "^4.21.2",
     "nodemon": "^3.1.9",
     "npm-run-all": "^4.1.5",
     "puppeteer": "^24.8.2",