@arcblock/crawler 1.5.1 → 1.5.3

package/lib/cjs/crawler.js

@@ -54,6 +54,7 @@ function createCrawlQueue(queue, handler) {
         options: {
             concurrency: config_1.config.concurrency,
             enableScheduledJob: true,
+            maxRetries: 3,
         },
         onJob: (job) => __awaiter(this, void 0, void 0, function* () {
             const startTime = Date.now();
@@ -205,6 +206,22 @@ const getPageContent = (_a, handler_1) => __awaiter(void 0, [_a, handler_1], voi
     let html = null;
     let screenshot = null;
     const meta = {};
+    const closePageSafely = () => __awaiter(void 0, void 0, void 0, function* () {
+        try {
+            yield page.close();
+        }
+        catch (error) {
+            if ((0, puppeteer_1.isBrowserConnectionError)(error)) {
+                try {
+                    yield (0, puppeteer_1.closeBrowser)({ trimCache: false });
+                }
+                catch (closeError) {
+                    config_1.logger.warn('Failed to close browser after page close error', { error: closeError });
+                }
+            }
+            config_1.logger.warn('Failed to close page:', { error });
+        }
+    });
     try {
         const response = yield page.goto(url, { timeout });
         if (!response) {
@@ -291,14 +308,23 @@ const getPageContent = (_a, handler_1) => __awaiter(void 0, [_a, handler_1], voi
             config_1.logger.error('Failed to get html:', err);
             throw err;
         }
+        yield closePageSafely();
     }
     catch (error) {
+        if ((0, puppeteer_1.isBrowserConnectionError)(error)) {
+            try {
+                yield (0, puppeteer_1.closeBrowser)({ trimCache: false });
+            }
+            catch (closeError) {
+                config_1.logger.warn('Failed to close browser after page error', { error: closeError });
+            }
+        }
+        else {
+            yield closePageSafely();
+        }
         config_1.logger.error('Failed to get page content:', error);
         throw error;
     }
-    finally {
-        yield page.close();
-    }
     return {
         html,
         screenshot,

package/lib/cjs/metrics.js

@@ -62,6 +62,8 @@ function collectMetrics() {
         try {
             // 收集队列大小
             const jobStats = yield store_1.Job.stats();
+            // Reset first to clear queues that no longer have jobs
+            exports.queueSize.reset();
             jobStats.queues.forEach((q) => {
                 exports.queueSize.set({ queue: q.queue }, q.count);
             });

package/lib/cjs/puppeteer.d.ts

@@ -1,16 +1,16 @@
-import puppeteer, { Browser, Page, ResourceType } from '@blocklet/puppeteer';
+import puppeteer, { Browser, ResourceType } from '@blocklet/puppeteer';
 export { puppeteer };
 export declare function ensurePuppeteerrc(): Promise<{
     cacheDirectory: string;
     temporaryDirectory: string;
 }>;
 export declare function ensureBrowser(): Promise<void>;
-export declare function connectBrowser(): Promise<Browser | null>;
 export declare function launchBrowser(): Promise<Browser>;
+export declare function isBrowserConnectionError(error: unknown): boolean;
 export declare const getBrowser: () => Promise<Browser>;
 export declare const closeBrowser: ({ trimCache }?: {
     trimCache?: boolean;
-}) => Promise<void>;
+}) => Promise<void> | undefined;
 export declare function initPage({ abortResourceTypes }?: {
     abortResourceTypes?: ResourceType[];
-}): Promise<Page>;
+}): Promise<any>;

package/lib/cjs/puppeteer.js

@@ -15,27 +15,25 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.closeBrowser = exports.getBrowser = exports.puppeteer = void 0;
 exports.ensurePuppeteerrc = ensurePuppeteerrc;
 exports.ensureBrowser = ensureBrowser;
-exports.connectBrowser = connectBrowser;
 exports.launchBrowser = launchBrowser;
+exports.isBrowserConnectionError = isBrowserConnectionError;
 exports.initPage = initPage;
 const puppeteer_1 = __importDefault(require("@blocklet/puppeteer"));
 exports.puppeteer = puppeteer_1.default;
 const fs_extra_1 = __importDefault(require("fs-extra"));
 const path_1 = __importDefault(require("path"));
-const timers_1 = require("timers");
 const config_1 = require("./config");
-const store_1 = require("./store");
 const utils_1 = require("./utils");
-const BrowserStatus = {
-    None: 'None',
-    Launching: 'Launching',
-    Ready: 'Ready',
-};
-let browserStatus = BrowserStatus.None;
-/** Chromium WebSocket endpoint that allows puppeteer browser instance to connect to the browser */
-let browserEndpoint = '';
 let browser;
-let browserActivatedTimer;
+let browserInitInFlight;
+let closingBrowser;
+const BROWSER_CONNECTION_ERROR_PATTERNS = [
+    /protocol error/i,
+    /target closed/i,
+    /browser disconnected/i,
+    /session closed/i,
+    /target crashed/i,
+];
 function ensurePuppeteerrc() {
     return __awaiter(this, void 0, void 0, function* () {
         const cacheDirectory = path_1.default.join(config_1.config.cacheDir, 'puppeteer', 'cache');
@@ -78,7 +76,7 @@ function ensureBrowser() {
         }
         // try to launch browser
         if (config_1.config.isProd) {
-            const browser = yield launchBrowser();
+            const browser = yield (0, exports.getBrowser)();
             if (!browser) {
                 throw new Error('Failed to launch browser');
             }
@@ -87,34 +85,8 @@ function ensureBrowser() {
         config_1.logger.info('Puppeteer is ready');
     });
 }
-function connectBrowser() {
-    return __awaiter(this, void 0, void 0, function* () {
-        if (!browserEndpoint) {
-            return null;
-        }
-        // retry if browser is launching
-        if (browserStatus === BrowserStatus.Launching) {
-            yield (0, utils_1.sleep)(Math.floor(Math.random() * 1000));
-            return connectBrowser();
-        }
-        try {
-            browser = yield puppeteer_1.default.connect({
-                browserWSEndpoint: browserEndpoint,
-            });
-            config_1.logger.info('Connect browser success');
-        }
-        catch (err) {
-            config_1.logger.warn('Connect browser failed, clear endpoint', err);
-            browserEndpoint = '';
-            return null;
-        }
-        return browser;
-    });
-}
 function launchBrowser() {
     return __awaiter(this, void 0, void 0, function* () {
-        browserEndpoint = '';
-        browserStatus = BrowserStatus.Launching;
         try {
             browser = yield puppeteer_1.default.launch({
                 headless: true,
@@ -146,125 +118,139 @@ function launchBrowser() {
                     '--disable-gpu-sandbox',
                 ],
             });
+            attachBrowserListeners(browser);
             config_1.logger.info('Launch browser');
         }
         catch (error) {
             config_1.logger.error('launch browser failed: ', error);
-            browserStatus = BrowserStatus.None;
-            browserEndpoint = '';
             throw error;
         }
-        // save browserWSEndpoint to cache
-        browserEndpoint = yield browser.wsEndpoint();
-        browserStatus = BrowserStatus.Ready;
         return browser;
     });
 }
-function checkBrowserActivated() {
-    clearBrowserActivatedTimer();
-    let count = 0;
-    browserActivatedTimer = (0, timers_1.setInterval)(() => __awaiter(this, void 0, void 0, function* () {
-        var _a;
-        if (browser) {
-            const pages = yield browser.pages().catch(() => []);
-            const jobCount = yield store_1.Job.count().catch(() => 0);
-            // Check if browser is inactive: only blank page AND no pending jobs
-            const isInactive = pages.length === 1 && ((_a = pages[0]) === null || _a === void 0 ? void 0 : _a.url()) === 'about:blank' && jobCount === 0;
-            if (isInactive) {
-                count++;
-                config_1.logger.debug(`Browser inactive count: ${count}/3`);
-            }
-            else {
-                count = 0;
-                if (jobCount > 0) {
-                    config_1.logger.debug(`Browser has ${jobCount} pending jobs, keeping active`);
-                }
-            }
-            if (count >= 3) {
-                config_1.logger.info('Browser inactive for 3 minutes, closing...');
-                yield (0, exports.closeBrowser)({
-                    trimCache: true,
-                });
-            }
+function resetBrowserState(reason) {
+    if (reason) {
+        config_1.logger.warn('Reset browser state', { reason });
+    }
+    browser = null;
+    browserInitInFlight = null;
+}
+function isBrowserConnectionError(error) {
+    const message = error instanceof Error ? error.message : String(error || '');
+    return BROWSER_CONNECTION_ERROR_PATTERNS.some((pattern) => pattern.test(message));
+}
+function attachBrowserListeners(target) {
+    target.on('disconnected', () => {
+        if (browser !== target) {
+            return;
         }
-    }), 1000 * 60);
+        config_1.logger.warn('Browser disconnected');
+        resetBrowserState('disconnected');
+    });
 }
-function clearBrowserActivatedTimer() {
-    if (browserActivatedTimer) {
-        (0, timers_1.clearInterval)(browserActivatedTimer);
-        browserActivatedTimer = null;
-    }
+function initBrowser() {
+    return __awaiter(this, void 0, void 0, function* () {
+        // sleep random time (0 ~ 5s),to avoid concurrent blocklet
+        yield (0, utils_1.sleep)(Math.floor(Math.random() * 1000 * 5));
+        const launchedBrowser = yield launchBrowser();
+        if (launchedBrowser) {
+            config_1.logger.debug('getBrowser.launchedBrowser');
+            browser = launchedBrowser;
+            return browser;
+        }
+        throw new Error('No browser to use, should install redis or browser');
+    });
 }
 const getBrowser = () => __awaiter(void 0, void 0, void 0, function* () {
-    if (browser)
-        return browser;
-    // sleep random time (0 ~ 5s),to avoid concurrent blocklet
-    yield (0, utils_1.sleep)(Math.floor(Math.random() * 1000 * 5));
-    // try to connect browser
-    const connectedBrowser = yield connectBrowser();
-    if (connectedBrowser) {
-        config_1.logger.debug('getBrowser.connectedBrowser');
-        browser = connectedBrowser;
-        checkBrowserActivated();
-        return browser;
+    // Wait for any ongoing browser close operation to complete
+    if (closingBrowser) {
+        yield closingBrowser;
     }
-    // try to launch browser
-    const launchedBrowser = yield launchBrowser();
-    if (launchedBrowser) {
-        config_1.logger.debug('getBrowser.launchedBrowser');
-        browser = launchedBrowser;
-        checkBrowserActivated();
-        return browser;
+    if (browser) {
+        if (browser.isConnected()) {
+            return browser;
+        }
+        config_1.logger.warn('Browser instance is disconnected, resetting');
+        resetBrowserState('disconnected');
+    }
+    if (browserInitInFlight) {
+        return browserInitInFlight;
     }
-    throw new Error('No browser to use, should install redis or browser');
+    const initPromise = initBrowser();
+    browserInitInFlight = initPromise;
+    return initPromise.finally(() => {
+        if (browserInitInFlight === initPromise) {
+            browserInitInFlight = null;
+        }
+    });
 });
 exports.getBrowser = getBrowser;
-const closeBrowser = (...args_1) => __awaiter(void 0, [...args_1], void 0, function* ({ trimCache = true } = {}) {
+const closeBrowser = ({ trimCache = true } = {}) => {
+    // Return existing close operation if already in progress
+    if (closingBrowser) {
+        return closingBrowser;
+    }
     if (!browser)
         return;
-    // close all pages
-    try {
-        const pages = yield browser.pages();
-        yield Promise.all(pages.map((page) => page.close()));
-    }
-    catch (err) {
-        config_1.logger.warn('Failed to close all pages:', err);
-    }
-    // close browser
-    try {
-        yield browser.close();
-    }
-    catch (err) {
-        config_1.logger.warn('Failed to close browser:', err);
-    }
-    // clear cache
-    try {
-        if (trimCache) {
-            yield puppeteer_1.default.trimCache();
-            config_1.logger.debug('Trim cache success');
+    const target = browser;
+    browser = null;
+    browserInitInFlight = null;
+    const doClose = () => __awaiter(void 0, void 0, void 0, function* () {
+        // close all pages
+        try {
+            const pages = yield target.pages();
+            yield Promise.all(pages.map((page) => page.close().catch(() => { })));
+        }
+        catch (err) {
+            config_1.logger.warn('Failed to close all pages:', err);
         }
-        // try to clear temporary directory
-        // if (puppeteerConfig) {
-        //   await fs.emptyDir(puppeteerConfig.temporaryDirectory);
-        // }
-        if (global.gc) {
-            global.gc();
+        // close browser
+        try {
+            yield target.close();
         }
-    }
-    catch (err) {
-        config_1.logger.warn('Failed to clear browser cache:', err);
-    }
-    browser = null;
-    clearBrowserActivatedTimer();
-    browserEndpoint = '';
-    browserStatus = BrowserStatus.None;
-    config_1.logger.info('Close browser success');
-});
+        catch (err) {
+            config_1.logger.warn('Failed to close browser:', err);
+        }
+        // clear cache
+        try {
+            if (trimCache) {
+                yield puppeteer_1.default.trimCache();
+                config_1.logger.debug('Trim cache success');
+            }
+            if (global.gc) {
+                global.gc();
+            }
+        }
+        catch (err) {
+            config_1.logger.warn('Failed to clear browser cache:', err);
+        }
+        config_1.logger.info('Close browser success');
+    });
+    closingBrowser = doClose().finally(() => {
+        closingBrowser = null;
+    });
+    return closingBrowser;
+};
 exports.closeBrowser = closeBrowser;
 function initPage() {
     return __awaiter(this, arguments, void 0, function* ({ abortResourceTypes = [] } = {}) {
-        const browser = yield (0, exports.getBrowser)();
-        const page = yield browser.newPage();
+        const currentBrowser = yield (0, exports.getBrowser)();
+        let page;
+        try {
+            page = yield currentBrowser.newPage();
+        }
+        catch (error) {
+            // If newPage fails due to connection error, close browser and retry once
+            if (isBrowserConnectionError(error)) {
+                config_1.logger.warn('Failed to create new page due to connection error, restarting browser');
+                yield (0, exports.closeBrowser)({ trimCache: false });
+                const newBrowser = yield (0, exports.getBrowser)();
+                page = yield newBrowser.newPage();
+            }
+            else {
+                throw error;
+            }
+        }
         yield page.setViewport({ width: 1440, height: 900 });
         // page setting
         // add custom headers

package/lib/cjs/services/snapshot.d.ts

@@ -4,7 +4,7 @@ export declare function convertJobToSnapshot({ job, snapshot }: {
     job: JobState;
     snapshot?: Partial<SnapshotModel>;
 }): SnapshotModel;
-export declare function formatSnapshot(snapshot: SnapshotModel, columns?: Array<keyof SnapshotModel>): Promise<SnapshotModel>;
+export declare function formatSnapshot(snapshot: SnapshotModel, columns?: Array<keyof SnapshotModel>): Promise<SnapshotModel | null>;
 /**
  * get snapshot from db or crawl queue
  */

package/lib/cjs/services/snapshot.js

@@ -54,6 +54,12 @@ function formatSnapshot(snapshot, columns) {
                     dataDir: config_1.config.dataDir,
                     snapshot,
                 });
+                // If the file is missing, delete the invalid snapshot record
+                if ((err === null || err === void 0 ? void 0 : err.code) === 'ENOENT') {
+                    config_1.logger.warn('HTML file missing, deleting invalid snapshot record', { jobId: snapshot.jobId });
+                    yield store_1.Snapshot.destroy({ where: { jobId: snapshot.jobId } });
+                    return null;
+                }
                 data.html = '';
             }
         }
@@ -114,14 +120,26 @@ function deleteSnapshots(where_1) {
         });
         const jobIds = yield Promise.all(snapshots.map((snapshot) => __awaiter(this, void 0, void 0, function* () {
             try {
+                // Check reference count before deleting files
+                // Only delete file if no other snapshots reference it
+                const deleteFilePromises = [];
+                if (snapshot.html) {
+                    const htmlRefCount = yield store_1.Snapshot.count({ where: { html: snapshot.html } });
+                    if (htmlRefCount <= 1) {
+                        deleteFilePromises.push(promises_1.default.unlink(node_path_1.default.join(config_1.config.dataDir, snapshot.html)).catch(() => { }));
+                    }
+                }
+                if (snapshot.screenshot) {
+                    const screenshotRefCount = yield store_1.Snapshot.count({ where: { screenshot: snapshot.screenshot } });
+                    if (screenshotRefCount <= 1) {
+                        deleteFilePromises.push(promises_1.default.unlink(node_path_1.default.join(config_1.config.dataDir, snapshot.screenshot)).catch(() => { }));
+                    }
+                }
                 try {
-                    yield Promise.all([
-                        snapshot.html && promises_1.default.unlink(node_path_1.default.join(config_1.config.dataDir, snapshot.html)),
-                        snapshot.screenshot && promises_1.default.unlink(node_path_1.default.join(config_1.config.dataDir, snapshot.screenshot)),
-                    ]);
+                    yield Promise.all(deleteFilePromises);
                 }
                 catch (err) {
-                    config_1.logger.error('Failed to delete snapshot', { err, snapshot, dataDir: config_1.config.dataDir });
+                    config_1.logger.error('Failed to delete snapshot files', { err, snapshot, dataDir: config_1.config.dataDir });
                 }
                 yield snapshot.destroy({ transaction: txn });
                 return snapshot.jobId;

package/lib/esm/crawler.js

@@ -15,7 +15,7 @@ import fs from 'fs-extra';
 import path from 'path';
 import { config, logger } from './config';
 import { jobDurationSeconds, jobTotalLatencySeconds, jobsEnqueuedTotal, jobsTotal } from './metrics';
-import { initPage } from './puppeteer';
+import { closeBrowser, initPage, isBrowserConnectionError } from './puppeteer';
 import { createCarbonImage } from './services/carbon';
 import { convertJobToSnapshot, deleteSnapshots, formatSnapshot } from './services/snapshot';
 import { Job, Snapshot, sequelize } from './store';
@@ -42,6 +42,7 @@ export function createCrawlQueue(queue, handler) {
         options: {
             concurrency: config.concurrency,
             enableScheduledJob: true,
+            maxRetries: 3,
         },
         onJob: (job) => __awaiter(this, void 0, void 0, function* () {
             const startTime = Date.now();
@@ -193,6 +194,22 @@ export const getPageContent = (_a, handler_1) => __awaiter(void 0, [_a, handler_
     let html = null;
     let screenshot = null;
     const meta = {};
+    const closePageSafely = () => __awaiter(void 0, void 0, void 0, function* () {
+        try {
+            yield page.close();
+        }
+        catch (error) {
+            if (isBrowserConnectionError(error)) {
+                try {
+                    yield closeBrowser({ trimCache: false });
+                }
+                catch (closeError) {
+                    logger.warn('Failed to close browser after page close error', { error: closeError });
+                }
+            }
+            logger.warn('Failed to close page:', { error });
+        }
+    });
     try {
         const response = yield page.goto(url, { timeout });
         if (!response) {
@@ -279,14 +296,23 @@ export const getPageContent = (_a, handler_1) => __awaiter(void 0, [_a, handler_
             logger.error('Failed to get html:', err);
             throw err;
         }
+        yield closePageSafely();
     }
     catch (error) {
+        if (isBrowserConnectionError(error)) {
+            try {
+                yield closeBrowser({ trimCache: false });
+            }
+            catch (closeError) {
+                logger.warn('Failed to close browser after page error', { error: closeError });
+            }
+        }
+        else {
+            yield closePageSafely();
+        }
         logger.error('Failed to get page content:', error);
         throw error;
     }
-    finally {
-        yield page.close();
-    }
     return {
         html,
         screenshot,

package/lib/esm/metrics.js

@@ -56,6 +56,8 @@ export function collectMetrics() {
         try {
             // 收集队列大小
             const jobStats = yield Job.stats();
+            // Reset first to clear queues that no longer have jobs
+            queueSize.reset();
             jobStats.queues.forEach((q) => {
                 queueSize.set({ queue: q.queue }, q.count);
             });

package/lib/esm/puppeteer.d.ts

@@ -1,16 +1,16 @@
-import puppeteer, { Browser, Page, ResourceType } from '@blocklet/puppeteer';
+import puppeteer, { Browser, ResourceType } from '@blocklet/puppeteer';
 export { puppeteer };
 export declare function ensurePuppeteerrc(): Promise<{
     cacheDirectory: string;
     temporaryDirectory: string;
 }>;
 export declare function ensureBrowser(): Promise<void>;
-export declare function connectBrowser(): Promise<Browser | null>;
 export declare function launchBrowser(): Promise<Browser>;
+export declare function isBrowserConnectionError(error: unknown): boolean;
 export declare const getBrowser: () => Promise<Browser>;
 export declare const closeBrowser: ({ trimCache }?: {
     trimCache?: boolean;
-}) => Promise<void>;
+}) => Promise<void> | undefined;
 export declare function initPage({ abortResourceTypes }?: {
     abortResourceTypes?: ResourceType[];
-}): Promise<Page>;
+}): Promise<any>;

package/lib/esm/puppeteer.js

@@ -10,20 +10,18 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 import puppeteer from '@blocklet/puppeteer';
 import fs from 'fs-extra';
 import path from 'path';
-import { clearInterval, setInterval } from 'timers';
 import { config, logger } from './config';
-import { Job } from './store';
 import { CRAWLER_FLAG, sleep } from './utils';
-const BrowserStatus = {
-    None: 'None',
-    Launching: 'Launching',
-    Ready: 'Ready',
-};
-let browserStatus = BrowserStatus.None;
-/** Chromium WebSocket endpoint that allows puppeteer browser instance to connect to the browser */
-let browserEndpoint = '';
 let browser;
-let browserActivatedTimer;
+let browserInitInFlight;
+let closingBrowser;
+const BROWSER_CONNECTION_ERROR_PATTERNS = [
+    /protocol error/i,
+    /target closed/i,
+    /browser disconnected/i,
+    /session closed/i,
+    /target crashed/i,
+];
 export { puppeteer };
 export function ensurePuppeteerrc() {
     return __awaiter(this, void 0, void 0, function* () {
@@ -67,7 +65,7 @@ export function ensureBrowser() {
         }
         // try to launch browser
         if (config.isProd) {
-            const browser = yield launchBrowser();
+            const browser = yield getBrowser();
             if (!browser) {
                 throw new Error('Failed to launch browser');
             }
@@ -76,34 +74,8 @@ export function ensureBrowser() {
         logger.info('Puppeteer is ready');
     });
 }
-export function connectBrowser() {
-    return __awaiter(this, void 0, void 0, function* () {
-        if (!browserEndpoint) {
-            return null;
-        }
-        // retry if browser is launching
-        if (browserStatus === BrowserStatus.Launching) {
-            yield sleep(Math.floor(Math.random() * 1000));
-            return connectBrowser();
-        }
-        try {
-            browser = yield puppeteer.connect({
-                browserWSEndpoint: browserEndpoint,
-            });
-            logger.info('Connect browser success');
-        }
-        catch (err) {
-            logger.warn('Connect browser failed, clear endpoint', err);
-            browserEndpoint = '';
-            return null;
-        }
-        return browser;
-    });
-}
 export function launchBrowser() {
     return __awaiter(this, void 0, void 0, function* () {
-        browserEndpoint = '';
-        browserStatus = BrowserStatus.Launching;
         try {
             browser = yield puppeteer.launch({
                 headless: true,
@@ -135,123 +107,137 @@ export function launchBrowser() {
                     '--disable-gpu-sandbox',
                 ],
             });
+            attachBrowserListeners(browser);
             logger.info('Launch browser');
         }
         catch (error) {
             logger.error('launch browser failed: ', error);
-            browserStatus = BrowserStatus.None;
-            browserEndpoint = '';
             throw error;
         }
-        // save browserWSEndpoint to cache
-        browserEndpoint = yield browser.wsEndpoint();
-        browserStatus = BrowserStatus.Ready;
         return browser;
     });
 }
-function checkBrowserActivated() {
-    clearBrowserActivatedTimer();
-    let count = 0;
-    browserActivatedTimer = setInterval(() => __awaiter(this, void 0, void 0, function* () {
-        var _a;
-        if (browser) {
-            const pages = yield browser.pages().catch(() => []);
-            const jobCount = yield Job.count().catch(() => 0);
-            // Check if browser is inactive: only blank page AND no pending jobs
-            const isInactive = pages.length === 1 && ((_a = pages[0]) === null || _a === void 0 ? void 0 : _a.url()) === 'about:blank' && jobCount === 0;
-            if (isInactive) {
-                count++;
-                logger.debug(`Browser inactive count: ${count}/3`);
-            }
-            else {
-                count = 0;
-                if (jobCount > 0) {
-                    logger.debug(`Browser has ${jobCount} pending jobs, keeping active`);
-                }
-            }
-            if (count >= 3) {
-                logger.info('Browser inactive for 3 minutes, closing...');
-                yield closeBrowser({
-                    trimCache: true,
-                });
-            }
+function resetBrowserState(reason) {
+    if (reason) {
+        logger.warn('Reset browser state', { reason });
+    }
+    browser = null;
+    browserInitInFlight = null;
+}
+export function isBrowserConnectionError(error) {
+    const message = error instanceof Error ? error.message : String(error || '');
+    return BROWSER_CONNECTION_ERROR_PATTERNS.some((pattern) => pattern.test(message));
+}
+function attachBrowserListeners(target) {
+    target.on('disconnected', () => {
+        if (browser !== target) {
+            return;
         }
-    }), 1000 * 60);
+        logger.warn('Browser disconnected');
+        resetBrowserState('disconnected');
+    });
 }
-function clearBrowserActivatedTimer() {
-    if (browserActivatedTimer) {
-        clearInterval(browserActivatedTimer);
-        browserActivatedTimer = null;
-    }
+function initBrowser() {
+    return __awaiter(this, void 0, void 0, function* () {
+        // sleep random time (0 ~ 5s),to avoid concurrent blocklet
+        yield sleep(Math.floor(Math.random() * 1000 * 5));
+        const launchedBrowser = yield launchBrowser();
+        if (launchedBrowser) {
+            logger.debug('getBrowser.launchedBrowser');
+            browser = launchedBrowser;
+            return browser;
+        }
+        throw new Error('No browser to use, should install redis or browser');
+    });
 }
 export const getBrowser = () => __awaiter(void 0, void 0, void 0, function* () {
-    if (browser)
-        return browser;
-    // sleep random time (0 ~ 5s),to avoid concurrent blocklet
-    yield sleep(Math.floor(Math.random() * 1000 * 5));
-    // try to connect browser
-    const connectedBrowser = yield connectBrowser();
-    if (connectedBrowser) {
-        logger.debug('getBrowser.connectedBrowser');
-        browser = connectedBrowser;
-        checkBrowserActivated();
-        return browser;
+    // Wait for any ongoing browser close operation to complete
+    if (closingBrowser) {
+        yield closingBrowser;
     }
-    // try to launch browser
-    const launchedBrowser = yield launchBrowser();
-    if (launchedBrowser) {
-        logger.debug('getBrowser.launchedBrowser');
-        browser = launchedBrowser;
-        checkBrowserActivated();
-        return browser;
+    if (browser) {
+        if (browser.isConnected()) {
+            return browser;
+        }
+        logger.warn('Browser instance is disconnected, resetting');
+        resetBrowserState('disconnected');
+    }
+    if (browserInitInFlight) {
+        return browserInitInFlight;
     }
-    throw new Error('No browser to use, should install redis or browser');
+    const initPromise = initBrowser();
+    browserInitInFlight = initPromise;
+    return initPromise.finally(() => {
+        if (browserInitInFlight === initPromise) {
+            browserInitInFlight = null;
+        }
+    });
 });
-export const closeBrowser = (...args_1) => __awaiter(void 0, [...args_1], void 0, function* ({ trimCache = true } = {}) {
+export const closeBrowser = ({ trimCache = true } = {}) => {
+    // Return existing close operation if already in progress
+    if (closingBrowser) {
+        return closingBrowser;
+    }
     if (!browser)
         return;
-    // close all pages
-    try {
-        const pages = yield browser.pages();
-        yield Promise.all(pages.map((page) => page.close()));
-    }
-    catch (err) {
-        logger.warn('Failed to close all pages:', err);
-    }
-    // close browser
-    try {
-        yield browser.close();
-    }
-    catch (err) {
-        logger.warn('Failed to close browser:', err);
-    }
-    // clear cache
-    try {
-        if (trimCache) {
-            yield puppeteer.trimCache();
-            logger.debug('Trim cache success');
+    const target = browser;
+    browser = null;
+    browserInitInFlight = null;
+    const doClose = () => __awaiter(void 0, void 0, void 0, function* () {
+        // close all pages
+        try {
+            const pages = yield target.pages();
+            yield Promise.all(pages.map((page) => page.close().catch(() => { })));
+        }
+        catch (err) {
+            logger.warn('Failed to close all pages:', err);
         }
-        // try to clear temporary directory
-        // if (puppeteerConfig) {
-        //   await fs.emptyDir(puppeteerConfig.temporaryDirectory);
-        // }
-        if (global.gc) {
-            global.gc();
+        // close browser
+        try {
+            yield target.close();
         }
-    }
-    catch (err) {
-        logger.warn('Failed to clear browser cache:', err);
-    }
-    browser = null;
-    clearBrowserActivatedTimer();
-    browserEndpoint = '';
-    browserStatus = BrowserStatus.None;
-    logger.info('Close browser success');
-});
+        catch (err) {
+            logger.warn('Failed to close browser:', err);
+        }
+        // clear cache
+        try {
+            if (trimCache) {
+                yield puppeteer.trimCache();
+                logger.debug('Trim cache success');
+            }
+            if (global.gc) {
+                global.gc();
+            }
+        }
+        catch (err) {
+            logger.warn('Failed to clear browser cache:', err);
+        }
+        logger.info('Close browser success');
+    });
+    closingBrowser = doClose().finally(() => {
+        closingBrowser = null;
+    });
+    return closingBrowser;
+};
 export function initPage() {
     return __awaiter(this, arguments, void 0, function* ({ abortResourceTypes = [] } = {}) {
-        const browser = yield getBrowser();
-        const page = yield browser.newPage();
+        const currentBrowser = yield getBrowser();
+        let page;
+        try {
+            page = yield currentBrowser.newPage();
+        }
+        catch (error) {
+            // If newPage fails due to connection error, close browser and retry once
+            if (isBrowserConnectionError(error)) {
+                logger.warn('Failed to create new page due to connection error, restarting browser');
+                yield closeBrowser({ trimCache: false });
+                const newBrowser = yield getBrowser();
+                page = yield newBrowser.newPage();
+            }
+            else {
+                throw error;
+            }
+        }
         yield page.setViewport({ width: 1440, height: 900 });
         // page setting
         // add custom headers

package/lib/esm/services/snapshot.d.ts

@@ -4,7 +4,7 @@ export declare function convertJobToSnapshot({ job, snapshot }: {
     job: JobState;
     snapshot?: Partial<SnapshotModel>;
 }): SnapshotModel;
-export declare function formatSnapshot(snapshot: SnapshotModel, columns?: Array<keyof SnapshotModel>): Promise<SnapshotModel>;
+export declare function formatSnapshot(snapshot: SnapshotModel, columns?: Array<keyof SnapshotModel>): Promise<SnapshotModel | null>;
 /**
  * get snapshot from db or crawl queue
  */

package/lib/esm/services/snapshot.js

@@ -44,6 +44,12 @@ export function formatSnapshot(snapshot, columns) {
                     dataDir: config.dataDir,
                     snapshot,
                 });
+                // If the file is missing, delete the invalid snapshot record
+                if ((err === null || err === void 0 ? void 0 : err.code) === 'ENOENT') {
+                    logger.warn('HTML file missing, deleting invalid snapshot record', { jobId: snapshot.jobId });
+                    yield Snapshot.destroy({ where: { jobId: snapshot.jobId } });
+                    return null;
+                }
                 data.html = '';
             }
         }
@@ -104,14 +110,26 @@ export function deleteSnapshots(where_1) {
         });
         const jobIds = yield Promise.all(snapshots.map((snapshot) => __awaiter(this, void 0, void 0, function* () {
             try {
+                // Check reference count before deleting files
+                // Only delete file if no other snapshots reference it
+                const deleteFilePromises = [];
+                if (snapshot.html) {
+                    const htmlRefCount = yield Snapshot.count({ where: { html: snapshot.html } });
+                    if (htmlRefCount <= 1) {
+                        deleteFilePromises.push(fs.unlink(path.join(config.dataDir, snapshot.html)).catch(() => { }));
+                    }
+                }
+                if (snapshot.screenshot) {
+                    const screenshotRefCount = yield Snapshot.count({ where: { screenshot: snapshot.screenshot } });
+                    if (screenshotRefCount <= 1) {
+                        deleteFilePromises.push(fs.unlink(path.join(config.dataDir, snapshot.screenshot)).catch(() => { }));
+                    }
+                }
                 try {
-                    yield Promise.all([
-                        snapshot.html && fs.unlink(path.join(config.dataDir, snapshot.html)),
-                        snapshot.screenshot && fs.unlink(path.join(config.dataDir, snapshot.screenshot)),
-                    ]);
+                    yield Promise.all(deleteFilePromises);
                 }
                 catch (err) {
-                    logger.error('Failed to delete snapshot', { err, snapshot, dataDir: config.dataDir });
+                    logger.error('Failed to delete snapshot files', { err, snapshot, dataDir: config.dataDir });
                 }
                 yield snapshot.destroy({ transaction: txn });
                 return snapshot.jobId;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arcblock/crawler",
-  "version": "1.5.1",
+  "version": "1.5.3",
   "main": "lib/cjs/index.js",
   "module": "lib/esm/index.js",
   "types": "lib/cjs/index.d.ts",
@@ -45,12 +45,12 @@
     ]
   },
   "dependencies": {
-    "@abtnode/cron": "^1.17.5",
-    "@abtnode/models": "^1.17.5",
-    "@abtnode/queue": "^1.17.5",
-    "@blocklet/logger": "^1.17.5",
+    "@abtnode/cron": "^1.17.7",
+    "@abtnode/models": "^1.17.7",
+    "@abtnode/queue": "^1.17.7",
+    "@blocklet/logger": "^1.17.7",
     "@blocklet/puppeteer": "^22.11.3",
-    "@blocklet/sdk": "^1.17.5",
+    "@blocklet/sdk": "^1.17.7",
     "@sequelize/core": "7.0.0-alpha.46",
     "@sequelize/sqlite3": "7.0.0-alpha.46",
     "axios": "^1.7.9",