@d-zero/puppeteer-dealer 0.3.0 → 0.4.0

package/dist/create-child-process.d.ts

@@ -0,0 +1,28 @@
+import type { Logger } from './types.js';
+import type { Page } from 'puppeteer';
+export type ChildProcessMethods<R> = {
+    eachPage: (params: EachPageParams, logger: Logger) => Promise<R>;
+};
+type EachPageParams = {
+    readonly page: Page;
+    readonly id: string;
+    readonly url: string;
+    readonly index: number;
+};
+export type ChildProcessCommonParams = {
+    readonly id: string;
+    readonly url: string;
+    readonly logger: Logger;
+};
+export type ChildProcessHandler<P, R> = (params: P) => Promise<ChildProcessMethods<R>> | ChildProcessMethods<R>;
+export type ChildProcessCommands<P, R> = {
+    init: () => Promise<P>;
+    each: (id: string, url: string, index: number) => Promise<R>;
+    log: Logger;
+};
+/**
+ *
+ * @param handler
+ */
+export declare function createChildProcess<P, R = void>(handler: ChildProcessHandler<P, R>): void;
+export {};

package/dist/create-child-process.js

@@ -0,0 +1,59 @@
+import { ProcTalk } from '@d-zero/proc-talk';
+import puppeteer from 'puppeteer';
+import { log } from './debug.js';
+const childLog = log.extend(`child:${process.pid}`);
+/**
+ *
+ * @param handler
+ */
+export function createChildProcess(handler) {
+    new ProcTalk({
+        type: 'child',
+        title: '@d-zero/puppeteer-dealer',
+        async process(options) {
+            const config = {
+                locale: 'ja-JP',
+                ...options,
+            };
+            childLog('Process started: %O', config);
+            const params = await this.call('init');
+            childLog('Params: %O', params);
+            const { eachPage } = await handler(params);
+            const launchOptions = {
+                headless: true,
+                args: [
+                    //
+                    `--lang=${config.locale}`,
+                    '--no-zygote',
+                    '--ignore-certificate-errors',
+                ],
+                ...config,
+            };
+            childLog('Launch options: %O', launchOptions);
+            const browser = await puppeteer.launch(launchOptions);
+            const page = await browser?.newPage();
+            if (!page) {
+                throw new Error('Failed to create page');
+            }
+            page.setDefaultNavigationTimeout(0);
+            if (config.locale) {
+                await page.setExtraHTTPHeaders({
+                    'Accept-Language': config.locale,
+                });
+            }
+            childLog('Page is ready');
+            this.bind('each', async (id, url, index) => {
+                const result = await eachPage({ page, id, url, index }, async (log) => {
+                    await this.call('log', log);
+                });
+                return result;
+            });
+            return async () => {
+                childLog('Close page and browser');
+                await page.close();
+                await browser.close();
+                childLog('Cleanup done');
+            };
+        },
+    });
+}

package/dist/create-main-process.d.ts

@@ -0,0 +1,17 @@
+import type { Logger, PuppeteerDealerOptions } from './types.js';
+import type { LaunchOptions } from 'puppeteer';
+/**
+ *
+ * @param subModulePath
+ * @param params
+ * @param options
+ */
+export declare function createProcess<P, R = void>(subModulePath: string, params: P, options?: PuppeteerDealerOptions & LaunchOptions): ChildProcessManager<P, R>;
+export declare class ChildProcessManager<P, R> {
+    #private;
+    constructor(subModulePath: string, params: P, options?: PuppeteerDealerOptions & LaunchOptions);
+    close(): Promise<void>;
+    each(id: string, url: string, index: number): Promise<R>;
+    log(logger: Logger): void;
+    ready(): Promise<void>;
+}

package/dist/create-main-process.js

@@ -0,0 +1,33 @@
+import { ProcTalk } from '@d-zero/proc-talk';
+/**
+ *
+ * @param subModulePath
+ * @param params
+ * @param options
+ */
+export function createProcess(subModulePath, params, options) {
+    return new ChildProcessManager(subModulePath, params, options);
+}
+export class ChildProcessManager {
+    #procTalk;
+    constructor(subModulePath, params, options) {
+        this.#procTalk = new ProcTalk({
+            type: 'main',
+            subModulePath,
+            options,
+        });
+        this.#procTalk.bind('init', () => Promise.resolve(params));
+    }
+    async close() {
+        await this.#procTalk.close();
+    }
+    async each(id, url, index) {
+        return await this.#procTalk.call('each', id, url, index);
+    }
+    log(logger) {
+        this.#procTalk.bind('log', logger);
+    }
+    async ready() {
+        await this.#procTalk.initialized();
+    }
+}

package/dist/deal.d.ts

@@ -1,11 +1,11 @@
-import type { PuppeteerDealerOptions, PuppeteerDealHandler, URLInfo } from './types.js';
+import type { ChildProcessManager } from './create-main-process.js';
+import type { URLInfo } from './types.js';
 import type { DealHeader } from '@d-zero/dealer';
-import type { PuppeteerLaunchOptions } from 'puppeteer';
 /**
  *
  * @param list
  * @param header
- * @param handler
- * @param options
+ * @param createProcess
+ * @param each
  */
-export declare function deal(list: readonly URLInfo[], header: DealHeader, handler: PuppeteerDealHandler, options?: PuppeteerDealerOptions & PuppeteerLaunchOptions): Promise<void>;
+export declare function deal<T extends Record<string, unknown>, R = void>(list: readonly URLInfo[], header: DealHeader, createProcess: () => ChildProcessManager<T, R>, each?: (result: R) => void | Promise<void>): Promise<void>;

package/dist/deal.js

@@ -1,107 +1,30 @@
 import { deal as coreDeal } from '@d-zero/dealer';
-import { createPage } from '@d-zero/puppeteer-page';
-import { delay } from '@d-zero/shared/delay';
 import c from 'ansi-colors';
-import { log } from './debug.js';
 /**
  *
  * @param list
  * @param header
- * @param handler
- * @param options
+ * @param createProcess
+ * @param each
  */
-export async function deal(list, header, handler, options) {
-    const config = {
-        locale: 'ja-JP',
-        ...options,
-    };
-    const childPrecessIds = new Set();
-    const cleanUp = () => {
-        log('child process IDs: %o', childPrecessIds);
-        for (const pid of childPrecessIds) {
-            try {
-                process.kill(pid);
-                log('killed %d', pid);
-            }
-            catch (error) {
-                log('Already dead: %d', pid);
-                if (error instanceof Error && 'code' in error && error.code === 'ESRCH') {
-                    // ignore
-                    continue;
-                }
-                throw error;
-            }
-        }
-        if (log.enabled) {
-            log('process.getActiveResourcesInfo(): %o', process.getActiveResourcesInfo());
-        }
-    };
-    process.on('exit', cleanUp);
-    await coreDeal(list, ({ id, url }, update, index) => {
+export function deal(list, header, createProcess, each) {
+    return coreDeal(list, ({ id, url }, update, index) => {
         const fileId = id || index.toString().padStart(3, '0');
         const lineHeader = `%braille% ${c.bgWhite(` ${fileId} `)} ${c.gray(url.toString())}: `;
         return async () => {
-            const continued = await handler.beforeOpenPage?.(fileId, url.toString(), (log) => update(`${lineHeader}${log}`), index);
-            if (continued === false) {
-                return;
-            }
-            const page = await createPage({
-                headless: true,
-                args: [
-                    //
-                    `--lang=${config.locale}`,
-                    '--no-zygote',
-                    '--ignore-certificate-errors',
-                ],
-                ...config,
+            const processManager = createProcess();
+            update(`${lineHeader}Booting ChildProcess%dots%`);
+            await processManager.ready();
+            processManager.log((log) => {
+                update(`${lineHeader}${log}`);
             });
-            if (page.pid !== null) {
-                childPrecessIds.add(page.pid);
-            }
-            await page.setDefaultNavigationTimeout(0);
-            if (config.locale) {
-                await page.setExtraHTTPHeaders({
-                    'Accept-Language': config.locale,
-                });
+            const result = await processManager.each(fileId, url.toString(), index);
+            if (each) {
+                await each(result);
             }
-            await handler
-                .deal(page, fileId, url.toString(), (log) => update(`${lineHeader}${log}`), index)
-                .catch(evaluationError(page, url.toString(), fileId, index));
-            update(`${lineHeader} ${c.blue('✓')} Closing page%dots%`);
-            await page.close();
-            update(`${lineHeader} ${c.greenBright('✓')} Page process completed!`);
-            await delay(600);
+            await processManager.close();
         };
     }, {
-        ...config,
         header,
     });
-    log('PuppeteerDealer.deal() completed');
-}
-/**
- *
- * @param page
- * @param url
- * @param fileId
- * @param index
- */
-function evaluationError(page, url, fileId, index) {
-    return (error) => {
-        if (error instanceof Error &&
-            error.message.includes('Execution context was destroyed')) {
-            error.message +=
-                '\n' +
-                    c.red([
-                        `PuppeteerDealer.deal() failed:`,
-                        `    URL: ${url}`,
-                        `    ID: ${fileId}`,
-                        `    Index: ${index}`,
-                        '    Page:',
-                        `        url: ${page.url()}`,
-                        `        isClosed: ${page.isClosed()}`,
-                    ].join('\n'));
-            throw error;
-        }
-        throw error;
-    };
 }

package/dist/index.d.ts

@@ -1,2 +1,4 @@
 export { deal } from './deal.js';
+export { createChildProcess } from './create-child-process.js';
+export { createProcess } from './create-main-process.js';
 export * from './types.js';

package/dist/index.js

@@ -1,2 +1,4 @@
 export { deal } from './deal.js';
+export { createChildProcess } from './create-child-process.js';
+export { createProcess } from './create-main-process.js';
 export * from './types.js';

package/dist/types.d.ts

@@ -1,5 +1,5 @@
 import type { DealOptions } from '@d-zero/dealer';
-import type { Page } from '@d-zero/puppeteer-page';
+import type { Page } from 'puppeteer';
 export type PuppeteerDealerOptions = {
     readonly locale?: string;
 } & DealOptions;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@d-zero/puppeteer-dealer",
-	"version": "0.3.0",
+	"version": "0.4.0",
 	"description": "Puppeteer handles each page",
 	"author": "D-ZERO",
 	"license": "MIT",
@@ -24,14 +24,17 @@
 		"clean": "tsc --build --clean"
 	},
 	"dependencies": {
-		"@d-zero/dealer": "1.3.0",
-		"@d-zero/puppeteer-page": "0.3.0",
-		"@d-zero/shared": "0.7.0",
+		"@d-zero/dealer": "1.3.1",
+		"@d-zero/proc-talk": "0.4.0",
+		"@d-zero/shared": "0.8.0",
 		"ansi-colors": "4.1.3",
-		"debug": "4.4.0"
+		"debug": "4.4.1"
 	},
 	"devDependencies": {
-		"puppeteer": "23.7.1"
+		"puppeteer": "24.9.0"
 	},
-	"gitHead": "e4fd17857e31022d121527b00fd7f009dbdb2142"
+	"peerDependencies": {
+		"puppeteer": "24.8.2"
+	},
+	"gitHead": "4e9cc7b87e0fef91b6f2d4edfb66ca9134b2491b"
 }