@d-zero/puppeteer-dealer 0.7.13 → 0.7.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -234
- package/package.json +4 -4
package/README.md
CHANGED
|
@@ -1,256 +1,44 @@
|
|
|
1
1
|
# `@d-zero/puppeteer-dealer`
|
|
2
2
|
|
|
3
|
-
Puppeteer
|
|
3
|
+
Puppeteer を使ってマルチプロセスで複数 URL を処理するためのユーティリティ。
|
|
4
4
|
|
|
5
|
-
##
|
|
5
|
+
## Installation
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
URLリストを処理するメイン関数です。
|
|
10
|
-
|
|
11
|
-
```ts
|
|
12
|
-
import { deal, createProcess } from '@d-zero/puppeteer-dealer';
|
|
13
|
-
|
|
14
|
-
await deal(
|
|
15
|
-
// URLリスト
|
|
16
|
-
[
|
|
17
|
-
{
|
|
18
|
-
id: '1',
|
|
19
|
-
url: 'https://example.com',
|
|
20
|
-
},
|
|
21
|
-
{
|
|
22
|
-
id: '2',
|
|
23
|
-
url: 'https://example.com',
|
|
24
|
-
},
|
|
25
|
-
],
|
|
26
|
-
|
|
27
|
-
// ヘッダーログ
|
|
28
|
-
(progress, done, total) => {
|
|
29
|
-
return `Header ${Math.ceil(progress * 100)}% ${done}/${total}`;
|
|
30
|
-
},
|
|
31
|
-
|
|
32
|
-
// プロセス作成関数
|
|
33
|
-
() => {
|
|
34
|
-
return createProcess(
|
|
35
|
-
'./child-process.js', // 子プロセスのパス
|
|
36
|
-
{
|
|
37
|
-
// 子プロセスに渡すパラメータ
|
|
38
|
-
// 任意のデータを渡すことができます
|
|
39
|
-
},
|
|
40
|
-
{
|
|
41
|
-
// オプション
|
|
42
|
-
locale: 'ja-JP',
|
|
43
|
-
headless: true,
|
|
44
|
-
},
|
|
45
|
-
);
|
|
46
|
-
},
|
|
47
|
-
|
|
48
|
-
// オプション
|
|
49
|
-
{
|
|
50
|
-
limit: 10,
|
|
51
|
-
debug: false,
|
|
52
|
-
verbose: false,
|
|
53
|
-
},
|
|
54
|
-
);
|
|
55
|
-
```
|
|
56
|
-
|
|
57
|
-
**型定義:**
|
|
58
|
-
|
|
59
|
-
```ts
|
|
60
|
-
function deal<T, R = void>(
|
|
61
|
-
list: readonly URLInfo[],
|
|
62
|
-
header: DealHeader,
|
|
63
|
-
createProcess: () => (needAuth: boolean) => ChildProcessManager<T, R>,
|
|
64
|
-
options?: Omit<DealOptions, 'header'> & {
|
|
65
|
-
each?: (
|
|
66
|
-
result: R,
|
|
67
|
-
push: (...items: URLInfo[]) => Promise<void>,
|
|
68
|
-
) => void | Promise<void>;
|
|
69
|
-
},
|
|
70
|
-
): Promise<void>;
|
|
71
|
-
|
|
72
|
-
type URLInfo = {
|
|
73
|
-
readonly id: string | null;
|
|
74
|
-
readonly url: string | URL;
|
|
75
|
-
};
|
|
76
|
-
|
|
77
|
-
type DealHeader = (
|
|
78
|
-
progress: number,
|
|
79
|
-
done: number,
|
|
80
|
-
total: number,
|
|
81
|
-
limit: number,
|
|
82
|
-
) => string;
|
|
83
|
-
```
|
|
84
|
-
|
|
85
|
-
### `createProcess`
|
|
86
|
-
|
|
87
|
-
子プロセスを作成する関数です。`deal`関数の第3引数で使用します。
|
|
88
|
-
|
|
89
|
-
```ts
|
|
90
|
-
import { createProcess } from '@d-zero/puppeteer-dealer';
|
|
91
|
-
|
|
92
|
-
const processFactory = createProcess(
|
|
93
|
-
'./child-process.js', // 子プロセスのパス
|
|
94
|
-
{
|
|
95
|
-
// 子プロセスに渡すパラメータ
|
|
96
|
-
param1: 'value1',
|
|
97
|
-
param2: 'value2',
|
|
98
|
-
},
|
|
99
|
-
{
|
|
100
|
-
// Puppeteerオプション
|
|
101
|
-
locale: 'ja-JP',
|
|
102
|
-
headless: true,
|
|
103
|
-
// その他のLaunchOptions
|
|
104
|
-
},
|
|
105
|
-
);
|
|
106
|
-
```
|
|
107
|
-
|
|
108
|
-
**型定義:**
|
|
109
|
-
|
|
110
|
-
```ts
|
|
111
|
-
function createProcess<P, R = void>(
|
|
112
|
-
subModulePath: string,
|
|
113
|
-
params: P,
|
|
114
|
-
options?: PuppeteerDealerOptions & LaunchOptions,
|
|
115
|
-
): (needAuth: boolean) => ChildProcessManager<P, R>;
|
|
116
|
-
|
|
117
|
-
type PuppeteerDealerOptions = {
|
|
118
|
-
readonly locale?: string;
|
|
119
|
-
} & DealOptions;
|
|
120
|
-
```
|
|
121
|
-
|
|
122
|
-
### `createChildProcess`
|
|
123
|
-
|
|
124
|
-
子プロセス側で使用する関数です。Puppeteerページの処理を定義します。
|
|
125
|
-
|
|
126
|
-
```ts
|
|
127
|
-
// child-process.ts
|
|
128
|
-
import { createChildProcess } from '@d-zero/puppeteer-dealer';
|
|
129
|
-
|
|
130
|
-
type ChildProcessParams = {
|
|
131
|
-
param1: string;
|
|
132
|
-
param2: string;
|
|
133
|
-
};
|
|
134
|
-
|
|
135
|
-
createChildProcess<ChildProcessParams>((params) => {
|
|
136
|
-
const { param1, param2, needAuth } = params;
|
|
137
|
-
|
|
138
|
-
return {
|
|
139
|
-
async eachPage({ page, id, url, index }, logger) {
|
|
140
|
-
// Puppeteerページの処理
|
|
141
|
-
logger('ページにアクセスしています...');
|
|
142
|
-
await page.goto(url);
|
|
143
|
-
|
|
144
|
-
logger('処理を実行しています...');
|
|
145
|
-
await page.evaluate(() => {
|
|
146
|
-
// ページ内での処理
|
|
147
|
-
});
|
|
148
|
-
|
|
149
|
-
// 戻り値を返すことができます
|
|
150
|
-
return { success: true };
|
|
151
|
-
},
|
|
152
|
-
};
|
|
153
|
-
});
|
|
154
|
-
```
|
|
155
|
-
|
|
156
|
-
**型定義:**
|
|
157
|
-
|
|
158
|
-
```ts
|
|
159
|
-
function createChildProcess<P, R = void>(
|
|
160
|
-
handler: ChildProcessHandler<P & CommonParams, R>,
|
|
161
|
-
): void;
|
|
162
|
-
|
|
163
|
-
type ChildProcessHandler<P extends CommonParams, R> = (
|
|
164
|
-
params: P,
|
|
165
|
-
) => Promise<ChildProcessMethods<R>> | ChildProcessMethods<R>;
|
|
166
|
-
|
|
167
|
-
type ChildProcessMethods<R> = {
|
|
168
|
-
eachPage: (params: EachPageParams, logger: Logger) => Promise<R>;
|
|
169
|
-
};
|
|
170
|
-
|
|
171
|
-
type EachPageParams = {
|
|
172
|
-
readonly page: Page;
|
|
173
|
-
readonly id: string;
|
|
174
|
-
readonly url: string;
|
|
175
|
-
readonly index: number;
|
|
176
|
-
};
|
|
177
|
-
|
|
178
|
-
type CommonParams = {
|
|
179
|
-
readonly needAuth: boolean;
|
|
180
|
-
};
|
|
181
|
-
|
|
182
|
-
type Logger = (log: string) => void;
|
|
7
|
+
```sh
|
|
8
|
+
yarn add @d-zero/puppeteer-dealer
|
|
183
9
|
```
|
|
184
10
|
|
|
185
|
-
##
|
|
186
|
-
|
|
187
|
-
完全な使用例:
|
|
11
|
+
## Usage
|
|
188
12
|
|
|
189
|
-
|
|
13
|
+
親プロセス:
|
|
190
14
|
|
|
191
15
|
```ts
|
|
192
|
-
import path from 'node:path';
|
|
193
16
|
import { deal, createProcess } from '@d-zero/puppeteer-dealer';
|
|
194
17
|
|
|
195
|
-
type ChildProcessParams = {
|
|
196
|
-
outputDir: string;
|
|
197
|
-
};
|
|
198
|
-
|
|
199
18
|
await deal(
|
|
200
|
-
[
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
return `処理中: ${done}/${total}`;
|
|
206
|
-
},
|
|
207
|
-
() => {
|
|
208
|
-
return createProcess<ChildProcessParams>(
|
|
209
|
-
path.resolve(import.meta.dirname, 'child-process.js'),
|
|
210
|
-
{
|
|
211
|
-
outputDir: './output',
|
|
212
|
-
},
|
|
19
|
+
[{ id: '1', url: 'https://example.com' }],
|
|
20
|
+
(progress, done, total) => `${done}/${total}`,
|
|
21
|
+
() =>
|
|
22
|
+
createProcess(
|
|
23
|
+
'./child.js',
|
|
213
24
|
{
|
|
214
|
-
|
|
215
|
-
headless: true,
|
|
25
|
+
/* params */
|
|
216
26
|
},
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
{
|
|
220
|
-
limit: 5,
|
|
221
|
-
debug: false,
|
|
222
|
-
},
|
|
27
|
+
{ locale: 'ja-JP', headless: true },
|
|
28
|
+
),
|
|
29
|
+
{ limit: 5 },
|
|
223
30
|
);
|
|
224
31
|
```
|
|
225
32
|
|
|
226
|
-
|
|
33
|
+
子プロセス(`./child.js`):
|
|
227
34
|
|
|
228
35
|
```ts
|
|
229
36
|
import { createChildProcess } from '@d-zero/puppeteer-dealer';
|
|
230
37
|
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
return {
|
|
239
|
-
async eachPage({ page, id, url }, logger) {
|
|
240
|
-
logger(`${url}を処理しています...`);
|
|
241
|
-
|
|
242
|
-
await page.goto(url);
|
|
243
|
-
|
|
244
|
-
const title = await page.title();
|
|
245
|
-
logger(`タイトル: ${title}`);
|
|
246
|
-
|
|
247
|
-
// スクリーンショットを撮る
|
|
248
|
-
await page.screenshot({
|
|
249
|
-
path: `${outputDir}/${id}.png`,
|
|
250
|
-
});
|
|
251
|
-
|
|
252
|
-
logger('完了');
|
|
253
|
-
},
|
|
254
|
-
};
|
|
255
|
-
});
|
|
38
|
+
createChildProcess<{ outputDir: string }>((params) => ({
|
|
39
|
+
async eachPage({ page, id, url }, logger) {
|
|
40
|
+
await page.goto(url);
|
|
41
|
+
await page.screenshot({ path: `${params.outputDir}/${id}.png` });
|
|
42
|
+
},
|
|
43
|
+
}));
|
|
256
44
|
```
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@d-zero/puppeteer-dealer",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.15",
|
|
4
4
|
"description": "Puppeteer handles each page",
|
|
5
5
|
"author": "D-ZERO",
|
|
6
6
|
"license": "MIT",
|
|
@@ -23,8 +23,8 @@
|
|
|
23
23
|
"clean": "tsc --build --clean"
|
|
24
24
|
},
|
|
25
25
|
"dependencies": {
|
|
26
|
-
"@d-zero/dealer": "1.9.
|
|
27
|
-
"@d-zero/proc-talk": "0.4.
|
|
26
|
+
"@d-zero/dealer": "1.9.2",
|
|
27
|
+
"@d-zero/proc-talk": "0.4.25",
|
|
28
28
|
"@d-zero/shared": "0.22.0",
|
|
29
29
|
"ansi-colors": "4.1.3",
|
|
30
30
|
"debug": "4.4.3",
|
|
@@ -42,5 +42,5 @@
|
|
|
42
42
|
"url": "https://github.com/d-zero-dev/tools.git",
|
|
43
43
|
"directory": "packages/@d-zero/puppeteer-dealer"
|
|
44
44
|
},
|
|
45
|
-
"gitHead": "
|
|
45
|
+
"gitHead": "16c831105a12bb635d49130e7f5add25b6643c40"
|
|
46
46
|
}
|