e10-ebuilder-prototype 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +113 -0
- package/dist/api.d.ts +12 -0
- package/dist/api.js +125 -0
- package/dist/application.d.ts +7 -0
- package/dist/application.js +16 -0
- package/dist/archive.d.ts +130 -0
- package/dist/archive.js +151 -0
- package/dist/capture.d.ts +15 -0
- package/dist/capture.js +440 -0
- package/dist/common.d.ts +20 -0
- package/dist/common.js +87 -0
- package/dist/dom.d.mts +1 -0
- package/dist/dom.mjs +58 -0
- package/dist/form-context.d.ts +3 -0
- package/dist/form-context.js +58 -0
- package/dist/form-runtime.d.mts +2 -0
- package/dist/form-runtime.mjs +149 -0
- package/dist/forms.d.ts +51 -0
- package/dist/forms.js +603 -0
- package/dist/html.d.ts +22 -0
- package/dist/html.js +427 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +370 -0
- package/dist/menus.d.ts +32 -0
- package/dist/menus.js +330 -0
- package/dist/model.d.ts +164 -0
- package/dist/model.js +8 -0
- package/dist/offline-store.d.mts +5 -0
- package/dist/offline-store.mjs +80 -0
- package/dist/platform.d.ts +10 -0
- package/dist/platform.js +123 -0
- package/dist/readiness.d.ts +124 -0
- package/dist/readiness.js +529 -0
- package/dist/runtime-support.d.mts +52 -0
- package/dist/runtime-support.mjs +279 -0
- package/dist/site.d.ts +34 -0
- package/dist/site.js +195 -0
- package/dist/store.d.ts +90 -0
- package/dist/store.js +296 -0
- package/dist/templates/form-guide.md +539 -0
- package/dist/templates/index.html +803 -0
- package/dist/templates/placeholder.html +143 -0
- package/dist/templates/workflow-guide.md +95 -0
- package/dist/templates/workflow-presets.json +89 -0
- package/dist/temporary-records.d.ts +15 -0
- package/dist/temporary-records.js +286 -0
- package/dist/vendor/environment-auth.d.ts +61 -0
- package/dist/vendor/environment-auth.js +455 -0
- package/dist/workflow-runtime.d.mts +2 -0
- package/dist/workflow-runtime.mjs +298 -0
- package/dist/workflows.d.ts +28 -0
- package/dist/workflows.js +90 -0
- package/docs/PROTOCOL.md +299 -0
- package/package.json +45 -0
package/dist/capture.js
ADDED
|
@@ -0,0 +1,440 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import os from 'node:os';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { chromeExecutable, cleanupOwnedDirectory, terminateProcessTree, processAlive, renameWithRetry, environmentValue, } from './runtime-support.mjs';
|
|
5
|
+
import { chromium } from 'playwright-core';
|
|
6
|
+
import { bounded, CaptureError, digest, errorInfo, sleep } from './common.js';
|
|
7
|
+
import { filenames } from './store.js';
|
|
8
|
+
import { Monitor, ready, sample, lazyScroll } from './readiness.js';
|
|
9
|
+
import { expandScrollers } from './dom.mjs';
|
|
10
|
+
export async function launchChrome() {
|
|
11
|
+
const env = { ...process.env };
|
|
12
|
+
if (process.platform === 'win32') {
|
|
13
|
+
const local = environmentValue(env, 'LOCALAPPDATA') || path.join(os.homedir(), 'AppData', 'Local');
|
|
14
|
+
if (!path.isAbsolute(local))
|
|
15
|
+
throw new CaptureError('BROWSER_PROFILE_INVALID', 'LOCALAPPDATA 必须为绝对路径');
|
|
16
|
+
await fs.mkdir(local, { recursive: true });
|
|
17
|
+
for (const key of Object.keys(env))
|
|
18
|
+
if (key.toLowerCase() === 'localappdata')
|
|
19
|
+
delete env[key];
|
|
20
|
+
env.LOCALAPPDATA = local;
|
|
21
|
+
}
|
|
22
|
+
const executablePath = chromeExecutable({ env });
|
|
23
|
+
const parent = await fs.realpath(os.tmpdir());
|
|
24
|
+
const directory = await fs.mkdtemp(path.join(parent, 'e10-capture-browser-'));
|
|
25
|
+
const artifactsDir = path.join(directory, 'artifacts');
|
|
26
|
+
await fs.mkdir(artifactsDir);
|
|
27
|
+
let context;
|
|
28
|
+
let pid;
|
|
29
|
+
const closeTimeout = process.platform === 'win32' ? 30000 : 15000;
|
|
30
|
+
let closed = false;
|
|
31
|
+
const close = async () => {
|
|
32
|
+
if (closed)
|
|
33
|
+
return;
|
|
34
|
+
closed = true;
|
|
35
|
+
let error;
|
|
36
|
+
try {
|
|
37
|
+
if (context)
|
|
38
|
+
await bounded(context.close(), closeTimeout, 'BROWSER_CLOSE_TIMEOUT');
|
|
39
|
+
}
|
|
40
|
+
catch (e) {
|
|
41
|
+
error = e;
|
|
42
|
+
}
|
|
43
|
+
try {
|
|
44
|
+
if (pid && processAlive(pid))
|
|
45
|
+
await terminateProcessTree(pid);
|
|
46
|
+
}
|
|
47
|
+
catch (e) {
|
|
48
|
+
error ??= e;
|
|
49
|
+
}
|
|
50
|
+
try {
|
|
51
|
+
await cleanupOwnedDirectory(directory, parent, 'e10-capture-browser-');
|
|
52
|
+
}
|
|
53
|
+
catch (e) {
|
|
54
|
+
error ??= e;
|
|
55
|
+
}
|
|
56
|
+
if (error)
|
|
57
|
+
throw error;
|
|
58
|
+
};
|
|
59
|
+
try {
|
|
60
|
+
// Own both directories: Playwright must not recursively delete an implicit
|
|
61
|
+
// temporary Chrome profile inside the WorkBuddy host process on Windows.
|
|
62
|
+
context = await chromium.launchPersistentContext(path.join(directory, 'profile'), {
|
|
63
|
+
executablePath,
|
|
64
|
+
artifactsDir,
|
|
65
|
+
env,
|
|
66
|
+
headless: true,
|
|
67
|
+
timeout: 30000,
|
|
68
|
+
handleSIGINT: false,
|
|
69
|
+
handleSIGTERM: false,
|
|
70
|
+
handleSIGHUP: false,
|
|
71
|
+
});
|
|
72
|
+
const browser = context.browser();
|
|
73
|
+
const cdp = await browser.newBrowserCDPSession();
|
|
74
|
+
try {
|
|
75
|
+
const info = await cdp.send('SystemInfo.getProcessInfo');
|
|
76
|
+
pid = info.processInfo.find((p) => p.type === 'browser')?.id;
|
|
77
|
+
if (!pid || !Number.isSafeInteger(pid))
|
|
78
|
+
throw new CaptureError('BROWSER_PID_MISSING', '无法确认受控 Chrome 进程');
|
|
79
|
+
}
|
|
80
|
+
finally {
|
|
81
|
+
await cdp.detach();
|
|
82
|
+
}
|
|
83
|
+
for (const page of context.pages())
|
|
84
|
+
await page.close();
|
|
85
|
+
return { browser, pid, version: browser.version(), directory, close };
|
|
86
|
+
}
|
|
87
|
+
catch (e) {
|
|
88
|
+
await close();
|
|
89
|
+
throw e;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
async function oneAttempt(context, store, item, state, auth, attempt, requireEntrance) {
|
|
93
|
+
let page, monitor, cancelled = false;
|
|
94
|
+
const operation = (async () => {
|
|
95
|
+
attempt.stage = 'create-page';
|
|
96
|
+
page = await context.newPage();
|
|
97
|
+
if (cancelled) {
|
|
98
|
+
await page.close();
|
|
99
|
+
throw new CaptureError('CANCELLED', '任务已取消');
|
|
100
|
+
}
|
|
101
|
+
page.setDefaultTimeout(15000);
|
|
102
|
+
monitor = new Monitor(page, new URL(auth.baseUrl).origin, {
|
|
103
|
+
id: item.id,
|
|
104
|
+
layoutType: item.type,
|
|
105
|
+
});
|
|
106
|
+
const navigationStarted = Date.now();
|
|
107
|
+
const initial = await bounded((async () => {
|
|
108
|
+
attempt.stage = 'navigate';
|
|
109
|
+
const navigation = new URL(item.url);
|
|
110
|
+
// Enter the platform's canonical EB route directly, avoiding its delayed full reload.
|
|
111
|
+
if (item.type && navigation.pathname === `/sp/ebdpage/view/${item.id}`)
|
|
112
|
+
navigation.pathname += `/page/${item.id}`;
|
|
113
|
+
await page.goto(navigation.href, {
|
|
114
|
+
waitUntil: 'domcontentloaded',
|
|
115
|
+
timeout: state.settings.timeoutMs,
|
|
116
|
+
});
|
|
117
|
+
if (new URL(page.url()).origin !== new URL(item.url).origin ||
|
|
118
|
+
/\/(login|passport)(\/|\?|$)/i.test(page.url()))
|
|
119
|
+
throw new CaptureError('E10_LOGIN_REQUIRED', '页面跳转登录或其它环境');
|
|
120
|
+
attempt.stage = 'initial-ready';
|
|
121
|
+
return ready(page, item.id, monitor, navigationStarted + state.settings.timeoutMs, state.settings.stableMs, requireEntrance);
|
|
122
|
+
})(), state.settings.timeoutMs, 'LOADING_TIMEOUT');
|
|
123
|
+
const loadingMs = Date.now() - navigationStarted;
|
|
124
|
+
const warningMap = new Map();
|
|
125
|
+
const retainWarnings = (sampled) => {
|
|
126
|
+
for (const w of sampled.evidence.warnings)
|
|
127
|
+
warningMap.set(`${w.code}:${w.message}`, w);
|
|
128
|
+
return sampled;
|
|
129
|
+
};
|
|
130
|
+
retainWarnings(initial);
|
|
131
|
+
const wait = async () => retainWarnings(await ready(page, item.id, monitor, Date.now() + state.settings.timeoutMs, state.settings.stableMs, requireEntrance));
|
|
132
|
+
attempt.stage = 'lazy-scroll';
|
|
133
|
+
// This iframe is the EB page itself. Ordinary embedded websites retain their viewport.
|
|
134
|
+
let embeddedPage;
|
|
135
|
+
if (monitor.pageConfig?.id === item.id && monitor.pageConfig.layoutType === 'ECODE_HTML') {
|
|
136
|
+
const element = await page.$('.weapp-de-sourcecode-render.sourcecode-html iframe');
|
|
137
|
+
try {
|
|
138
|
+
const frame = await element?.contentFrame();
|
|
139
|
+
if (!frame ||
|
|
140
|
+
new URL(frame.url()).origin !== new URL(item.url).origin ||
|
|
141
|
+
!new URL(frame.url()).pathname.startsWith('/sp/chtml/'))
|
|
142
|
+
throw new CaptureError('PAGE_FRAME_UNAVAILABLE', 'EB 页面内容尚未进入可读取的预览框架');
|
|
143
|
+
await lazyScroll(frame, wait);
|
|
144
|
+
const inside = await expandScrollers(frame, undefined);
|
|
145
|
+
if (inside.residual)
|
|
146
|
+
throw new CaptureError('SCROLL_REMAINS', '页面内容仍有未展开的内部滚动区');
|
|
147
|
+
embeddedPage = await frame.evaluate(() => ({
|
|
148
|
+
width: Math.max(document.documentElement.scrollWidth, document.body.scrollWidth),
|
|
149
|
+
height: Math.max(document.documentElement.scrollHeight, document.body.scrollHeight),
|
|
150
|
+
}));
|
|
151
|
+
if (embeddedPage.height > 50000 || embeddedPage.width * embeddedPage.height > 60000000)
|
|
152
|
+
throw new CaptureError('IMAGE_TOO_LARGE', '页面内容超过截图尺寸上限');
|
|
153
|
+
await element.evaluate((e, size) => {
|
|
154
|
+
const node = e;
|
|
155
|
+
node.style.setProperty('height', `${size.height}px`, 'important');
|
|
156
|
+
node.style.setProperty('width', `${size.width}px`, 'important');
|
|
157
|
+
node.style.setProperty('max-height', 'none', 'important');
|
|
158
|
+
node.style.setProperty('flex-shrink', '0', 'important');
|
|
159
|
+
for (let p = node.parentElement; p; p = p.parentElement) {
|
|
160
|
+
p.style.setProperty('height', 'auto', 'important');
|
|
161
|
+
p.style.setProperty('max-height', 'none', 'important');
|
|
162
|
+
p.style.setProperty('overflow', 'visible', 'important');
|
|
163
|
+
}
|
|
164
|
+
}, embeddedPage);
|
|
165
|
+
await wait();
|
|
166
|
+
}
|
|
167
|
+
finally {
|
|
168
|
+
await element?.dispose();
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
const scrollers = await lazyScroll(page, wait);
|
|
172
|
+
attempt.stage = 'expand';
|
|
173
|
+
const expansion = await expandScrollers(page, undefined);
|
|
174
|
+
if (expansion.residual)
|
|
175
|
+
throw new CaptureError('SCROLL_REMAINS', '仍有内部滚动区未展开', expansion);
|
|
176
|
+
attempt.stage = 'final-ready';
|
|
177
|
+
const before = await wait();
|
|
178
|
+
const missing = initial.evidence.components.filter((c) => !before.evidence.components.some((v) => v.id === c.id));
|
|
179
|
+
if (missing.length)
|
|
180
|
+
throw new CaptureError('COMPONENT_DISAPPEARED', '滚动后组件缺失', { missing });
|
|
181
|
+
if (before.evidence.height > 50000 || before.evidence.height * before.evidence.width > 60000000)
|
|
182
|
+
throw new CaptureError('IMAGE_TOO_LARGE', '页面超过 50000px 高度或 6000 万像素上限', before.evidence);
|
|
183
|
+
attempt.stage = 'screenshot';
|
|
184
|
+
const epoch = monitor.epoch;
|
|
185
|
+
const png = await page.screenshot({
|
|
186
|
+
fullPage: true,
|
|
187
|
+
type: 'png',
|
|
188
|
+
scale: 'css',
|
|
189
|
+
timeout: 10000,
|
|
190
|
+
caret: 'initial',
|
|
191
|
+
});
|
|
192
|
+
attempt.stage = 'post-capture';
|
|
193
|
+
const after = retainWarnings(await sample(page, item.id, requireEntrance, monitor));
|
|
194
|
+
if (!after.ready || before.contentSignature !== after.contentSignature)
|
|
195
|
+
throw new CaptureError('CAPTURE_CHANGED', '截图期间内容、布局或网络状态发生变化', {
|
|
196
|
+
before: before.evidence,
|
|
197
|
+
after: after.evidence,
|
|
198
|
+
ready: after.ready,
|
|
199
|
+
contentChanged: before.contentSignature !== after.contentSignature,
|
|
200
|
+
layoutChanged: before.signature !== after.signature,
|
|
201
|
+
networkChanged: epoch !== monitor.epoch,
|
|
202
|
+
});
|
|
203
|
+
const width = png.readUInt32BE(16), height = png.readUInt32BE(20);
|
|
204
|
+
if (Math.min(Math.abs(width - before.evidence.width), Math.abs(width - after.evidence.width)) >
|
|
205
|
+
4 ||
|
|
206
|
+
Math.min(Math.abs(height - before.evidence.height), Math.abs(height - after.evidence.height)) > 4)
|
|
207
|
+
throw new CaptureError('PNG_BOUNDS_MISMATCH', 'PNG 尺寸与完整页面不一致', {
|
|
208
|
+
width,
|
|
209
|
+
height,
|
|
210
|
+
expected: [before.evidence.width, before.evidence.height],
|
|
211
|
+
});
|
|
212
|
+
return {
|
|
213
|
+
png,
|
|
214
|
+
width,
|
|
215
|
+
height,
|
|
216
|
+
warnings: [...warningMap.values()],
|
|
217
|
+
evidence: {
|
|
218
|
+
loadingMs,
|
|
219
|
+
initial: initial.evidence,
|
|
220
|
+
final: after.evidence,
|
|
221
|
+
scrollers,
|
|
222
|
+
expansion,
|
|
223
|
+
embeddedPage,
|
|
224
|
+
requests: monitor.evidence(),
|
|
225
|
+
},
|
|
226
|
+
};
|
|
227
|
+
})();
|
|
228
|
+
let output, error;
|
|
229
|
+
try {
|
|
230
|
+
output = await bounded(operation, state.settings.timeoutMs + 130000, 'ATTEMPT_TIMEOUT');
|
|
231
|
+
}
|
|
232
|
+
catch (e) {
|
|
233
|
+
error = e;
|
|
234
|
+
attempt.evidence = monitor?.evidence();
|
|
235
|
+
if (page && !page.isClosed())
|
|
236
|
+
try {
|
|
237
|
+
const relative = `diagnostics/${filenames(state.pages || [item])
|
|
238
|
+
.get(item.id)
|
|
239
|
+
.replace(/\.png$/, '')}-attempt-${attempt.number}.png`;
|
|
240
|
+
await fs.mkdir(store.artifact('diagnostics'), { recursive: true });
|
|
241
|
+
await bounded(page.screenshot({ path: store.artifact(relative), timeout: 3000 }), 4000, 'DIAGNOSTIC_TIMEOUT');
|
|
242
|
+
attempt.diagnostic = relative;
|
|
243
|
+
}
|
|
244
|
+
catch {
|
|
245
|
+
/* diagnostic is best effort, original error remains */
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
finally {
|
|
249
|
+
cancelled = true;
|
|
250
|
+
if (page)
|
|
251
|
+
try {
|
|
252
|
+
await bounded(page.close(), 8000, 'PAGE_CLOSE_TIMEOUT');
|
|
253
|
+
}
|
|
254
|
+
catch (e) {
|
|
255
|
+
attempt.cleanupError = errorInfo(e).code;
|
|
256
|
+
error ??= e;
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
if (error)
|
|
260
|
+
throw error;
|
|
261
|
+
return output;
|
|
262
|
+
}
|
|
263
|
+
export async function capture(store, state, auth, options = {}) {
|
|
264
|
+
if (!state.pages)
|
|
265
|
+
throw new CaptureError('DISCOVERY_REQUIRED', '先执行 discover');
|
|
266
|
+
store.bind(state, auth);
|
|
267
|
+
const pending = [];
|
|
268
|
+
for (const item of state.pages) {
|
|
269
|
+
const r = await store.result(item.id);
|
|
270
|
+
if (await store.verified(r))
|
|
271
|
+
continue;
|
|
272
|
+
if (r?.status === 'failed' && !options.retryFailed)
|
|
273
|
+
continue;
|
|
274
|
+
pending.push(item);
|
|
275
|
+
}
|
|
276
|
+
if (!pending.length)
|
|
277
|
+
return;
|
|
278
|
+
delete state.archive;
|
|
279
|
+
await store.save(state);
|
|
280
|
+
const chrome = await launchChrome();
|
|
281
|
+
const contexts = new Map();
|
|
282
|
+
const names = filenames(state.pages);
|
|
283
|
+
let cursor = 0, active = 0, peak = 0, closed = 0, stopped = false, fatal;
|
|
284
|
+
const signal = () => {
|
|
285
|
+
stopped = true;
|
|
286
|
+
void chrome.browser.close().catch(() => { });
|
|
287
|
+
};
|
|
288
|
+
process.once('SIGINT', signal);
|
|
289
|
+
process.once('SIGTERM', signal);
|
|
290
|
+
const started = Date.now();
|
|
291
|
+
options.progress?.({
|
|
292
|
+
event: 'browser-start',
|
|
293
|
+
pid: chrome.pid,
|
|
294
|
+
version: chrome.version,
|
|
295
|
+
concurrency: state.settings.concurrency,
|
|
296
|
+
pages: pending.length,
|
|
297
|
+
});
|
|
298
|
+
async function worker() {
|
|
299
|
+
while (!stopped && cursor < pending.length) {
|
|
300
|
+
const item = pending[cursor++];
|
|
301
|
+
active++;
|
|
302
|
+
peak = Math.max(peak, active);
|
|
303
|
+
try {
|
|
304
|
+
const previous = await store.result(item.id);
|
|
305
|
+
const result = previous || {
|
|
306
|
+
id: item.id,
|
|
307
|
+
name: item.name,
|
|
308
|
+
url: item.url,
|
|
309
|
+
status: 'running',
|
|
310
|
+
history: [],
|
|
311
|
+
};
|
|
312
|
+
if (result.history.at(-1)?.status === 'running') {
|
|
313
|
+
const last = result.history.at(-1);
|
|
314
|
+
last.status = 'failed';
|
|
315
|
+
last.error = { code: 'INTERRUPTED', message: '上次进程中断' };
|
|
316
|
+
last.finishedAt = new Date().toISOString();
|
|
317
|
+
}
|
|
318
|
+
delete result.file;
|
|
319
|
+
delete result.sha256;
|
|
320
|
+
delete result.bytes;
|
|
321
|
+
delete result.warnings;
|
|
322
|
+
result.status = 'running';
|
|
323
|
+
for (let retry = 0; retry <= state.settings.retries && !stopped; retry++) {
|
|
324
|
+
const attempt = {
|
|
325
|
+
number: result.history.length + 1,
|
|
326
|
+
status: 'running',
|
|
327
|
+
stage: 'start',
|
|
328
|
+
startedAt: new Date().toISOString(),
|
|
329
|
+
};
|
|
330
|
+
result.history.push(attempt);
|
|
331
|
+
await store.saveResult(result);
|
|
332
|
+
try {
|
|
333
|
+
const output = await oneAttempt(contexts.get(item.terminal), store, item, state, auth, attempt, options.requireEntrance !== false);
|
|
334
|
+
attempt.evidence = output.evidence;
|
|
335
|
+
const file = names.get(item.id);
|
|
336
|
+
await fs.mkdir(store.artifacts, { recursive: true });
|
|
337
|
+
const temp = store.artifact(`${file}.${process.pid}.tmp`);
|
|
338
|
+
await fs.writeFile(temp, output.png, { mode: 0o600 });
|
|
339
|
+
await renameWithRetry(temp, store.artifact(file));
|
|
340
|
+
Object.assign(result, {
|
|
341
|
+
status: 'succeeded',
|
|
342
|
+
warnings: output.warnings,
|
|
343
|
+
file,
|
|
344
|
+
sha256: digest(output.png),
|
|
345
|
+
bytes: output.png.length,
|
|
346
|
+
width: output.width,
|
|
347
|
+
height: output.height,
|
|
348
|
+
});
|
|
349
|
+
attempt.status = 'succeeded';
|
|
350
|
+
}
|
|
351
|
+
catch (e) {
|
|
352
|
+
attempt.status = 'failed';
|
|
353
|
+
attempt.error = errorInfo(e, auth.eteamsId);
|
|
354
|
+
result.status = 'failed';
|
|
355
|
+
if (attempt.cleanupError || attempt.error.code === 'E10_LOGIN_REQUIRED') {
|
|
356
|
+
stopped = true;
|
|
357
|
+
fatal = e;
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
finally {
|
|
361
|
+
attempt.finishedAt = new Date().toISOString();
|
|
362
|
+
attempt.elapsedMs = Date.parse(attempt.finishedAt) - Date.parse(attempt.startedAt);
|
|
363
|
+
await store.saveResult(result);
|
|
364
|
+
}
|
|
365
|
+
options.progress?.({
|
|
366
|
+
event: 'page-attempt',
|
|
367
|
+
id: item.id,
|
|
368
|
+
name: item.name,
|
|
369
|
+
attempt: attempt.number,
|
|
370
|
+
status: attempt.status,
|
|
371
|
+
stage: attempt.stage,
|
|
372
|
+
elapsedMs: attempt.elapsedMs,
|
|
373
|
+
error: attempt.error,
|
|
374
|
+
});
|
|
375
|
+
if (attempt.status === 'succeeded' ||
|
|
376
|
+
stopped ||
|
|
377
|
+
/TIMEOUT/.test(attempt.error?.code || '') ||
|
|
378
|
+
/Timeout .*exceeded/.test(attempt.error?.message || ''))
|
|
379
|
+
break;
|
|
380
|
+
await sleep(300 * (retry + 1));
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
catch (e) {
|
|
384
|
+
fatal = e;
|
|
385
|
+
stopped = true;
|
|
386
|
+
}
|
|
387
|
+
finally {
|
|
388
|
+
active--;
|
|
389
|
+
closed++;
|
|
390
|
+
options.progress?.({
|
|
391
|
+
event: 'page-released',
|
|
392
|
+
id: item.id,
|
|
393
|
+
active,
|
|
394
|
+
completed: closed,
|
|
395
|
+
total: pending.length,
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
try {
|
|
401
|
+
for (const terminal of new Set(pending.map((p) => p.terminal))) {
|
|
402
|
+
const context = await bounded(chrome.browser.newContext(terminal === 'MOBILE'
|
|
403
|
+
? {
|
|
404
|
+
viewport: { width: 390, height: 844 },
|
|
405
|
+
deviceScaleFactor: 1,
|
|
406
|
+
isMobile: true,
|
|
407
|
+
hasTouch: true,
|
|
408
|
+
userAgent: 'Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/153.0.0.0 Mobile Safari/537.36',
|
|
409
|
+
}
|
|
410
|
+
: {
|
|
411
|
+
viewport: { width: state.settings.width, height: state.settings.height },
|
|
412
|
+
deviceScaleFactor: 1,
|
|
413
|
+
}), 15000, 'CONTEXT_CREATE_TIMEOUT');
|
|
414
|
+
contexts.set(terminal, context);
|
|
415
|
+
await context.addCookies([{ name: 'ETEAMSID', value: auth.eteamsId, url: auth.baseUrl }]);
|
|
416
|
+
}
|
|
417
|
+
await Promise.all(Array.from({ length: Math.min(state.settings.concurrency, pending.length) }, worker));
|
|
418
|
+
}
|
|
419
|
+
finally {
|
|
420
|
+
process.removeListener('SIGINT', signal);
|
|
421
|
+
process.removeListener('SIGTERM', signal);
|
|
422
|
+
try {
|
|
423
|
+
await Promise.all([...contexts.values()].map((c) => bounded(c.close(), 8000, 'CONTEXT_CLOSE_TIMEOUT')));
|
|
424
|
+
}
|
|
425
|
+
finally {
|
|
426
|
+
await chrome.close();
|
|
427
|
+
}
|
|
428
|
+
options.progress?.({
|
|
429
|
+
event: 'browser-closed',
|
|
430
|
+
pid: chrome.pid,
|
|
431
|
+
directory: chrome.directory,
|
|
432
|
+
peakConcurrency: peak,
|
|
433
|
+
elapsedMs: Date.now() - started,
|
|
434
|
+
});
|
|
435
|
+
}
|
|
436
|
+
if (fatal)
|
|
437
|
+
throw fatal;
|
|
438
|
+
if (stopped)
|
|
439
|
+
throw new CaptureError('INTERRUPTED', '批次已中断,可重新 run 恢复');
|
|
440
|
+
}
|
package/dist/common.d.ts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
export declare const VERSION = "0.5.0";
|
|
2
|
+
export declare class CaptureError extends Error {
|
|
3
|
+
code: string;
|
|
4
|
+
details?: unknown | undefined;
|
|
5
|
+
constructor(code: string, message: string, details?: unknown | undefined);
|
|
6
|
+
}
|
|
7
|
+
export declare const sleep: (ms: number) => Promise<void>;
|
|
8
|
+
export declare function digest(data: string | Uint8Array): string;
|
|
9
|
+
export declare function fileDigest(filename: string): Promise<string>;
|
|
10
|
+
export declare function mapLimit<T, R>(items: T[], limit: number, action: (item: T) => Promise<R>): Promise<R[]>;
|
|
11
|
+
export declare function atomicJson(filename: string, value: unknown): Promise<void>;
|
|
12
|
+
export declare function readJson<T>(filename: string): Promise<T>;
|
|
13
|
+
export declare function bounded<T>(promise: Promise<T>, ms: number, code: string): Promise<T>;
|
|
14
|
+
export declare function safeName(value: string): string;
|
|
15
|
+
export declare function id(value: unknown, label?: string): string;
|
|
16
|
+
export declare function errorInfo(error: unknown, secret?: string): {
|
|
17
|
+
details?: unknown;
|
|
18
|
+
code: string;
|
|
19
|
+
message: string;
|
|
20
|
+
};
|
package/dist/common.js
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import { renameWithRetry } from './runtime-support.mjs';
|
|
3
|
+
import { createReadStream } from 'node:fs';
|
|
4
|
+
import path from 'node:path';
|
|
5
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
6
|
+
export const VERSION = '0.5.0';
|
|
7
|
+
export class CaptureError extends Error {
|
|
8
|
+
code;
|
|
9
|
+
details;
|
|
10
|
+
constructor(code, message, details) {
|
|
11
|
+
super(message);
|
|
12
|
+
this.code = code;
|
|
13
|
+
this.details = details;
|
|
14
|
+
this.name = 'CaptureError';
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
export const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
18
|
+
export function digest(data) {
|
|
19
|
+
return createHash('sha256').update(data).digest('hex');
|
|
20
|
+
}
|
|
21
|
+
export async function fileDigest(filename) {
|
|
22
|
+
const hash = createHash('sha256');
|
|
23
|
+
for await (const chunk of createReadStream(filename))
|
|
24
|
+
hash.update(chunk);
|
|
25
|
+
return hash.digest('hex');
|
|
26
|
+
}
|
|
27
|
+
export async function mapLimit(items, limit, action) {
|
|
28
|
+
const out = new Array(items.length);
|
|
29
|
+
let cursor = 0;
|
|
30
|
+
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, async () => {
|
|
31
|
+
while (cursor < items.length) {
|
|
32
|
+
const i = cursor++;
|
|
33
|
+
out[i] = await action(items[i]);
|
|
34
|
+
}
|
|
35
|
+
}));
|
|
36
|
+
return out;
|
|
37
|
+
}
|
|
38
|
+
export async function atomicJson(filename, value) {
|
|
39
|
+
await fs.mkdir(path.dirname(filename), { recursive: true });
|
|
40
|
+
const temp = `${filename}.${process.pid}.${randomUUID()}.tmp`;
|
|
41
|
+
try {
|
|
42
|
+
await fs.writeFile(temp, JSON.stringify(value, null, 2) + '\n', { flag: 'wx', mode: 0o600 });
|
|
43
|
+
await renameWithRetry(temp, filename);
|
|
44
|
+
}
|
|
45
|
+
finally {
|
|
46
|
+
await fs.rm(temp, { force: true });
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
export async function readJson(filename) {
|
|
50
|
+
return JSON.parse(await fs.readFile(filename, 'utf8'));
|
|
51
|
+
}
|
|
52
|
+
export async function bounded(promise, ms, code) {
|
|
53
|
+
let timer;
|
|
54
|
+
try {
|
|
55
|
+
return await Promise.race([
|
|
56
|
+
promise,
|
|
57
|
+
new Promise((_, reject) => {
|
|
58
|
+
timer = setTimeout(() => reject(new CaptureError(code, `超过 ${ms}ms 上限`)), ms);
|
|
59
|
+
}),
|
|
60
|
+
]);
|
|
61
|
+
}
|
|
62
|
+
finally {
|
|
63
|
+
clearTimeout(timer);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
export function safeName(value) {
|
|
67
|
+
return (Array.from(value
|
|
68
|
+
.normalize('NFC')
|
|
69
|
+
.replace(/[\x00-\x1f<>:"/\\|?*]/g, '_')
|
|
70
|
+
.replace(/[.\s]+$/g, ''))
|
|
71
|
+
.slice(0, 65)
|
|
72
|
+
.join('') || 'page');
|
|
73
|
+
}
|
|
74
|
+
export function id(value, label = 'id') {
|
|
75
|
+
if (typeof value !== 'string' || !/^\d{1,30}$/.test(value))
|
|
76
|
+
throw new CaptureError('INVALID_ID', `${label} 必须是十进制字符串`);
|
|
77
|
+
return value;
|
|
78
|
+
}
|
|
79
|
+
export function errorInfo(error, secret = '') {
|
|
80
|
+
const e = error;
|
|
81
|
+
const clean = (v) => (secret ? v.replaceAll(secret, '[REDACTED]') : v);
|
|
82
|
+
return {
|
|
83
|
+
code: e.code || (e.name === 'TimeoutError' ? 'OPERATION_TIMEOUT' : 'UNEXPECTED_ERROR'),
|
|
84
|
+
message: clean(e.message || String(error)),
|
|
85
|
+
...(e.details ? { details: JSON.parse(clean(JSON.stringify(e.details))) } : {}),
|
|
86
|
+
};
|
|
87
|
+
}
|
package/dist/dom.d.mts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export function expandScrollers(frame: any, scrollSelector: any): Promise<any>;
|
package/dist/dom.mjs
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
// Browser-side DOM helpers extracted from the validated screenshot prototype.
|
|
2
|
+
export async function expandScrollers(frame, scrollSelector) {
|
|
3
|
+
return frame.evaluate(({ scrollSelector }) => {
|
|
4
|
+
const root = document.scrollingElement || document.documentElement;
|
|
5
|
+
const explicit = new Set(scrollSelector ? document.querySelectorAll(scrollSelector) : []);
|
|
6
|
+
const modified = new Set();
|
|
7
|
+
const grow = (e, property, value) => e.style.setProperty(property, value, 'important');
|
|
8
|
+
for (let pass = 0; pass < 8; pass++) {
|
|
9
|
+
const regions = [...document.querySelectorAll('*')].filter((e) => {
|
|
10
|
+
if (e === root ||
|
|
11
|
+
e.clientHeight < 60 ||
|
|
12
|
+
e.clientWidth < 60 ||
|
|
13
|
+
getComputedStyle(e).visibility === 'hidden')
|
|
14
|
+
return false;
|
|
15
|
+
const style = getComputedStyle(e);
|
|
16
|
+
return (((explicit.has(e) || /(auto|scroll)/.test(style.overflowY)) &&
|
|
17
|
+
e.scrollHeight > e.clientHeight + 3) ||
|
|
18
|
+
(/(auto|scroll)/.test(style.overflowX) && e.scrollWidth > e.clientWidth + 3));
|
|
19
|
+
});
|
|
20
|
+
if (!regions.length)
|
|
21
|
+
break;
|
|
22
|
+
for (const e of regions.reverse()) {
|
|
23
|
+
const height = e.scrollHeight + Math.max(0, e.offsetHeight - e.clientHeight);
|
|
24
|
+
const width = e.scrollWidth + Math.max(0, e.offsetWidth - e.clientWidth);
|
|
25
|
+
if (e.scrollHeight > e.clientHeight + 3)
|
|
26
|
+
grow(e, 'height', `${height}px`);
|
|
27
|
+
if (e.scrollWidth > e.clientWidth + 3)
|
|
28
|
+
grow(e, 'width', `${width}px`);
|
|
29
|
+
for (const [key, value] of Object.entries({
|
|
30
|
+
'max-height': 'none',
|
|
31
|
+
'max-width': 'none',
|
|
32
|
+
overflow: 'visible',
|
|
33
|
+
'flex-shrink': '0',
|
|
34
|
+
contain: 'none',
|
|
35
|
+
}))
|
|
36
|
+
grow(e, key, value);
|
|
37
|
+
e.scrollTop = 0;
|
|
38
|
+
e.scrollLeft = 0;
|
|
39
|
+
modified.add(e);
|
|
40
|
+
for (let p = e.parentElement; p; p = p.parentElement) {
|
|
41
|
+
grow(p, 'overflow', 'visible');
|
|
42
|
+
grow(p, 'max-height', 'none');
|
|
43
|
+
grow(p, 'height', 'auto');
|
|
44
|
+
grow(p, 'contain', 'none');
|
|
45
|
+
grow(p, 'flex-shrink', '0');
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
window.scrollTo(0, 0);
|
|
50
|
+
const residual = [...document.querySelectorAll('*')].filter((e) => e !== root &&
|
|
51
|
+
e.clientHeight > 60 &&
|
|
52
|
+
e.clientWidth > 60 &&
|
|
53
|
+
getComputedStyle(e).visibility !== 'hidden' &&
|
|
54
|
+
/(auto|scroll)/.test(getComputedStyle(e).overflowY) &&
|
|
55
|
+
e.scrollHeight > e.clientHeight + 3).length;
|
|
56
|
+
return { expanded: modified.size, residual };
|
|
57
|
+
}, { scrollSelector });
|
|
58
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { fileDigest } from './common.js';
|
|
4
|
+
import { formInputPath } from './forms.js';
|
|
5
|
+
import { navigationKey } from './menus.js';
|
|
6
|
+
function* parts(value, location) {
|
|
7
|
+
if (JSON.stringify(value).length <= 18000) {
|
|
8
|
+
yield { location, value };
|
|
9
|
+
return;
|
|
10
|
+
}
|
|
11
|
+
if (Array.isArray(value)) {
|
|
12
|
+
for (const [i, item] of value.entries())
|
|
13
|
+
yield* parts(item, `${location}[${i}]`);
|
|
14
|
+
}
|
|
15
|
+
else if (value && typeof value === 'object') {
|
|
16
|
+
for (const [key, item] of Object.entries(value))
|
|
17
|
+
yield* parts(item, `${location}.${key}`);
|
|
18
|
+
}
|
|
19
|
+
else if (typeof value === 'string') {
|
|
20
|
+
for (let offset = 0; offset < value.length; offset += 6000)
|
|
21
|
+
yield {
|
|
22
|
+
location: `${location} (characters ${offset}:${Math.min(value.length, offset + 6000)})`,
|
|
23
|
+
value: value.slice(offset, offset + 6000),
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
else
|
|
27
|
+
yield { location, value };
|
|
28
|
+
}
|
|
29
|
+
/** Generated source views are disposable; regenerate from the verified canonical input on resume. */
|
|
30
|
+
export async function formContext(store, key) {
|
|
31
|
+
const source = formInputPath(store, key), input = JSON.parse(await fs.readFile(source, 'utf8'));
|
|
32
|
+
const folder = path.join(store.meta, 'form-context', navigationKey(key), await fileDigest(source));
|
|
33
|
+
await fs.mkdir(folder, { recursive: true });
|
|
34
|
+
const index = [];
|
|
35
|
+
for (const [category, value] of Object.entries(input)) {
|
|
36
|
+
let i = 0;
|
|
37
|
+
for (const part of parts(value, category)) {
|
|
38
|
+
const filename = `${category.replace(/[^a-zA-Z0-9_-]/g, '_')}-${++i}.json`;
|
|
39
|
+
await fs.writeFile(path.join(folder, filename), JSON.stringify(part, null, 2), {
|
|
40
|
+
mode: 0o600,
|
|
41
|
+
});
|
|
42
|
+
index.push({ path: filename, location: part.location });
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
const indexPath = path.join(folder, 'index.json');
|
|
46
|
+
await fs.writeFile(indexPath, JSON.stringify({
|
|
47
|
+
source,
|
|
48
|
+
appId: input.appId,
|
|
49
|
+
objId: input.objId,
|
|
50
|
+
menu: input.page,
|
|
51
|
+
fieldCount: input.fields?.length,
|
|
52
|
+
customFieldCount: input.fields?.filter((f) => !f.system).length,
|
|
53
|
+
layoutSource: input.layout?.source,
|
|
54
|
+
warnings: input.warnings,
|
|
55
|
+
fragments: index,
|
|
56
|
+
}, null, 2), { mode: 0o600 });
|
|
57
|
+
return indexPath;
|
|
58
|
+
}
|