@dsh-workbench/adapter-runtime 1.2.11 → 1.3.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/channel/core.js +2 -1
- package/channel/opencli-plugin.json +1 -1
- package/channel/package.json +1 -1
- package/dist/browser/browser-read.d.ts +2 -0
- package/dist/browser/browser-read.js +57 -16
- package/dist/data/attachment.d.ts +3 -0
- package/dist/data/attachment.js +31 -0
- package/dist/data/channel.js +3 -0
- package/dist/data/command-read.d.ts +7 -0
- package/dist/data/command-read.js +52 -0
- package/dist/data/http.d.ts +5 -0
- package/dist/data/http.js +36 -0
- package/dist/data/index.d.ts +3 -8
- package/dist/data/index.js +197 -205
- package/dist/data/plan-cache.d.ts +11 -0
- package/dist/data/plan-cache.js +53 -0
- package/dist/data/queue.d.ts +11 -0
- package/dist/data/queue.js +50 -0
- package/dist/data/result.d.ts +9 -0
- package/dist/data/result.js +93 -0
- package/dist/data/submit-plan.d.ts +7 -0
- package/dist/data/submit-plan.js +150 -0
- package/dist/data/types.d.ts +19 -0
- package/dist/data/types.js +4 -0
- package/package.json +2 -2
package/dist/data/index.js
CHANGED
|
@@ -1,150 +1,113 @@
|
|
|
1
|
-
import { callsProblemOf, noWrite, recipeProblemOf, scriptOf, scriptOfMany } from '@dsh-workbench/shared/host';
|
|
1
|
+
import { callsProblemOf, noWrite, now, planProblemOf, planResultsOf, recipeProblemOf, scriptOf, scriptOfMany, scriptOfPlan, allSame, } from '@dsh-workbench/shared/host';
|
|
2
2
|
import { browserRead } from '../browser/index.js';
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
3
|
+
import { attachOf } from './attachment.js';
|
|
4
|
+
import { readHttp } from './http.js';
|
|
5
|
+
import { createCommandRead } from './command-read.js';
|
|
6
|
+
import { createSessionQueue } from './queue.js';
|
|
7
|
+
import { submitPlanOf } from './submit-plan.js';
|
|
8
|
+
import { sessionOf } from './types.js';
|
|
9
|
+
import { allFailure, asRecord, browserFailure, failure, pageFailure, pickOf, } from './result.js';
|
|
5
10
|
import { channelCallOf, channelCommandLine, ensureChannel, } from './channel.js';
|
|
6
11
|
export * from './channel-plugin.js';
|
|
7
|
-
const
|
|
8
|
-
function
|
|
9
|
-
|
|
12
|
+
const MAX_WIN32_ARGV = 24_000;
|
|
13
|
+
function scriptArgProblemOf(script) {
|
|
14
|
+
if (typeof process === 'undefined' || process.platform !== 'win32')
|
|
15
|
+
return undefined;
|
|
16
|
+
if (Math.ceil(Buffer.byteLength(script, 'utf8') / 3) * 4 <= MAX_WIN32_ARGV)
|
|
17
|
+
return undefined;
|
|
18
|
+
return '这份计划在 Windows 上太长(脚本超过命令行上限):拆成几份计划,或把步骤写短一点';
|
|
10
19
|
}
|
|
11
|
-
function
|
|
12
|
-
|
|
13
|
-
|
|
20
|
+
export function createDataReader(options = {}) {
|
|
21
|
+
const fetchOf = options.fetch ?? globalThis.fetch;
|
|
22
|
+
const command = options.command ?? 'opencli';
|
|
23
|
+
const queue = createSessionQueue();
|
|
24
|
+
const queueWaitMs = options.queueWaitMs ?? 30_000;
|
|
25
|
+
const readCommand = createCommandRead({ seam: options.seam, command });
|
|
26
|
+
function browserCall(input) {
|
|
27
|
+
return {
|
|
28
|
+
seam: input.seam,
|
|
29
|
+
script: input.script,
|
|
30
|
+
command,
|
|
31
|
+
...(input.url === undefined ? {} : { url: input.url }),
|
|
32
|
+
...(input.loginUrl === undefined || input.loginUrl === '' ? {} : { loginUrl: input.loginUrl }),
|
|
33
|
+
session: input.session,
|
|
34
|
+
...(input.open === undefined ? {} : { open: input.open }),
|
|
35
|
+
...(input.timeoutMs === undefined ? {} : { timeoutMs: input.timeoutMs }),
|
|
36
|
+
...(input.signal === undefined ? {} : { signal: input.signal }),
|
|
37
|
+
...(input.retry === undefined ? {} : { retry: input.retry }),
|
|
38
|
+
};
|
|
14
39
|
}
|
|
15
|
-
|
|
16
|
-
|
|
40
|
+
function unwrapBrowser(got, loginUrl) {
|
|
41
|
+
if (got.kind === 'failed')
|
|
42
|
+
return got.result;
|
|
43
|
+
if (!got.value.ok)
|
|
44
|
+
return browserFailure(got.value, loginUrl);
|
|
45
|
+
return { ok: true, result: got.value };
|
|
17
46
|
}
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
code === 'PERMISSION_DENIED' ||
|
|
30
|
-
code === 'FORBIDDEN' ||
|
|
31
|
-
code.includes('UNAUTH')) {
|
|
32
|
-
return 'auth';
|
|
47
|
+
function recipeBrowserInput(recipe, script, seam, callOptions) {
|
|
48
|
+
return {
|
|
49
|
+
seam,
|
|
50
|
+
script,
|
|
51
|
+
url: recipe.origin,
|
|
52
|
+
...(recipe.loginUrl === undefined ? {} : { loginUrl: recipe.loginUrl }),
|
|
53
|
+
...(recipe.session === undefined ? {} : { session: recipe.session }),
|
|
54
|
+
...(callOptions.open === undefined ? {} : { open: callOptions.open }),
|
|
55
|
+
...(recipe.timeoutMs === undefined ? {} : { timeoutMs: recipe.timeoutMs }),
|
|
56
|
+
...(callOptions.signal === undefined ? {} : { signal: callOptions.signal }),
|
|
57
|
+
};
|
|
33
58
|
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
return
|
|
40
|
-
return 'browser';
|
|
41
|
-
}
|
|
42
|
-
function statusOf(pageCode) {
|
|
43
|
-
const match = /^http-(\d{3})$/i.exec(pageCode.trim());
|
|
44
|
-
return match === null ? undefined : Number(match[1]);
|
|
45
|
-
}
|
|
46
|
-
function browserFailure(result, loginUrl) {
|
|
47
|
-
const pageCode = result.pageCode ?? result.code;
|
|
48
|
-
const mapped = dataCodeOf(result.code);
|
|
49
|
-
const code = mapped === 'browser' ? dataCodeOf(pageCode) : mapped;
|
|
50
|
-
const status = statusOf(result.code) ?? statusOf(pageCode);
|
|
51
|
-
const helpUrl = result.helpUrl ?? (code === 'auth' ? loginUrl : undefined);
|
|
52
|
-
return {
|
|
53
|
-
ok: false,
|
|
54
|
-
error: {
|
|
55
|
-
code,
|
|
56
|
-
message: result.message,
|
|
57
|
-
...(status === undefined ? {} : { status }),
|
|
58
|
-
...(result.detail === undefined ? {} : { detail: result.detail }),
|
|
59
|
-
...(result.hint === undefined ? {} : { hint: result.hint }),
|
|
60
|
-
...(helpUrl === undefined ? {} : { helpUrl }),
|
|
61
|
-
...(pageCode === '' ? {} : { pageCode }),
|
|
62
|
-
},
|
|
63
|
-
};
|
|
64
|
-
}
|
|
65
|
-
function pageFailure(pageCode, message, loginUrl) {
|
|
66
|
-
const code = dataCodeOf(pageCode);
|
|
67
|
-
const status = statusOf(pageCode);
|
|
68
|
-
const helpUrl = code === 'auth' ? loginUrl : undefined;
|
|
69
|
-
return {
|
|
70
|
-
ok: false,
|
|
71
|
-
error: {
|
|
72
|
-
code,
|
|
73
|
-
message,
|
|
74
|
-
...(status === undefined ? {} : { status }),
|
|
75
|
-
...(helpUrl === undefined ? {} : { helpUrl }),
|
|
76
|
-
...(pageCode === '' ? {} : { pageCode }),
|
|
77
|
-
},
|
|
78
|
-
};
|
|
79
|
-
}
|
|
80
|
-
function pickOf(value, path) {
|
|
81
|
-
let at = value;
|
|
82
|
-
for (const key of path.split('.')) {
|
|
83
|
-
const box = asRecord(at);
|
|
84
|
-
if (box === undefined)
|
|
85
|
-
return undefined;
|
|
86
|
-
at = box[key];
|
|
59
|
+
async function guardedBrowser(input) {
|
|
60
|
+
const argv = scriptArgProblemOf(input.script);
|
|
61
|
+
if (argv !== undefined)
|
|
62
|
+
return { kind: 'failed', result: failure('invalid-request', argv) };
|
|
63
|
+
const session = sessionOf(input.session);
|
|
64
|
+
return serialGuard(session, input.signal, () => browserRead(browserCall({ ...input, session })));
|
|
87
65
|
}
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
const out = {};
|
|
92
|
-
for (const name of names)
|
|
93
|
-
out[name] = { ok: false, error: { code, message, ...extra } };
|
|
94
|
-
return out;
|
|
95
|
-
}
|
|
96
|
-
export function createDataReader(options = {}) {
|
|
97
|
-
const fetchOf = options.fetch ?? globalThis.fetch;
|
|
98
|
-
const command = options.command ?? 'opencli';
|
|
99
|
-
async function readHttp(request) {
|
|
100
|
-
if (fetchOf === undefined)
|
|
101
|
-
return failure('invalid-request', '这个运行时没有 fetch');
|
|
102
|
-
const timeoutMs = request.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
103
|
-
const controller = new AbortController();
|
|
104
|
-
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
66
|
+
async function serialGuard(session, signal, work) {
|
|
67
|
+
const started = now();
|
|
68
|
+
const outcome = await queue.acquire(session, signal, queueWaitMs);
|
|
105
69
|
try {
|
|
106
|
-
|
|
107
|
-
method: 'GET',
|
|
108
|
-
...(request.headers === undefined ? {} : { headers: request.headers }),
|
|
109
|
-
signal: controller.signal,
|
|
110
|
-
});
|
|
111
|
-
const text = await response.text();
|
|
112
|
-
if (!response.ok) {
|
|
113
|
-
return failure(response.status === 401 || response.status === 403 ? 'auth' : 'http', `HTTP ${response.status}`, {
|
|
114
|
-
status: response.status,
|
|
115
|
-
detail: text.slice(0, 2000),
|
|
116
|
-
});
|
|
117
|
-
}
|
|
118
|
-
return parse(text);
|
|
70
|
+
options.onQueue?.({ session, waitedMs: now() - started, outcome });
|
|
119
71
|
}
|
|
120
|
-
catch
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
return failure('
|
|
72
|
+
catch {
|
|
73
|
+
}
|
|
74
|
+
if (outcome === 'aborted') {
|
|
75
|
+
return { kind: 'failed', result: failure('aborted', '已停止', { pageCode: 'aborted' }) };
|
|
76
|
+
}
|
|
77
|
+
if (outcome === 'timeout') {
|
|
78
|
+
return {
|
|
79
|
+
kind: 'failed',
|
|
80
|
+
result: failure('timeout', '排队太久,没轮到这次读', { hint: '稍后重试' }),
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
try {
|
|
84
|
+
return { kind: 'done', value: await work() };
|
|
124
85
|
}
|
|
125
86
|
finally {
|
|
126
|
-
|
|
87
|
+
queue.release(session);
|
|
127
88
|
}
|
|
128
89
|
}
|
|
129
|
-
async function readBrowser(request) {
|
|
90
|
+
async function readBrowser(request, callOptions) {
|
|
130
91
|
if (options.seam === undefined)
|
|
131
92
|
return failure('invalid-request', '没有执行缝,读不了浏览器');
|
|
132
|
-
const
|
|
93
|
+
const open = callOptions.open ?? request.open;
|
|
94
|
+
const got = await guardedBrowser({
|
|
133
95
|
seam: options.seam,
|
|
134
96
|
script: request.script,
|
|
135
|
-
command,
|
|
136
97
|
...(request.url === undefined ? {} : { url: request.url }),
|
|
137
98
|
...(request.loginUrl === undefined ? {} : { loginUrl: request.loginUrl }),
|
|
138
99
|
...(request.session === undefined ? {} : { session: request.session }),
|
|
139
|
-
...(
|
|
100
|
+
...(open === undefined ? {} : { open }),
|
|
140
101
|
...(request.timeoutMs === undefined ? {} : { timeoutMs: request.timeoutMs }),
|
|
102
|
+
...(callOptions.signal === undefined ? {} : { signal: callOptions.signal }),
|
|
141
103
|
});
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
104
|
+
const unwrapped = unwrapBrowser(got, request.loginUrl);
|
|
105
|
+
if (!unwrapped.ok)
|
|
106
|
+
return unwrapped;
|
|
107
|
+
return { ok: true, data: unwrapped.result.value };
|
|
145
108
|
}
|
|
146
109
|
const channelState = { value: 'unknown' };
|
|
147
|
-
async function readChannel(seam, call, recipe) {
|
|
110
|
+
async function readChannel(seam, call, recipe, signal) {
|
|
148
111
|
const result = await seam
|
|
149
112
|
.run({
|
|
150
113
|
command: channelCommandLine(call),
|
|
@@ -152,10 +115,13 @@ export function createDataReader(options = {}) {
|
|
|
152
115
|
retry: recipe.request.method === 'GET' ? 'safe' : 'never',
|
|
153
116
|
timeoutMs: call.timeoutMs + 5000,
|
|
154
117
|
bestEffort: true,
|
|
118
|
+
...(signal === undefined ? {} : { signal }),
|
|
155
119
|
})
|
|
156
120
|
.catch(() => undefined);
|
|
157
121
|
if (result === undefined)
|
|
158
122
|
return undefined;
|
|
123
|
+
if (result.aborted)
|
|
124
|
+
return failure('aborted', '已停止', { pageCode: 'aborted' });
|
|
159
125
|
if (result.timedOut)
|
|
160
126
|
return failure('timeout', `通道读超时(${call.timeoutMs}ms)`);
|
|
161
127
|
if (result.exitCode !== 0)
|
|
@@ -190,19 +156,24 @@ export function createDataReader(options = {}) {
|
|
|
190
156
|
? failure('parse', `返回里没有 ${path} 这一格`, { detail: JSON.stringify(parsed).slice(0, 2000) })
|
|
191
157
|
: { ok: true, data: value };
|
|
192
158
|
}
|
|
193
|
-
async function readRecipe(recipe, overrides, mode) {
|
|
194
|
-
const problem = recipeProblemOf(recipe);
|
|
159
|
+
async function readRecipe(recipe, overrides, mode, callOptions) {
|
|
160
|
+
const problem = recipeProblemOf(recipe, overrides);
|
|
195
161
|
if (problem !== undefined)
|
|
196
162
|
return failure('invalid-request', problem);
|
|
163
|
+
if (recipe.origin === undefined || recipe.origin === '') {
|
|
164
|
+
return failure('invalid-request', '这条配方只写了 site,没经过宿主解析');
|
|
165
|
+
}
|
|
197
166
|
if (mode === 'request' && recipe.request.method !== 'GET') {
|
|
198
167
|
return failure('invalid-request', 'request 只读 GET;写走 submit');
|
|
199
168
|
}
|
|
200
169
|
if (options.seam === undefined)
|
|
201
170
|
return failure('invalid-request', '没有执行缝,读不了浏览器');
|
|
202
171
|
const seam = options.seam;
|
|
203
|
-
const call = options.channel === false
|
|
172
|
+
const call = options.channel === false || recipe.authLoss !== undefined
|
|
173
|
+
? undefined
|
|
174
|
+
: channelCallOf(recipe, overrides ?? {}, command);
|
|
204
175
|
if (call !== undefined && (await ensureChannel(seam, command, channelState, options.channelInstalled))) {
|
|
205
|
-
const viaChannel = await readChannel(seam, call, recipe);
|
|
176
|
+
const viaChannel = await readChannel(seam, call, recipe, callOptions.signal);
|
|
206
177
|
if (viaChannel !== undefined)
|
|
207
178
|
return viaChannel;
|
|
208
179
|
channelState.value = 'unavailable';
|
|
@@ -214,17 +185,11 @@ export function createDataReader(options = {}) {
|
|
|
214
185
|
catch (error) {
|
|
215
186
|
return failure('invalid-request', error instanceof Error ? error.message : String(error));
|
|
216
187
|
}
|
|
217
|
-
const
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
...(recipe.loginUrl === undefined ? {} : { loginUrl: recipe.loginUrl }),
|
|
223
|
-
...(recipe.session === undefined ? {} : { session: recipe.session }),
|
|
224
|
-
...(recipe.timeoutMs === undefined ? {} : { timeoutMs: recipe.timeoutMs }),
|
|
225
|
-
});
|
|
226
|
-
if (!result.ok)
|
|
227
|
-
return browserFailure(result, recipe.loginUrl);
|
|
188
|
+
const got = await guardedBrowser(recipeBrowserInput(recipe, script, seam, callOptions));
|
|
189
|
+
const unwrapped = unwrapBrowser(got, recipe.loginUrl);
|
|
190
|
+
if (!unwrapped.ok)
|
|
191
|
+
return unwrapped;
|
|
192
|
+
const result = unwrapped.result;
|
|
228
193
|
const box = asRecord(result.value);
|
|
229
194
|
if (box !== undefined && box['ok'] === true) {
|
|
230
195
|
if ('data' in box)
|
|
@@ -236,11 +201,14 @@ export function createDataReader(options = {}) {
|
|
|
236
201
|
detail: JSON.stringify(result.value).slice(0, 2000),
|
|
237
202
|
});
|
|
238
203
|
}
|
|
239
|
-
async function readMany(recipe, calls) {
|
|
240
|
-
const names = Object.keys(calls);
|
|
204
|
+
async function readMany(recipe, calls, callOptions) {
|
|
205
|
+
const names = calls !== null && typeof calls === 'object' ? Object.keys(calls) : [];
|
|
241
206
|
const problem = callsProblemOf(recipe, calls);
|
|
242
207
|
if (problem !== undefined)
|
|
243
208
|
return allFailure(names, 'invalid-request', problem);
|
|
209
|
+
if (recipe.origin === undefined || recipe.origin === '') {
|
|
210
|
+
return allFailure(names, 'invalid-request', '这条配方只写了 site,没经过宿主解析');
|
|
211
|
+
}
|
|
244
212
|
if (recipe.request.method !== 'GET') {
|
|
245
213
|
return allFailure(names, 'invalid-request', 'requestMany 只读 GET;写走 submit');
|
|
246
214
|
}
|
|
@@ -254,15 +222,10 @@ export function createDataReader(options = {}) {
|
|
|
254
222
|
catch (error) {
|
|
255
223
|
return allFailure(names, 'invalid-request', error instanceof Error ? error.message : String(error));
|
|
256
224
|
}
|
|
257
|
-
const
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
url: recipe.origin,
|
|
262
|
-
...(recipe.loginUrl === undefined ? {} : { loginUrl: recipe.loginUrl }),
|
|
263
|
-
...(recipe.session === undefined ? {} : { session: recipe.session }),
|
|
264
|
-
...(recipe.timeoutMs === undefined ? {} : { timeoutMs: recipe.timeoutMs }),
|
|
265
|
-
});
|
|
225
|
+
const got = await guardedBrowser(recipeBrowserInput(recipe, script, seam, callOptions));
|
|
226
|
+
if (got.kind === 'failed')
|
|
227
|
+
return allFailure(names, got.result.error.code, got.result.error.message, got.result.error);
|
|
228
|
+
const result = got.value;
|
|
266
229
|
if (!result.ok) {
|
|
267
230
|
const one = browserFailure(result, recipe.loginUrl);
|
|
268
231
|
const failed = {};
|
|
@@ -301,71 +264,100 @@ export function createDataReader(options = {}) {
|
|
|
301
264
|
}
|
|
302
265
|
return out;
|
|
303
266
|
}
|
|
304
|
-
const openclis = new Map();
|
|
305
|
-
function opencliOf(plugin, timeoutMs) {
|
|
306
|
-
const key = `${plugin}:${String(timeoutMs)}`;
|
|
307
|
-
const cached = openclis.get(key);
|
|
308
|
-
if (cached !== undefined)
|
|
309
|
-
return cached;
|
|
310
|
-
const created = createOpencli({
|
|
311
|
-
seam: options.seam,
|
|
312
|
-
plugin,
|
|
313
|
-
command,
|
|
314
|
-
timeoutMs,
|
|
315
|
-
});
|
|
316
|
-
openclis.set(key, created);
|
|
317
|
-
return created;
|
|
318
|
-
}
|
|
319
|
-
async function readCommand(request) {
|
|
320
|
-
if (options.seam === undefined)
|
|
321
|
-
return failure('invalid-request', '没有执行缝,跑不了命令');
|
|
322
|
-
if (request.adapter !== 'opencli') {
|
|
323
|
-
return failure('invalid-request', `不认识的命令适配器:${request.adapter}`);
|
|
324
|
-
}
|
|
325
|
-
const plugin = request.argv[0];
|
|
326
|
-
if (plugin === undefined)
|
|
327
|
-
return failure('invalid-request', '命令缺少 argv[0](插件名)');
|
|
328
|
-
const timeoutMs = request.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
329
|
-
try {
|
|
330
|
-
const { value } = await opencliOf(plugin, timeoutMs).json(request.argv.slice(1));
|
|
331
|
-
return { ok: true, data: value };
|
|
332
|
-
}
|
|
333
|
-
catch (error) {
|
|
334
|
-
if (!(error instanceof WorkbenchDataError))
|
|
335
|
-
return failure('command', String(error));
|
|
336
|
-
const code = error.code === 'auth-required' || error.code === 'permission-denied'
|
|
337
|
-
? 'auth'
|
|
338
|
-
: error.code === 'cli-timeout'
|
|
339
|
-
? 'timeout'
|
|
340
|
-
: error.code === 'cli-bad-output'
|
|
341
|
-
? 'parse'
|
|
342
|
-
: 'command';
|
|
343
|
-
return failure(code, error.message, {
|
|
344
|
-
...(error.detail === undefined ? {} : { detail: error.detail }),
|
|
345
|
-
...(error.hint === undefined ? {} : { hint: error.hint }),
|
|
346
|
-
...(error.helpUrl === undefined ? {} : { helpUrl: error.helpUrl }),
|
|
347
|
-
pageCode: error.code,
|
|
348
|
-
});
|
|
349
|
-
}
|
|
350
|
-
}
|
|
351
267
|
return {
|
|
352
|
-
async read(request) {
|
|
268
|
+
async read(request, callOptions = {}) {
|
|
353
269
|
if (request.kind === 'http')
|
|
354
|
-
return (await readHttp(request));
|
|
270
|
+
return (await readHttp(request, callOptions, fetchOf));
|
|
355
271
|
if (request.kind === 'browser')
|
|
356
|
-
return (await readBrowser(request));
|
|
272
|
+
return (await readBrowser(request, callOptions));
|
|
357
273
|
if (request.kind === 'command')
|
|
358
|
-
return (await readCommand(request));
|
|
274
|
+
return (await readCommand(request, callOptions));
|
|
359
275
|
return failure('invalid-request', '不认识的读类型');
|
|
360
276
|
},
|
|
361
|
-
async request(recipe, overrides) {
|
|
362
|
-
return (await readRecipe(recipe, overrides, 'request'));
|
|
277
|
+
async request(recipe, overrides, callOptions = {}) {
|
|
278
|
+
return (await readRecipe(recipe, overrides, 'request', callOptions));
|
|
279
|
+
},
|
|
280
|
+
async requestMany(recipe, calls, callOptions = {}) {
|
|
281
|
+
return (await readMany(recipe, calls, callOptions));
|
|
282
|
+
},
|
|
283
|
+
async plan(plan, callOptions = {}) {
|
|
284
|
+
if (options.seam === undefined) {
|
|
285
|
+
return allSame(plan, { code: 'invalid-request', message: '没有执行缝,读不了浏览器' });
|
|
286
|
+
}
|
|
287
|
+
const problem = planProblemOf(plan)[0];
|
|
288
|
+
if (problem !== undefined) {
|
|
289
|
+
return allSame(plan, { code: 'invalid-request', message: `${problem.path}: ${problem.problem}` });
|
|
290
|
+
}
|
|
291
|
+
if (plan.origin === undefined || plan.origin === '') {
|
|
292
|
+
return allSame(plan, { code: 'invalid-request', message: '这份计划只写了 site,没经过宿主解析' });
|
|
293
|
+
}
|
|
294
|
+
const session = sessionOf(plan.session);
|
|
295
|
+
const names = plan.steps.map((one) => one.name);
|
|
296
|
+
let script;
|
|
297
|
+
try {
|
|
298
|
+
script = scriptOfPlan(plan);
|
|
299
|
+
}
|
|
300
|
+
catch (error) {
|
|
301
|
+
const wrong = {
|
|
302
|
+
code: 'invalid-request',
|
|
303
|
+
message: error instanceof Error ? error.message : String(error),
|
|
304
|
+
};
|
|
305
|
+
return allSame(plan, wrong);
|
|
306
|
+
}
|
|
307
|
+
const got = await guardedBrowser({
|
|
308
|
+
seam: options.seam,
|
|
309
|
+
script,
|
|
310
|
+
url: plan.origin,
|
|
311
|
+
...(plan.loginUrl === undefined ? {} : { loginUrl: plan.loginUrl }),
|
|
312
|
+
session,
|
|
313
|
+
open: callOptions.open ?? false,
|
|
314
|
+
...(plan.timeoutMs === undefined ? {} : { timeoutMs: plan.timeoutMs }),
|
|
315
|
+
...(callOptions.signal === undefined ? {} : { signal: callOptions.signal }),
|
|
316
|
+
});
|
|
317
|
+
if (got.kind === 'failed')
|
|
318
|
+
return allSame(plan, got.result.error);
|
|
319
|
+
const result = got.value;
|
|
320
|
+
if (!result.ok)
|
|
321
|
+
return allSame(plan, browserFailure(result, plan.loginUrl).error);
|
|
322
|
+
const outcome = planResultsOf(result.value, names, {
|
|
323
|
+
...(plan.loginUrl === undefined ? {} : { loginUrl: plan.loginUrl }),
|
|
324
|
+
});
|
|
325
|
+
const final = { ...outcome };
|
|
326
|
+
for (const step of plan.steps) {
|
|
327
|
+
if (step.response?.saveAs !== 'attachment')
|
|
328
|
+
continue;
|
|
329
|
+
const one = outcome[step.name];
|
|
330
|
+
if (one !== undefined)
|
|
331
|
+
final[step.name] = await attachOf(step, one, options.saveBytes);
|
|
332
|
+
}
|
|
333
|
+
return final;
|
|
363
334
|
},
|
|
364
|
-
async
|
|
365
|
-
|
|
335
|
+
async submitPlan(plan, submitOptions) {
|
|
336
|
+
if (options.seam === undefined) {
|
|
337
|
+
return allSame(plan, { code: 'invalid-request', message: '没有执行缝,写不了' });
|
|
338
|
+
}
|
|
339
|
+
const run = async (script) => {
|
|
340
|
+
const got = await guardedBrowser({
|
|
341
|
+
seam: options.seam,
|
|
342
|
+
script,
|
|
343
|
+
url: plan.origin,
|
|
344
|
+
...(plan.loginUrl === undefined ? {} : { loginUrl: plan.loginUrl }),
|
|
345
|
+
...(plan.session === undefined ? {} : { session: plan.session }),
|
|
346
|
+
open: submitOptions.open ?? false,
|
|
347
|
+
...(plan.timeoutMs === undefined ? {} : { timeoutMs: plan.timeoutMs }),
|
|
348
|
+
...(submitOptions.signal === undefined ? {} : { signal: submitOptions.signal }),
|
|
349
|
+
retry: false,
|
|
350
|
+
});
|
|
351
|
+
if (got.kind === 'failed')
|
|
352
|
+
return got.result;
|
|
353
|
+
if (!got.value.ok)
|
|
354
|
+
return browserFailure(got.value, plan.loginUrl);
|
|
355
|
+
return { ok: true, value: got.value.value };
|
|
356
|
+
};
|
|
357
|
+
return submitPlanOf(plan, submitOptions, run, options.saveBytes);
|
|
366
358
|
},
|
|
367
|
-
async submit(recipe, overrides) {
|
|
368
|
-
return (await readRecipe(recipe, overrides, 'submit'));
|
|
359
|
+
async submit(recipe, overrides, callOptions = {}) {
|
|
360
|
+
return (await readRecipe(recipe, overrides, 'submit', callOptions));
|
|
369
361
|
},
|
|
370
362
|
};
|
|
371
363
|
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export interface PlanCacheEntry {
|
|
2
|
+
readonly value: unknown;
|
|
3
|
+
readonly expiresAt: number;
|
|
4
|
+
}
|
|
5
|
+
export interface PlanCache {
|
|
6
|
+
take(now: number, keyOf: (name: string) => string, names: readonly string[]): Record<string, {
|
|
7
|
+
readonly value: unknown;
|
|
8
|
+
}>;
|
|
9
|
+
put(now: number, key: string, value: unknown, ttlMs: number): void;
|
|
10
|
+
}
|
|
11
|
+
export declare function createPlanCache(): PlanCache;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
const MAX_ENTRIES = 500;
|
|
2
|
+
const SWEEP_AT = 200;
|
|
3
|
+
const MAX_ENTRY_BYTES = 8_000;
|
|
4
|
+
const MAX_TOTAL_BYTES = 24_000;
|
|
5
|
+
function bytesOf(value) {
|
|
6
|
+
const serialized = JSON.stringify(value);
|
|
7
|
+
return serialized === undefined ? undefined : Buffer.byteLength(serialized, 'utf8');
|
|
8
|
+
}
|
|
9
|
+
export function createPlanCache() {
|
|
10
|
+
const entries = new Map();
|
|
11
|
+
const sweep = (now) => {
|
|
12
|
+
if (entries.size <= SWEEP_AT)
|
|
13
|
+
return;
|
|
14
|
+
for (const [key, one] of entries) {
|
|
15
|
+
if (one.expiresAt <= now)
|
|
16
|
+
entries.delete(key);
|
|
17
|
+
}
|
|
18
|
+
for (const key of entries.keys()) {
|
|
19
|
+
if (entries.size <= MAX_ENTRIES)
|
|
20
|
+
break;
|
|
21
|
+
entries.delete(key);
|
|
22
|
+
}
|
|
23
|
+
};
|
|
24
|
+
return {
|
|
25
|
+
take(now, keyOf, names) {
|
|
26
|
+
const out = Object.create(null);
|
|
27
|
+
let used = 0;
|
|
28
|
+
for (const name of names) {
|
|
29
|
+
const key = keyOf(name);
|
|
30
|
+
const hit = entries.get(key);
|
|
31
|
+
if (hit === undefined)
|
|
32
|
+
continue;
|
|
33
|
+
if (hit.expiresAt <= now) {
|
|
34
|
+
entries.delete(key);
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
const size = bytesOf(hit.value);
|
|
38
|
+
if (size === undefined || size > MAX_ENTRY_BYTES || used + size > MAX_TOTAL_BYTES)
|
|
39
|
+
continue;
|
|
40
|
+
used += size;
|
|
41
|
+
out[name] = { value: hit.value };
|
|
42
|
+
}
|
|
43
|
+
return out;
|
|
44
|
+
},
|
|
45
|
+
put(now, key, value, ttlMs) {
|
|
46
|
+
const size = bytesOf(value);
|
|
47
|
+
if (size === undefined || size > MAX_ENTRY_BYTES)
|
|
48
|
+
return;
|
|
49
|
+
entries.set(key, { value, expiresAt: now + ttlMs });
|
|
50
|
+
sweep(now);
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export type QueueOutcome = 'acquired' | 'aborted' | 'timeout';
|
|
2
|
+
export interface QueueWaitEvent {
|
|
3
|
+
readonly session: string;
|
|
4
|
+
readonly waitedMs: number;
|
|
5
|
+
readonly outcome: QueueOutcome;
|
|
6
|
+
}
|
|
7
|
+
export interface SessionQueue {
|
|
8
|
+
acquire(session: string, signal: AbortSignal | undefined, waitMs: number): Promise<QueueOutcome>;
|
|
9
|
+
release(session: string): void;
|
|
10
|
+
}
|
|
11
|
+
export declare function createSessionQueue(): SessionQueue;
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
export function createSessionQueue() {
|
|
2
|
+
const slots = new Map();
|
|
3
|
+
return {
|
|
4
|
+
acquire(session, signal, waitMs) {
|
|
5
|
+
const slot = slots.get(session) ?? { busy: false, waiters: [] };
|
|
6
|
+
slots.set(session, slot);
|
|
7
|
+
if (!slot.busy) {
|
|
8
|
+
slot.busy = true;
|
|
9
|
+
return Promise.resolve('acquired');
|
|
10
|
+
}
|
|
11
|
+
if (signal?.aborted === true)
|
|
12
|
+
return Promise.resolve('aborted');
|
|
13
|
+
if (waitMs <= 0)
|
|
14
|
+
return Promise.resolve('timeout');
|
|
15
|
+
return new Promise((resolve) => {
|
|
16
|
+
let settled = false;
|
|
17
|
+
const finish = (outcome) => {
|
|
18
|
+
if (settled)
|
|
19
|
+
return;
|
|
20
|
+
settled = true;
|
|
21
|
+
clearTimeout(timer);
|
|
22
|
+
signal?.removeEventListener('abort', onAbort);
|
|
23
|
+
const at = slot.waiters.indexOf(waiter);
|
|
24
|
+
if (at >= 0)
|
|
25
|
+
slot.waiters.splice(at, 1);
|
|
26
|
+
if (slot.waiters.length === 0 && !slot.busy)
|
|
27
|
+
slots.delete(session);
|
|
28
|
+
resolve(outcome);
|
|
29
|
+
};
|
|
30
|
+
const onAbort = () => finish('aborted');
|
|
31
|
+
const waiter = { finish };
|
|
32
|
+
slot.waiters.push(waiter);
|
|
33
|
+
const timer = setTimeout(() => finish('timeout'), waitMs);
|
|
34
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
35
|
+
});
|
|
36
|
+
},
|
|
37
|
+
release(session) {
|
|
38
|
+
const slot = slots.get(session);
|
|
39
|
+
if (slot === undefined)
|
|
40
|
+
return;
|
|
41
|
+
const next = slot.waiters.shift();
|
|
42
|
+
if (next !== undefined) {
|
|
43
|
+
next.finish('acquired');
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
slot.busy = false;
|
|
47
|
+
slots.delete(session);
|
|
48
|
+
},
|
|
49
|
+
};
|
|
50
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { DataError, DataErrorCode, DataFailure, DataResult } from '@dsh-workbench/shared/host';
|
|
2
|
+
import type { BrowserReadResult } from '../browser/index.js';
|
|
3
|
+
export declare function failure(code: DataErrorCode, message: string, extra?: Partial<DataError>): DataFailure;
|
|
4
|
+
export declare function parse(text: string): DataResult<unknown>;
|
|
5
|
+
export declare function asRecord(value: unknown): Record<string, unknown> | undefined;
|
|
6
|
+
export declare function browserFailure(result: BrowserReadResult, loginUrl?: string): DataFailure;
|
|
7
|
+
export declare function pageFailure(pageCode: string, message: string, loginUrl?: string): DataFailure;
|
|
8
|
+
export declare function pickOf(value: unknown, path: string): unknown;
|
|
9
|
+
export declare function allFailure(names: readonly string[], code: DataErrorCode, message: string, extra?: Partial<DataError>): Record<string, DataResult<never>>;
|