@opencraw/driver 0.7.9 → 0.7.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -77
- package/package.json +4 -45
- package/dist/index.esm.js +0 -655
- package/dist/main.esm.js +0 -563
- package/dist/src/callout-bridge/client-hooks.client.d.ts +0 -47
- package/dist/src/callout-bridge/index.d.ts +0 -3
- package/dist/src/index.d.ts +0 -16
- package/dist/src/main.d.ts +0 -3
- package/dist/src/protocol/driver-json-schema.mapper.d.ts +0 -9
- package/dist/src/protocol/driver-message.contract.d.ts +0 -492
- package/dist/src/protocol/index.d.ts +0 -4
- package/dist/src/record-files/diff-record-files.use-case.d.ts +0 -10
- package/dist/src/record-files/export-record-files.use-case.d.ts +0 -11
- package/dist/src/record-files/index.d.ts +0 -3
- package/dist/src/runtime-manifest/index.d.ts +0 -4
- package/dist/src/runtime-manifest/runtime-manifest-json-schema.mapper.d.ts +0 -8
- package/dist/src/runtime-manifest/runtime-manifest.contract.d.ts +0 -28
- package/dist/src/server/index.d.ts +0 -3
- package/dist/src/server/serve-driver.use-case.d.ts +0 -17
- package/dist/src/session/index.d.ts +0 -5
- package/dist/src/session/recipe-loading.use-case.d.ts +0 -23
- package/dist/src/session/run-session.use-case.d.ts +0 -35
- package/dist/src/session/streaming-sink.repository.d.ts +0 -8
- package/dist/src/site-access/index.d.ts +0 -3
- package/dist/src/site-access/resolve-site-access.use-case.d.ts +0 -26
- package/dist/src/site-probe/index.d.ts +0 -2
- package/dist/src/site-probe/probe-site.use-case.d.ts +0 -13
- package/dist/src/transport/index.d.ts +0 -5
- package/dist/src/transport/json-rpc.contract.d.ts +0 -31
- package/dist/src/transport/line-channel.client.d.ts +0 -27
- package/dist/src/work-bridge/client-work-source.client.d.ts +0 -17
- package/dist/src/work-bridge/index.d.ts +0 -2
- package/schemas/driver-protocol.schema.json +0 -1104
- package/schemas/driver-runtime.fixture.json +0 -9
- package/schemas/driver-runtime.schema.json +0 -63
package/dist/index.esm.js
DELETED
|
@@ -1,655 +0,0 @@
|
|
|
1
|
-
// src/server/serve-driver.use-case.ts
|
|
2
|
-
import { once as once2 } from "node:events";
|
|
3
|
-
|
|
4
|
-
// src/protocol/driver-message.contract.ts
|
|
5
|
-
import { calloutRequestSchema, calloutResponseSchema } from "@opencraw/core";
|
|
6
|
-
import { z } from "zod";
|
|
7
|
-
var PROTOCOL_VERSION = "1.2.0";
|
|
8
|
-
var recipeObject = z.record(z.string(), z.unknown());
|
|
9
|
-
var coreObject = z.record(z.string(), z.unknown());
|
|
10
|
-
var accessFields = {
|
|
11
|
-
accessConfig: z.string().optional(),
|
|
12
|
-
access: z.string().optional(),
|
|
13
|
-
/** Access plugins the client serves: a profile `{ "kind": "plugin", "name": … }` naming one leases through a `callout` to the client. */
|
|
14
|
-
accessPlugins: z.array(z.string()).optional()
|
|
15
|
-
};
|
|
16
|
-
var recipeSource = {
|
|
17
|
-
paths: z.array(z.string()).min(1).optional(),
|
|
18
|
-
recipes: z.array(recipeObject).min(1).optional()
|
|
19
|
-
};
|
|
20
|
-
var initializeParamsSchema = z.strictObject({
|
|
21
|
-
clientName: z.string().min(1),
|
|
22
|
-
clientVersion: z.string().min(1),
|
|
23
|
-
protocolVersion: z.string().regex(/^\d+\.\d+\.\d+$/)
|
|
24
|
-
});
|
|
25
|
-
var initializeResultSchema = z.strictObject({
|
|
26
|
-
driverVersion: z.string(),
|
|
27
|
-
protocolVersion: z.string(),
|
|
28
|
-
methods: z.array(z.string())
|
|
29
|
-
});
|
|
30
|
-
var validateParamsSchema = z.strictObject({ ...recipeSource, only: z.array(z.string()).optional() });
|
|
31
|
-
var issueSchema = z.strictObject({ source: z.string().optional(), recipeId: z.string().optional(), path: z.string(), message: z.string() });
|
|
32
|
-
var validateResultSchema = z.strictObject({
|
|
33
|
-
ok: z.boolean(),
|
|
34
|
-
/** The input recipes that would run, when ok. */
|
|
35
|
-
inputs: z.array(z.string()),
|
|
36
|
-
/** Every hook the recipes call, which a `run` must serve. */
|
|
37
|
-
hooks: z.array(z.string()),
|
|
38
|
-
issues: z.array(issueSchema)
|
|
39
|
-
});
|
|
40
|
-
var runParamsSchema = z.strictObject({
|
|
41
|
-
...recipeSource,
|
|
42
|
-
only: z.array(z.string()).optional(),
|
|
43
|
-
/** A file to write the records to, its extension naming the format; without it, each record is sent as a `record` notification. */
|
|
44
|
-
out: z.string().optional(),
|
|
45
|
-
append: z.boolean().optional(),
|
|
46
|
-
resume: z.boolean().optional(),
|
|
47
|
-
dryRun: z.boolean().optional(),
|
|
48
|
-
parallel: z.number().int().min(1).max(16).optional(),
|
|
49
|
-
browser: z.strictObject({ headless: z.boolean().optional(), executablePath: z.string().optional(), ignoreHTTPSErrors: z.boolean().optional() }).optional(),
|
|
50
|
-
/** The hooks the client answers itself: each becomes a `callout` request to the client when a recipe calls it. */
|
|
51
|
-
hooks: z.array(z.string()).optional(),
|
|
52
|
-
/** Captcha solvers the client serves, by name: a challenge goes to the client as a `captcha` callout. */
|
|
53
|
-
captchaSolvers: z.array(z.string()).optional(),
|
|
54
|
-
...accessFields
|
|
55
|
-
});
|
|
56
|
-
var runResultSchema = z.strictObject({ runId: z.string() });
|
|
57
|
-
var windowsSchema = z.union([z.number().int().min(1).max(64), z.record(z.string(), z.unknown())]);
|
|
58
|
-
var workParamsSchema = z.strictObject({
|
|
59
|
-
...recipeSource,
|
|
60
|
-
only: z.array(z.string()).optional(),
|
|
61
|
-
out: z.string().optional(),
|
|
62
|
-
append: z.boolean().optional(),
|
|
63
|
-
browser: z.strictObject({ headless: z.boolean().optional(), executablePath: z.string().optional(), ignoreHTTPSErrors: z.boolean().optional() }).optional(),
|
|
64
|
-
hooks: z.array(z.string()).optional(),
|
|
65
|
-
captchaSolvers: z.array(z.string()).optional(),
|
|
66
|
-
...accessFields,
|
|
67
|
-
windows: windowsSchema.optional()
|
|
68
|
-
});
|
|
69
|
-
var varValue = z.union([z.string(), z.number(), z.boolean()]);
|
|
70
|
-
var workItemSchema = z.strictObject({
|
|
71
|
-
id: z.string().min(1),
|
|
72
|
-
vars: z.record(z.string(), varValue),
|
|
73
|
-
recipe: z.string().optional()
|
|
74
|
-
});
|
|
75
|
-
var cancelParamsSchema = z.strictObject({ runId: z.string() });
|
|
76
|
-
var cancelResultSchema = z.strictObject({ cancelled: z.boolean() });
|
|
77
|
-
var probeParamsSchema = z.strictObject({
|
|
78
|
-
url: z.string().min(1),
|
|
79
|
-
/** Render the page in Chromium first (for pages built by scripts). */
|
|
80
|
-
browser: z.boolean().optional(),
|
|
81
|
-
browserPath: z.string().optional(),
|
|
82
|
-
insecureTls: z.boolean().optional(),
|
|
83
|
-
userAgent: z.string().optional(),
|
|
84
|
-
...accessFields
|
|
85
|
-
});
|
|
86
|
-
var probeResultSchema = coreObject;
|
|
87
|
-
var diffParamsSchema = z.strictObject({
|
|
88
|
-
/** The earlier run's JSON Lines file. */
|
|
89
|
-
previous: z.string().min(1),
|
|
90
|
-
current: z.string().min(1),
|
|
91
|
-
/** The fields that identify a record; default: each line's `_key`. */
|
|
92
|
-
key: z.array(z.string()).optional(),
|
|
93
|
-
/** Fields left out of the comparison. */
|
|
94
|
-
ignore: z.array(z.string()).optional(),
|
|
95
|
-
/** Changes returned at most (default 200); the counts cover all of them. */
|
|
96
|
-
limit: z.number().int().min(0).optional()
|
|
97
|
-
});
|
|
98
|
-
var diffResultSchema = z.strictObject({
|
|
99
|
-
added: z.number(),
|
|
100
|
-
removed: z.number(),
|
|
101
|
-
changed: z.number(),
|
|
102
|
-
unchanged: z.number(),
|
|
103
|
-
/** Set when the current run lost most of the previous run's records. */
|
|
104
|
-
shrunk: z.strictObject({ previous: z.number(), current: z.number() }).optional(),
|
|
105
|
-
/** The report a person reads, one line per change. */
|
|
106
|
-
summary: z.array(z.string()),
|
|
107
|
-
/** The changes, up to `limit`: each `added`, `removed` or `changed` (with each field's before and after). */
|
|
108
|
-
changes: z.array(coreObject),
|
|
109
|
-
/** More changes than `limit` were found. */
|
|
110
|
-
truncated: z.boolean()
|
|
111
|
-
});
|
|
112
|
-
var exportParamsSchema = z.strictObject({
|
|
113
|
-
/** A run's JSON Lines file. */
|
|
114
|
-
records: z.string().min(1),
|
|
115
|
-
/** The output recipe that gives the columns. */
|
|
116
|
-
output: z.string().min(1),
|
|
117
|
-
/** The file to write; its extension names the format, as for `run`. */
|
|
118
|
-
out: z.string().min(1)
|
|
119
|
-
});
|
|
120
|
-
var exportResultSchema = z.strictObject({ written: z.number(), location: z.string() });
|
|
121
|
-
var shutdownResultSchema = z.strictObject({});
|
|
122
|
-
var workNextParamsSchema = z.strictObject({ runId: z.string() });
|
|
123
|
-
var workNextResultSchema = z.strictObject({ item: workItemSchema.nullable() });
|
|
124
|
-
var workRecordSchema = z.strictObject({ key: z.string().nullable(), data: z.record(z.string(), z.unknown()) });
|
|
125
|
-
var workDoneParamsSchema = z.strictObject({ runId: z.string(), item: workItemSchema, report: coreObject, records: z.array(workRecordSchema) });
|
|
126
|
-
var workFailedParamsSchema = z.strictObject({ runId: z.string(), item: workItemSchema, report: coreObject, outcome: z.enum(["failure", "neutral"]) });
|
|
127
|
-
var eventParamsSchema = z.strictObject({ runId: z.string(), event: z.record(z.string(), z.unknown()) });
|
|
128
|
-
var recordParamsSchema = z.strictObject({ runId: z.string(), recipeId: z.string(), key: z.string().nullable(), data: z.record(z.string(), z.unknown()) });
|
|
129
|
-
var finishedParamsSchema = z.strictObject({
|
|
130
|
-
runId: z.string(),
|
|
131
|
-
/** The crawl report (`CrawlReport`), or for `work` the work report (`WorkReport`); absent when the run could not start. */
|
|
132
|
-
report: z.record(z.string(), z.unknown()).optional(),
|
|
133
|
-
/** Why the run could not start, or stopped as a whole. */
|
|
134
|
-
error: z.string().optional()
|
|
135
|
-
});
|
|
136
|
-
var calloutParamsSchema = z.strictObject({ runId: z.string(), request: calloutRequestSchema });
|
|
137
|
-
var calloutResultSchema = calloutResponseSchema;
|
|
138
|
-
var MESSAGES = {
|
|
139
|
-
"initialize": { direction: "client-to-driver", params: initializeParamsSchema, result: initializeResultSchema },
|
|
140
|
-
"validate": { direction: "client-to-driver", params: validateParamsSchema, result: validateResultSchema },
|
|
141
|
-
"run": { direction: "client-to-driver", params: runParamsSchema, result: runResultSchema },
|
|
142
|
-
"work": { direction: "client-to-driver", params: workParamsSchema, result: runResultSchema },
|
|
143
|
-
"cancel": { direction: "client-to-driver", params: cancelParamsSchema, result: cancelResultSchema },
|
|
144
|
-
"probe": { direction: "client-to-driver", params: probeParamsSchema, result: probeResultSchema },
|
|
145
|
-
"diff": { direction: "client-to-driver", params: diffParamsSchema, result: diffResultSchema },
|
|
146
|
-
"export": { direction: "client-to-driver", params: exportParamsSchema, result: exportResultSchema },
|
|
147
|
-
"shutdown": { direction: "client-to-driver", params: void 0, result: shutdownResultSchema },
|
|
148
|
-
"callout": { direction: "driver-to-client", params: calloutParamsSchema, result: calloutResultSchema },
|
|
149
|
-
"work.next": { direction: "driver-to-client", params: workNextParamsSchema, result: workNextResultSchema },
|
|
150
|
-
"work.done": { direction: "driver-to-client", params: workDoneParamsSchema, result: void 0 },
|
|
151
|
-
"work.failed": { direction: "driver-to-client", params: workFailedParamsSchema, result: void 0 },
|
|
152
|
-
"event": { direction: "driver-to-client", params: eventParamsSchema, result: void 0 },
|
|
153
|
-
"record": { direction: "driver-to-client", params: recordParamsSchema, result: void 0 },
|
|
154
|
-
"finished": { direction: "driver-to-client", params: finishedParamsSchema, result: void 0 }
|
|
155
|
-
};
|
|
156
|
-
|
|
157
|
-
// src/protocol/driver-json-schema.mapper.ts
|
|
158
|
-
import { z as z2 } from "zod";
|
|
159
|
-
var OPTIONS = { target: "draft-2020-12", io: "output", unrepresentable: "any" };
|
|
160
|
-
function driverProtocolJsonSchema() {
|
|
161
|
-
const defs = {};
|
|
162
|
-
const methods = {};
|
|
163
|
-
for (const [name, message] of Object.entries(MESSAGES)) {
|
|
164
|
-
const entry = { direction: message.direction };
|
|
165
|
-
if (message.params !== void 0) {
|
|
166
|
-
defs[`${name}.params`] = z2.toJSONSchema(message.params, OPTIONS);
|
|
167
|
-
entry.params = { $ref: `#/$defs/${name}.params` };
|
|
168
|
-
}
|
|
169
|
-
if (message.result !== void 0) {
|
|
170
|
-
defs[`${name}.result`] = z2.toJSONSchema(message.result, OPTIONS);
|
|
171
|
-
entry.result = { $ref: `#/$defs/${name}.result` };
|
|
172
|
-
}
|
|
173
|
-
methods[name] = entry;
|
|
174
|
-
}
|
|
175
|
-
return {
|
|
176
|
-
$schema: "https://json-schema.org/draft/2020-12/schema",
|
|
177
|
-
$id: "https://opencraw/schemas/driver-protocol.schema.json",
|
|
178
|
-
title: "OpenCraw driver protocol",
|
|
179
|
-
description: "JSON-RPC 2.0 over stdio, one message per line. Methods with a result are requests; the rest are notifications.",
|
|
180
|
-
protocolVersion: PROTOCOL_VERSION,
|
|
181
|
-
methods,
|
|
182
|
-
$defs: defs
|
|
183
|
-
};
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
// src/record-files/diff-record-files.use-case.ts
|
|
187
|
-
import { diffReport } from "@opencraw/cli";
|
|
188
|
-
import { diffRecords, readRecordsFile } from "@opencraw/core";
|
|
189
|
-
var CHANGE_LIMIT = 200;
|
|
190
|
-
async function diffRecordFiles(params) {
|
|
191
|
-
const [previous, current] = await Promise.all([readRecordsFile(params.previous), readRecordsFile(params.current)]);
|
|
192
|
-
const diff = diffRecords(previous, current, { key: params.key, ignore: params.ignore });
|
|
193
|
-
const limit = params.limit ?? CHANGE_LIMIT;
|
|
194
|
-
return {
|
|
195
|
-
added: diff.added,
|
|
196
|
-
removed: diff.removed,
|
|
197
|
-
changed: diff.changed,
|
|
198
|
-
unchanged: diff.unchanged,
|
|
199
|
-
...diff.shrunk !== void 0 && { shrunk: diff.shrunk },
|
|
200
|
-
summary: diffReport(diff),
|
|
201
|
-
changes: diff.changes.slice(0, limit).map((change) => ({ ...change })),
|
|
202
|
-
truncated: diff.changes.length > limit
|
|
203
|
-
};
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
// src/record-files/export-record-files.use-case.ts
|
|
207
|
-
import { readFile } from "node:fs/promises";
|
|
208
|
-
import { fileSink, outputRecordOf, parseOutputRecipe, readRecordsFile as readRecordsFile2 } from "@opencraw/core";
|
|
209
|
-
async function exportRecordFiles(params) {
|
|
210
|
-
const output = parseOutputRecipe(JSON.parse(await readFile(params.output, "utf8")), params.output);
|
|
211
|
-
const stored = await readRecordsFile2(params.records);
|
|
212
|
-
const sink = fileSink(params.out, { browser: { executablePath: process.env.OPENCRAW_CHROMIUM } });
|
|
213
|
-
await sink.open(output);
|
|
214
|
-
for (const line of stored) await sink.write(outputRecordOf(line));
|
|
215
|
-
const summary = await sink.close();
|
|
216
|
-
return { written: summary.written, location: summary.location ?? params.out };
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
// src/session/run-session.use-case.ts
|
|
220
|
-
import { createCrawler, fileSink as fileSink2, RecipeSet } from "@opencraw/core";
|
|
221
|
-
|
|
222
|
-
// src/callout-bridge/client-hooks.client.ts
|
|
223
|
-
import { accessPluginVia, calloutHook, calloutResponseSchema as calloutResponseSchema2, captchaSolverVia } from "@opencraw/core";
|
|
224
|
-
function clientTransport(label, runId, ask) {
|
|
225
|
-
return {
|
|
226
|
-
label,
|
|
227
|
-
call: async (request) => calloutResponseSchema2.parse(await ask("callout", { runId, request }))
|
|
228
|
-
};
|
|
229
|
-
}
|
|
230
|
-
function clientHooks(names, runId, ask) {
|
|
231
|
-
return Object.fromEntries(names.map((name) => [name, calloutHook(name, clientTransport(`client hook ${name}`, runId, ask))]));
|
|
232
|
-
}
|
|
233
|
-
function clientCaptchaSolvers(names, runId, ask) {
|
|
234
|
-
return names.map((name) => captchaSolverVia(name, clientTransport(`client captcha solver ${name}`, runId, ask)));
|
|
235
|
-
}
|
|
236
|
-
function clientAccessPlugins(names, runId, ask) {
|
|
237
|
-
return names.map((name) => accessPluginVia(name, clientTransport(`client access plugin ${name}`, runId, ask)));
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
// src/site-access/resolve-site-access.use-case.ts
|
|
241
|
-
import { resolveAccess } from "@opencraw/cli";
|
|
242
|
-
async function resolveSiteAccess(params, runId, ask, env = process.env) {
|
|
243
|
-
const file = params.accessConfig ?? (env.OPENCRAW_ACCESS === "" ? void 0 : env.OPENCRAW_ACCESS);
|
|
244
|
-
if (file === void 0 && params.access !== void 0) throw new Error(`access profile "${params.access}" needs an access config, and there is none: no "accessConfig" was given, and OPENCRAW_ACCESS is not set where OpenCraw runs`);
|
|
245
|
-
const access = await resolveAccess({ insecureTls: false, access: file, accessProfile: params.access });
|
|
246
|
-
return { access, accessPlugins: clientAccessPlugins(params.accessPlugins ?? [], runId, ask) };
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
// src/work-bridge/client-work-source.client.ts
|
|
250
|
-
import { once } from "node:events";
|
|
251
|
-
function clientWorkSource(runId, ask, notify) {
|
|
252
|
-
const kept = [];
|
|
253
|
-
let ended = false;
|
|
254
|
-
const askClient = async () => {
|
|
255
|
-
const { item } = workNextResultSchema.parse(await ask("work.next", { runId }));
|
|
256
|
-
if (item === null) {
|
|
257
|
-
ended = true;
|
|
258
|
-
return void 0;
|
|
259
|
-
}
|
|
260
|
-
return item;
|
|
261
|
-
};
|
|
262
|
-
const keepLate = async (answer) => {
|
|
263
|
-
try {
|
|
264
|
-
const item = await answer;
|
|
265
|
-
if (item !== void 0) kept.push(item);
|
|
266
|
-
} catch {
|
|
267
|
-
}
|
|
268
|
-
};
|
|
269
|
-
return {
|
|
270
|
-
next: async (options) => {
|
|
271
|
-
const waiting = kept.shift();
|
|
272
|
-
if (waiting !== void 0) return waiting;
|
|
273
|
-
if (ended) return;
|
|
274
|
-
const answer = askClient();
|
|
275
|
-
const signal = options?.signal;
|
|
276
|
-
if (signal === void 0) return await answer;
|
|
277
|
-
signal.throwIfAborted();
|
|
278
|
-
const settled = new AbortController();
|
|
279
|
-
const answered = (async () => {
|
|
280
|
-
const item = await answer;
|
|
281
|
-
return { item };
|
|
282
|
-
})();
|
|
283
|
-
const retired = (async () => {
|
|
284
|
-
try {
|
|
285
|
-
await once(signal, "abort", { signal: settled.signal });
|
|
286
|
-
return "retired";
|
|
287
|
-
} catch {
|
|
288
|
-
return "answered";
|
|
289
|
-
}
|
|
290
|
-
})();
|
|
291
|
-
try {
|
|
292
|
-
const first = await Promise.race([answered, retired]);
|
|
293
|
-
if (first !== "retired" && first !== "answered") return first.item;
|
|
294
|
-
void keepLate(answer);
|
|
295
|
-
throw signal.reason instanceof Error ? signal.reason : new Error(String(signal.reason));
|
|
296
|
-
} finally {
|
|
297
|
-
settled.abort();
|
|
298
|
-
}
|
|
299
|
-
},
|
|
300
|
-
done: (item, report, records) => {
|
|
301
|
-
notify("work.done", { runId, item, report, records: records.map((record) => ({ key: record.key, data: record.data })) });
|
|
302
|
-
},
|
|
303
|
-
failed: (item, report, outcome) => {
|
|
304
|
-
notify("work.failed", { runId, item, report, outcome });
|
|
305
|
-
}
|
|
306
|
-
};
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
// src/session/recipe-loading.use-case.ts
|
|
310
|
-
import { loadForRun } from "@opencraw/cli";
|
|
311
|
-
import { hookUses, RecipeBindingError, RecipeValidationError } from "@opencraw/core";
|
|
312
|
-
function sourceOf(params) {
|
|
313
|
-
if (params.paths === void 0 === (params.recipes === void 0)) throw new Error('give exactly one of "paths" or "recipes"');
|
|
314
|
-
return params.paths ?? params.recipes ?? [];
|
|
315
|
-
}
|
|
316
|
-
async function loadRecipes(params) {
|
|
317
|
-
return await loadForRun(sourceOf(params), params.only ?? []);
|
|
318
|
-
}
|
|
319
|
-
function hooksOf(set) {
|
|
320
|
-
return [...new Set(set.inputs.flatMap((input) => hookUses(input).map((use) => use.name)))].sort((x, y) => x.localeCompare(y));
|
|
321
|
-
}
|
|
322
|
-
async function validateRecipes(params) {
|
|
323
|
-
try {
|
|
324
|
-
const set = await loadRecipes(params);
|
|
325
|
-
return { ok: true, inputs: set.inputs.map((input) => input.id), hooks: hooksOf(set), issues: [] };
|
|
326
|
-
} catch (error) {
|
|
327
|
-
if (error instanceof RecipeValidationError) return { ok: false, inputs: [], hooks: [], issues: error.issues.map((issue) => ({ source: error.source, path: issue.path, message: issue.message })) };
|
|
328
|
-
if (error instanceof RecipeBindingError) return { ok: false, inputs: [], hooks: [], issues: error.issues.map((issue) => ({ recipeId: issue.recipeId, path: issue.path, message: issue.message })) };
|
|
329
|
-
return { ok: false, inputs: [], hooks: [], issues: [{ path: "", message: error instanceof Error ? error.message : String(error) }] };
|
|
330
|
-
}
|
|
331
|
-
}
|
|
332
|
-
|
|
333
|
-
// src/session/streaming-sink.repository.ts
|
|
334
|
-
function streamingSink() {
|
|
335
|
-
let written = 0;
|
|
336
|
-
return {
|
|
337
|
-
open: async () => {
|
|
338
|
-
},
|
|
339
|
-
write: async () => {
|
|
340
|
-
written += 1;
|
|
341
|
-
},
|
|
342
|
-
close: async () => ({ written })
|
|
343
|
-
};
|
|
344
|
-
}
|
|
345
|
-
|
|
346
|
-
// src/session/run-session.use-case.ts
|
|
347
|
-
function runSessions(notify, ask, local = {}) {
|
|
348
|
-
const running = /* @__PURE__ */ new Map();
|
|
349
|
-
let next = 1;
|
|
350
|
-
const crawlerFor = async (runId, params) => {
|
|
351
|
-
const { access, accessPlugins } = await resolveSiteAccess(params, runId, ask);
|
|
352
|
-
return createCrawler({
|
|
353
|
-
sink: params.out === void 0 ? streamingSink() : fileSink2(params.out, { append: params.append, browser: { executablePath: params.browser?.executablePath } }),
|
|
354
|
-
hooks: { ...local.hooks, ...clientHooks(params.hooks ?? [], runId, ask) },
|
|
355
|
-
captchaSolvers: [...local.captchaSolvers ?? [], ...clientCaptchaSolvers(params.captchaSolvers ?? [], runId, ask)],
|
|
356
|
-
access,
|
|
357
|
-
accessPlugins: [...local.accessPlugins ?? [], ...accessPlugins],
|
|
358
|
-
profilesDir: process.env.OPENCRAW_PROFILES,
|
|
359
|
-
parallel: params.parallel,
|
|
360
|
-
resume: params.resume,
|
|
361
|
-
debug: params.dryRun,
|
|
362
|
-
browser: { headless: params.browser?.headless ?? true, executablePath: params.browser?.executablePath, ignoreHTTPSErrors: params.browser?.ignoreHTTPSErrors },
|
|
363
|
-
onEvent: (event) => {
|
|
364
|
-
notify("event", { runId, event });
|
|
365
|
-
if (event.type === "record:emit" && params.out === void 0) notify("record", { runId, recipeId: event.recipeId, key: event.key, data: event.data });
|
|
366
|
-
}
|
|
367
|
-
});
|
|
368
|
-
};
|
|
369
|
-
const track = (runId, crawler, crawling) => {
|
|
370
|
-
running.set(runId, crawler);
|
|
371
|
-
void (async () => {
|
|
372
|
-
await new Promise((resolve) => setImmediate(resolve));
|
|
373
|
-
try {
|
|
374
|
-
notify("finished", { runId, report: await crawling() });
|
|
375
|
-
} catch (error) {
|
|
376
|
-
notify("finished", { runId, error: error instanceof Error ? error.message : String(error) });
|
|
377
|
-
} finally {
|
|
378
|
-
running.delete(runId);
|
|
379
|
-
await crawler.close();
|
|
380
|
-
}
|
|
381
|
-
})();
|
|
382
|
-
};
|
|
383
|
-
const startLoaded = async (loaded, params) => {
|
|
384
|
-
const set = params.dryRun === true ? new RecipeSet(loaded.output, loaded.inputs.map((input) => ({ ...input, limits: { ...input.limits, maxRecords: 1 } }))) : loaded;
|
|
385
|
-
const runId = `run-${next++}`;
|
|
386
|
-
const crawler = await crawlerFor(runId, params);
|
|
387
|
-
track(runId, crawler, async () => await crawler.run(set));
|
|
388
|
-
return runId;
|
|
389
|
-
};
|
|
390
|
-
return {
|
|
391
|
-
start: async (params) => await startLoaded(await loadRecipes(params), params),
|
|
392
|
-
startLoaded,
|
|
393
|
-
startWork: async (params) => {
|
|
394
|
-
const set = await loadRecipes(params);
|
|
395
|
-
const runId = `run-${next++}`;
|
|
396
|
-
const crawler = await crawlerFor(runId, params);
|
|
397
|
-
const windows = params.windows;
|
|
398
|
-
track(runId, crawler, async () => await crawler.work(set, clientWorkSource(runId, ask, notify), { windows }));
|
|
399
|
-
return runId;
|
|
400
|
-
},
|
|
401
|
-
cancel: async (runId) => {
|
|
402
|
-
const crawler = running.get(runId);
|
|
403
|
-
if (crawler === void 0) return false;
|
|
404
|
-
running.delete(runId);
|
|
405
|
-
await crawler.close();
|
|
406
|
-
return true;
|
|
407
|
-
},
|
|
408
|
-
closeAll: async () => {
|
|
409
|
-
const crawlers = running.values().toArray();
|
|
410
|
-
running.clear();
|
|
411
|
-
await Promise.all(crawlers.map(async (crawler) => {
|
|
412
|
-
await crawler.close();
|
|
413
|
-
}));
|
|
414
|
-
}
|
|
415
|
-
};
|
|
416
|
-
}
|
|
417
|
-
|
|
418
|
-
// src/site-probe/probe-site.use-case.ts
|
|
419
|
-
import { probeUrl } from "@opencraw/cli";
|
|
420
|
-
async function probeSite(params, probeId, ask) {
|
|
421
|
-
const { access, accessPlugins } = await resolveSiteAccess(params, probeId, ask);
|
|
422
|
-
const result = await probeUrl(params.url, {
|
|
423
|
-
browser: params.browser ?? false,
|
|
424
|
-
browserPath: params.browserPath ?? process.env.OPENCRAW_CHROMIUM,
|
|
425
|
-
insecureTls: params.insecureTls ?? false,
|
|
426
|
-
userAgent: params.userAgent,
|
|
427
|
-
accessConfig: access,
|
|
428
|
-
accessPlugins
|
|
429
|
-
});
|
|
430
|
-
return { ...result };
|
|
431
|
-
}
|
|
432
|
-
|
|
433
|
-
// src/transport/line-channel.client.ts
|
|
434
|
-
import { createInterface } from "node:readline";
|
|
435
|
-
|
|
436
|
-
// src/transport/json-rpc.contract.ts
|
|
437
|
-
var RPC_ERRORS = {
|
|
438
|
-
parse: -32700,
|
|
439
|
-
invalidRequest: -32600,
|
|
440
|
-
methodNotFound: -32601,
|
|
441
|
-
invalidParams: -32602,
|
|
442
|
-
internal: -32603,
|
|
443
|
-
/** A failure the caller must act on: the message says what to do. */
|
|
444
|
-
server: -32e3
|
|
445
|
-
};
|
|
446
|
-
var RpcFailure = class extends Error {
|
|
447
|
-
code;
|
|
448
|
-
data;
|
|
449
|
-
constructor(code, message, data) {
|
|
450
|
-
super(message);
|
|
451
|
-
this.name = "RpcFailure";
|
|
452
|
-
this.code = code;
|
|
453
|
-
this.data = data;
|
|
454
|
-
}
|
|
455
|
-
};
|
|
456
|
-
|
|
457
|
-
// src/transport/line-channel.client.ts
|
|
458
|
-
var isRequestId = (id) => typeof id === "number" || typeof id === "string";
|
|
459
|
-
function openLineChannel(input, output, handle) {
|
|
460
|
-
const pending = /* @__PURE__ */ new Map();
|
|
461
|
-
const answering = /* @__PURE__ */ new Set();
|
|
462
|
-
const track = (answered) => {
|
|
463
|
-
answering.add(answered);
|
|
464
|
-
void (async () => {
|
|
465
|
-
try {
|
|
466
|
-
await answered;
|
|
467
|
-
} finally {
|
|
468
|
-
answering.delete(answered);
|
|
469
|
-
}
|
|
470
|
-
})();
|
|
471
|
-
};
|
|
472
|
-
let nextId = 1;
|
|
473
|
-
const write = (message) => {
|
|
474
|
-
output.write(`${JSON.stringify(message)}
|
|
475
|
-
`);
|
|
476
|
-
};
|
|
477
|
-
const fail = (id, error) => {
|
|
478
|
-
write({ jsonrpc: "2.0", id, error });
|
|
479
|
-
};
|
|
480
|
-
const answer = async (message) => {
|
|
481
|
-
const method = message.method;
|
|
482
|
-
try {
|
|
483
|
-
const result = await handle(method, message.params);
|
|
484
|
-
if (isRequestId(message.id)) write({ jsonrpc: "2.0", id: message.id, result: result ?? null });
|
|
485
|
-
} catch (error) {
|
|
486
|
-
if (!isRequestId(message.id)) return;
|
|
487
|
-
if (error instanceof RpcFailure) fail(message.id, { code: error.code, message: error.message, ...error.data !== void 0 && { data: error.data } });
|
|
488
|
-
else fail(message.id, { code: RPC_ERRORS.server, message: error instanceof Error ? error.message : String(error) });
|
|
489
|
-
}
|
|
490
|
-
};
|
|
491
|
-
const settle = (message) => {
|
|
492
|
-
const waiting = typeof message.id === "number" ? pending.get(message.id) : void 0;
|
|
493
|
-
if (waiting === void 0) return;
|
|
494
|
-
pending.delete(message.id);
|
|
495
|
-
if (message.error === void 0) waiting.resolve(message.result);
|
|
496
|
-
else waiting.reject(new RpcFailure(message.error.code, message.error.message, message.error.data));
|
|
497
|
-
};
|
|
498
|
-
const lines = createInterface({ input, crlfDelay: Infinity });
|
|
499
|
-
lines.on("line", (line) => {
|
|
500
|
-
if (line.trim() === "") return;
|
|
501
|
-
let message;
|
|
502
|
-
try {
|
|
503
|
-
message = JSON.parse(line);
|
|
504
|
-
} catch {
|
|
505
|
-
fail(null, { code: RPC_ERRORS.parse, message: "the line is not JSON" });
|
|
506
|
-
return;
|
|
507
|
-
}
|
|
508
|
-
if (message === null || typeof message !== "object" || message.jsonrpc !== "2.0") {
|
|
509
|
-
fail(isRequestId(message?.id) ? message.id : null, { code: RPC_ERRORS.invalidRequest, message: "not a JSON-RPC 2.0 message" });
|
|
510
|
-
return;
|
|
511
|
-
}
|
|
512
|
-
if (typeof message.method === "string") track(answer(message));
|
|
513
|
-
else settle(message);
|
|
514
|
-
});
|
|
515
|
-
let ended = false;
|
|
516
|
-
const closed = new Promise((resolve) => {
|
|
517
|
-
lines.on("close", () => {
|
|
518
|
-
ended = true;
|
|
519
|
-
for (const waiting of pending.values()) waiting.reject(new Error("the client closed the channel"));
|
|
520
|
-
pending.clear();
|
|
521
|
-
resolve();
|
|
522
|
-
});
|
|
523
|
-
});
|
|
524
|
-
return {
|
|
525
|
-
request: async (method, params) => await new Promise((resolve, reject) => {
|
|
526
|
-
if (ended) {
|
|
527
|
-
reject(new Error("the channel is closed"));
|
|
528
|
-
return;
|
|
529
|
-
}
|
|
530
|
-
const id = nextId++;
|
|
531
|
-
pending.set(id, { resolve, reject });
|
|
532
|
-
write({ jsonrpc: "2.0", id, method, params });
|
|
533
|
-
}),
|
|
534
|
-
notify: (method, params) => {
|
|
535
|
-
write({ jsonrpc: "2.0", method, params });
|
|
536
|
-
},
|
|
537
|
-
closed,
|
|
538
|
-
drained: async () => {
|
|
539
|
-
while (answering.size > 0) await Promise.allSettled(answering);
|
|
540
|
-
}
|
|
541
|
-
};
|
|
542
|
-
}
|
|
543
|
-
|
|
544
|
-
// src/server/serve-driver.use-case.ts
|
|
545
|
-
var METHODS = ["initialize", "validate", "run", "work", "cancel", "probe", "diff", "export", "shutdown"];
|
|
546
|
-
var major = (version) => version.split(".", 1)[0];
|
|
547
|
-
function parsed(schema, params) {
|
|
548
|
-
const result = schema.safeParse(params ?? {});
|
|
549
|
-
if (!result.success) throw new RpcFailure(RPC_ERRORS.invalidParams, result.error.issues.map((issue) => `${issue.path.join(".") || "(params)"}: ${issue.message}`).join("; "), result.error.issues);
|
|
550
|
-
return result.data;
|
|
551
|
-
}
|
|
552
|
-
async function serveDriver(input, output, identity) {
|
|
553
|
-
let initialized = false;
|
|
554
|
-
const stopping = new AbortController();
|
|
555
|
-
let probes = 0;
|
|
556
|
-
const channel = openLineChannel(input, output, async (method, params) => {
|
|
557
|
-
if (method === "initialize") {
|
|
558
|
-
const hello = parsed(initializeParamsSchema, params);
|
|
559
|
-
if (major(hello.protocolVersion) !== major(PROTOCOL_VERSION)) throw new RpcFailure(RPC_ERRORS.server, `this driver speaks protocol ${PROTOCOL_VERSION}; ${hello.clientName} ${hello.clientVersion} asked for ${hello.protocolVersion}: use a client for protocol ${major(PROTOCOL_VERSION)}.x`);
|
|
560
|
-
initialized = true;
|
|
561
|
-
const result = { driverVersion: identity.driverVersion, protocolVersion: PROTOCOL_VERSION, methods: METHODS };
|
|
562
|
-
return result;
|
|
563
|
-
}
|
|
564
|
-
if (!METHODS.includes(method)) throw new RpcFailure(RPC_ERRORS.methodNotFound, `no method "${method}": the driver has ${METHODS.join(", ")}`);
|
|
565
|
-
if (!initialized) throw new RpcFailure(RPC_ERRORS.server, "call initialize first");
|
|
566
|
-
if (method === "validate") return await validateRecipes(parsed(validateParamsSchema, params));
|
|
567
|
-
if (method === "run") return { runId: await sessions.start(parsed(runParamsSchema, params)) };
|
|
568
|
-
if (method === "work") return { runId: await sessions.startWork(parsed(workParamsSchema, params)) };
|
|
569
|
-
if (method === "cancel") return { cancelled: await sessions.cancel(parsed(cancelParamsSchema, params).runId) };
|
|
570
|
-
if (method === "probe") return await probeSite(parsed(probeParamsSchema, params), `probe-${++probes}`, channel.request);
|
|
571
|
-
if (method === "diff") return await diffRecordFiles(parsed(diffParamsSchema, params));
|
|
572
|
-
if (method === "export") return await exportRecordFiles(parsed(exportParamsSchema, params));
|
|
573
|
-
await sessions.closeAll();
|
|
574
|
-
setImmediate(() => {
|
|
575
|
-
stopping.abort();
|
|
576
|
-
});
|
|
577
|
-
return {};
|
|
578
|
-
});
|
|
579
|
-
const sessions = runSessions(channel.notify, channel.request);
|
|
580
|
-
await Promise.race([channel.closed, once2(stopping.signal, "abort")]);
|
|
581
|
-
await channel.drained();
|
|
582
|
-
await sessions.closeAll();
|
|
583
|
-
}
|
|
584
|
-
|
|
585
|
-
// src/runtime-manifest/runtime-manifest.contract.ts
|
|
586
|
-
import { z as z3 } from "zod";
|
|
587
|
-
var RUNTIME_PLATFORMS = ["win32-x64", "darwin-arm64", "darwin-x64", "linux-x64", "linux-arm64"];
|
|
588
|
-
var archiveSchema = z3.strictObject({
|
|
589
|
-
/** The archive's file name, an asset of the same GitHub release as the manifest. */
|
|
590
|
-
file: z3.string().regex(/^opencraw-driver-[\w.-]+\.tar\.gz$/),
|
|
591
|
-
/** Its SHA-256, lowercase hex: a client refuses an archive that does not match. */
|
|
592
|
-
sha256: z3.string().regex(/^[0-9a-f]{64}$/),
|
|
593
|
-
size: z3.number().int().positive()
|
|
594
|
-
});
|
|
595
|
-
var runtimeManifestSchema = z3.strictObject({
|
|
596
|
-
driverVersion: z3.string().min(1),
|
|
597
|
-
protocolVersion: z3.string().regex(/^\d+\.\d+\.\d+$/),
|
|
598
|
-
/** The Node version every archive carries. */
|
|
599
|
-
node: z3.string().regex(/^v\d+\.\d+\.\d+$/),
|
|
600
|
-
archives: z3.partialRecord(z3.enum(RUNTIME_PLATFORMS), archiveSchema)
|
|
601
|
-
});
|
|
602
|
-
|
|
603
|
-
// src/runtime-manifest/runtime-manifest-json-schema.mapper.ts
|
|
604
|
-
import { z as z4 } from "zod";
|
|
605
|
-
function runtimeManifestJsonSchema() {
|
|
606
|
-
return { $id: "https://opencraw.dev/schemas/driver-runtime.schema.json", ...z4.toJSONSchema(runtimeManifestSchema, { target: "draft-2020-12" }) };
|
|
607
|
-
}
|
|
608
|
-
export {
|
|
609
|
-
MESSAGES,
|
|
610
|
-
PROTOCOL_VERSION,
|
|
611
|
-
RPC_ERRORS,
|
|
612
|
-
RUNTIME_PLATFORMS,
|
|
613
|
-
RpcFailure,
|
|
614
|
-
calloutParamsSchema,
|
|
615
|
-
calloutResultSchema,
|
|
616
|
-
cancelParamsSchema,
|
|
617
|
-
cancelResultSchema,
|
|
618
|
-
clientAccessPlugins,
|
|
619
|
-
clientCaptchaSolvers,
|
|
620
|
-
clientHooks,
|
|
621
|
-
clientWorkSource,
|
|
622
|
-
diffParamsSchema,
|
|
623
|
-
diffRecordFiles,
|
|
624
|
-
diffResultSchema,
|
|
625
|
-
driverProtocolJsonSchema,
|
|
626
|
-
eventParamsSchema,
|
|
627
|
-
exportParamsSchema,
|
|
628
|
-
exportRecordFiles,
|
|
629
|
-
exportResultSchema,
|
|
630
|
-
finishedParamsSchema,
|
|
631
|
-
initializeParamsSchema,
|
|
632
|
-
initializeResultSchema,
|
|
633
|
-
openLineChannel,
|
|
634
|
-
probeParamsSchema,
|
|
635
|
-
probeResultSchema,
|
|
636
|
-
probeSite,
|
|
637
|
-
recordParamsSchema,
|
|
638
|
-
runParamsSchema,
|
|
639
|
-
runResultSchema,
|
|
640
|
-
runSessions,
|
|
641
|
-
runtimeManifestJsonSchema,
|
|
642
|
-
runtimeManifestSchema,
|
|
643
|
-
serveDriver,
|
|
644
|
-
streamingSink,
|
|
645
|
-
validateParamsSchema,
|
|
646
|
-
validateRecipes,
|
|
647
|
-
validateResultSchema,
|
|
648
|
-
workDoneParamsSchema,
|
|
649
|
-
workFailedParamsSchema,
|
|
650
|
-
workItemSchema,
|
|
651
|
-
workNextParamsSchema,
|
|
652
|
-
workNextResultSchema,
|
|
653
|
-
workParamsSchema
|
|
654
|
-
};
|
|
655
|
-
//# sourceMappingURL=index.esm.js.map
|