@opencraw/cli 0.1.53 → 0.1.55
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access/index.d.ts +2 -0
- package/dist/access/resolve-access.use-case.d.ts +12 -0
- package/dist/arguments/command.contract.d.ts +86 -0
- package/dist/arguments/index.d.ts +3 -0
- package/dist/arguments/parse-arguments.algorithm.d.ts +12 -0
- package/dist/diff/diff-files.use-case.d.ts +22 -0
- package/dist/diff/diff-report.mapper.d.ts +12 -0
- package/dist/diff/index.d.ts +3 -0
- package/dist/export/export-records.use-case.d.ts +14 -0
- package/dist/export/index.d.ts +2 -0
- package/dist/hooks-module/index.d.ts +3 -0
- package/dist/hooks-module/load-hooks.repository.d.ts +47 -0
- package/dist/index.d.ts +18 -1
- package/dist/index.esm.js +354 -740
- package/dist/main.d.ts +12 -0
- package/dist/probe/index.d.ts +6 -0
- package/dist/probe/probe-page.use-case.d.ts +65 -0
- package/dist/probe/probe-report.mapper.d.ts +59 -0
- package/dist/recipe-files/index.d.ts +3 -0
- package/dist/recipe-files/read-recipe-files.repository.d.ts +26 -0
- package/dist/run/index.d.ts +3 -0
- package/dist/run/load-for-run.use-case.d.ts +13 -0
- package/dist/run/run-recipes.use-case.d.ts +17 -0
- package/dist/studio/index.d.ts +3 -0
- package/dist/studio/run-studio.use-case.d.ts +32 -0
- package/dist/terminal/index.d.ts +3 -0
- package/dist/terminal/terminal.client.d.ts +4 -0
- package/dist/terminal/terminal.contract.d.ts +6 -0
- package/dist/validate/index.d.ts +2 -0
- package/dist/validate/validate-recipes.use-case.d.ts +10 -0
- package/package.json +10 -38
package/dist/index.esm.js
CHANGED
|
@@ -1,13 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
import { resolve } from 'node:path';
|
|
5
|
-
import { pathToFileURL } from 'node:url';
|
|
6
|
-
import { describePdf, findData, describeWorkbook, describeDeck, describeJson, describeXml, describeHtml } from '@opencraw/probe';
|
|
7
|
-
export { findData } from '@opencraw/probe';
|
|
8
|
-
import { readFileSync } from 'node:fs';
|
|
9
|
-
|
|
10
|
-
const USAGE = `opencraw <command> [options]
|
|
1
|
+
// src/arguments/parse-arguments.algorithm.ts
|
|
2
|
+
import { parseArgs } from "node:util";
|
|
3
|
+
var USAGE = `opencraw <command> [options]
|
|
11
4
|
|
|
12
5
|
Commands
|
|
13
6
|
validate <recipe files or directories...> Parse and bind the recipes; list every problem with its path.
|
|
@@ -65,437 +58,235 @@ Options for both
|
|
|
65
58
|
(for session.captcha and captcha steps); --hooks, or OPENCRAW_PLUGINS /
|
|
66
59
|
OPENCRAW_HOOKS. It runs as your code: trust it.
|
|
67
60
|
--help, --version`;
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
},
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
},
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
},
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
},
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
},
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
},
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
},
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
},
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
},
|
|
97
|
-
'parallel': {
|
|
98
|
-
type: 'string'
|
|
99
|
-
},
|
|
100
|
-
'diff': {
|
|
101
|
-
type: 'string'
|
|
102
|
-
},
|
|
103
|
-
'changes': {
|
|
104
|
-
type: 'string'
|
|
105
|
-
},
|
|
106
|
-
'key': {
|
|
107
|
-
type: 'string'
|
|
108
|
-
},
|
|
109
|
-
'ignore': {
|
|
110
|
-
type: 'string'
|
|
111
|
-
},
|
|
112
|
-
'host-delay': {
|
|
113
|
-
type: 'string'
|
|
114
|
-
},
|
|
115
|
-
'host-concurrency': {
|
|
116
|
-
type: 'string'
|
|
117
|
-
},
|
|
118
|
-
'hooks': {
|
|
119
|
-
type: 'string'
|
|
120
|
-
},
|
|
121
|
-
'open': {
|
|
122
|
-
type: 'string'
|
|
123
|
-
},
|
|
124
|
-
'plugins': {
|
|
125
|
-
type: 'string'
|
|
126
|
-
},
|
|
127
|
-
'browser': {
|
|
128
|
-
type: 'boolean'
|
|
129
|
-
},
|
|
130
|
-
'browser-path': {
|
|
131
|
-
type: 'string'
|
|
132
|
-
},
|
|
133
|
-
'insecure-tls': {
|
|
134
|
-
type: 'boolean'
|
|
135
|
-
},
|
|
136
|
-
'user-agent': {
|
|
137
|
-
type: 'string'
|
|
138
|
-
},
|
|
139
|
-
'access': {
|
|
140
|
-
type: 'string'
|
|
141
|
-
},
|
|
142
|
-
'access-profile': {
|
|
143
|
-
type: 'string'
|
|
144
|
-
},
|
|
145
|
-
'help': {
|
|
146
|
-
type: 'boolean',
|
|
147
|
-
short: 'h'
|
|
148
|
-
},
|
|
149
|
-
'version': {
|
|
150
|
-
type: 'boolean',
|
|
151
|
-
short: 'v'
|
|
152
|
-
}
|
|
61
|
+
var OPTIONS = {
|
|
62
|
+
"out": { type: "string" },
|
|
63
|
+
"append": { type: "boolean" },
|
|
64
|
+
"resume": { type: "boolean" },
|
|
65
|
+
"only": { type: "string", multiple: true },
|
|
66
|
+
"dry-run": { type: "boolean" },
|
|
67
|
+
"trace": { type: "boolean" },
|
|
68
|
+
"headed": { type: "boolean" },
|
|
69
|
+
"profiles": { type: "string" },
|
|
70
|
+
"retries": { type: "string" },
|
|
71
|
+
"parallel": { type: "string" },
|
|
72
|
+
"diff": { type: "string" },
|
|
73
|
+
"changes": { type: "string" },
|
|
74
|
+
"key": { type: "string" },
|
|
75
|
+
"ignore": { type: "string" },
|
|
76
|
+
"host-delay": { type: "string" },
|
|
77
|
+
"host-concurrency": { type: "string" },
|
|
78
|
+
"hooks": { type: "string" },
|
|
79
|
+
"open": { type: "string" },
|
|
80
|
+
"plugins": { type: "string" },
|
|
81
|
+
"browser": { type: "boolean" },
|
|
82
|
+
"browser-path": { type: "string" },
|
|
83
|
+
"insecure-tls": { type: "boolean" },
|
|
84
|
+
"user-agent": { type: "string" },
|
|
85
|
+
"access": { type: "string" },
|
|
86
|
+
"access-profile": { type: "string" },
|
|
87
|
+
"help": { type: "boolean", short: "h" },
|
|
88
|
+
"version": { type: "boolean", short: "v" }
|
|
153
89
|
};
|
|
154
|
-
/**
|
|
155
|
-
* Turns argv into a command.
|
|
156
|
-
*
|
|
157
|
-
* @param argv - The arguments after the program name.
|
|
158
|
-
* @param env - The environment, for the defaults some options have.
|
|
159
|
-
* @returns The command.
|
|
160
|
-
* @throws Error with a usage message on bad input.
|
|
161
|
-
*/
|
|
162
90
|
function parseArguments(argv, env = {}) {
|
|
163
|
-
const {
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
} = parseArgs({
|
|
167
|
-
args: [...argv],
|
|
168
|
-
options: OPTIONS,
|
|
169
|
-
allowPositionals: true,
|
|
170
|
-
strict: true
|
|
171
|
-
});
|
|
172
|
-
if (values.help === true) return {
|
|
173
|
-
name: 'help'
|
|
174
|
-
};
|
|
175
|
-
if (values.version === true) return {
|
|
176
|
-
name: 'version'
|
|
177
|
-
};
|
|
91
|
+
const { values, positionals } = parseArgs({ args: [...argv], options: OPTIONS, allowPositionals: true, strict: true });
|
|
92
|
+
if (values.help === true) return { name: "help" };
|
|
93
|
+
if (values.version === true) return { name: "version" };
|
|
178
94
|
const [name, ...rest] = positionals;
|
|
179
95
|
const options = {
|
|
180
|
-
browserPath: values[
|
|
181
|
-
insecureTls: values[
|
|
182
|
-
userAgent: values[
|
|
183
|
-
access: values.access ?? (env.OPENCRAW_ACCESS ===
|
|
184
|
-
accessProfile: values[
|
|
96
|
+
browserPath: values["browser-path"] ?? env.OPENCRAW_CHROMIUM,
|
|
97
|
+
insecureTls: values["insecure-tls"] === true || env.OPENCRAW_INSECURE_TLS === "1",
|
|
98
|
+
userAgent: values["user-agent"],
|
|
99
|
+
access: values.access ?? (env.OPENCRAW_ACCESS === "" ? void 0 : env.OPENCRAW_ACCESS),
|
|
100
|
+
accessProfile: values["access-profile"],
|
|
185
101
|
plugins: values.plugins ?? values.hooks ?? nonEmpty(env.OPENCRAW_PLUGINS) ?? nonEmpty(env.OPENCRAW_HOOKS)
|
|
186
102
|
};
|
|
187
103
|
switch (name) {
|
|
188
|
-
case
|
|
189
|
-
{
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
changes: values.changes
|
|
224
|
-
};
|
|
225
|
-
}
|
|
226
|
-
case 'run':
|
|
227
|
-
{
|
|
228
|
-
if (rest.length === 0) throw new Error('run needs at least one recipe file or directory');
|
|
229
|
-
if (values.diff !== undefined && values.resume === true) throw new Error('--diff compares the records a run emits, and --resume skips some: use one or the other');
|
|
230
|
-
if (values.changes !== undefined && values.diff === undefined) throw new Error('--changes needs --diff (or the diff command)');
|
|
231
|
-
if (values.resume === true && values.append !== true) throw new Error('--resume needs --append (and --out)');
|
|
232
|
-
if ((values.append === true || values.resume === true) && values.out === undefined) throw new Error('--append and --resume need --out');
|
|
233
|
-
return {
|
|
234
|
-
name: 'run',
|
|
235
|
-
paths: rest,
|
|
236
|
-
out: values.out,
|
|
237
|
-
append: values.append === true,
|
|
238
|
-
resume: values.resume === true,
|
|
239
|
-
trace: values.trace === true,
|
|
240
|
-
dryRun: values['dry-run'] === true,
|
|
241
|
-
only: values.only ?? [],
|
|
242
|
-
headed: values.headed === true,
|
|
243
|
-
profiles: values.profiles ?? nonEmpty(env.OPENCRAW_PROFILES),
|
|
244
|
-
retries: integer(values.retries, '--retries', 1),
|
|
245
|
-
parallel: integer(values.parallel, '--parallel', 1),
|
|
246
|
-
diff: values.diff,
|
|
247
|
-
changes: values.changes,
|
|
248
|
-
throttle: {
|
|
249
|
-
delayMs: integer(values['host-delay'], '--host-delay', 0),
|
|
250
|
-
concurrency: integer(values['host-concurrency'], '--host-concurrency', 1)
|
|
251
|
-
},
|
|
252
|
-
options
|
|
253
|
-
};
|
|
254
|
-
}
|
|
255
|
-
case 'probe':
|
|
256
|
-
{
|
|
257
|
-
if (rest.length !== 1) throw new Error('probe needs exactly one URL');
|
|
258
|
-
return {
|
|
259
|
-
name: 'probe',
|
|
260
|
-
url: rest[0],
|
|
261
|
-
browser: values.browser === true,
|
|
262
|
-
options
|
|
263
|
-
};
|
|
264
|
-
}
|
|
265
|
-
case 'studio':
|
|
266
|
-
{
|
|
267
|
-
if (rest.length > 1) throw new Error('studio takes at most one folder');
|
|
268
|
-
return {
|
|
269
|
-
name: 'studio',
|
|
270
|
-
folder: rest[0],
|
|
271
|
-
plugins: options.plugins,
|
|
272
|
-
open: values.open
|
|
273
|
-
};
|
|
274
|
-
}
|
|
275
|
-
default:
|
|
276
|
-
{
|
|
277
|
-
throw new Error(`unknown command "${name}"`);
|
|
278
|
-
}
|
|
104
|
+
case void 0: {
|
|
105
|
+
return { name: "help" };
|
|
106
|
+
}
|
|
107
|
+
case "validate": {
|
|
108
|
+
if (rest.length === 0) throw new Error("validate needs at least one recipe file or directory");
|
|
109
|
+
return { name: "validate", paths: rest };
|
|
110
|
+
}
|
|
111
|
+
case "export": {
|
|
112
|
+
if (rest.length !== 2) throw new Error("export needs the records file (JSON Lines) and the output recipe");
|
|
113
|
+
if (values.out === void 0) throw new Error("export needs --out <file>: its extension is the format");
|
|
114
|
+
return { name: "export", records: rest[0], output: rest[1], out: values.out, options };
|
|
115
|
+
}
|
|
116
|
+
case "diff": {
|
|
117
|
+
if (rest.length !== 2) throw new Error("diff needs two JSON Lines files: the previous run, then the current one");
|
|
118
|
+
return { name: "diff", previous: rest[0], current: rest[1], key: list(values.key), ignore: list(values.ignore), changes: values.changes };
|
|
119
|
+
}
|
|
120
|
+
case "run": {
|
|
121
|
+
if (rest.length === 0) throw new Error("run needs at least one recipe file or directory");
|
|
122
|
+
if (values.diff !== void 0 && values.resume === true) throw new Error("--diff compares the records a run emits, and --resume skips some: use one or the other");
|
|
123
|
+
if (values.changes !== void 0 && values.diff === void 0) throw new Error("--changes needs --diff (or the diff command)");
|
|
124
|
+
if (values.resume === true && values.append !== true) throw new Error("--resume needs --append (and --out)");
|
|
125
|
+
if ((values.append === true || values.resume === true) && values.out === void 0) throw new Error("--append and --resume need --out");
|
|
126
|
+
return { name: "run", paths: rest, out: values.out, append: values.append === true, resume: values.resume === true, trace: values.trace === true, dryRun: values["dry-run"] === true, only: values.only ?? [], headed: values.headed === true, profiles: values.profiles ?? nonEmpty(env.OPENCRAW_PROFILES), retries: integer(values.retries, "--retries", 1), parallel: integer(values.parallel, "--parallel", 1), diff: values.diff, changes: values.changes, throttle: { delayMs: integer(values["host-delay"], "--host-delay", 0), concurrency: integer(values["host-concurrency"], "--host-concurrency", 1) }, options };
|
|
127
|
+
}
|
|
128
|
+
case "probe": {
|
|
129
|
+
if (rest.length !== 1) throw new Error("probe needs exactly one URL");
|
|
130
|
+
return { name: "probe", url: rest[0], browser: values.browser === true, options };
|
|
131
|
+
}
|
|
132
|
+
case "studio": {
|
|
133
|
+
if (rest.length > 1) throw new Error("studio takes at most one folder");
|
|
134
|
+
return { name: "studio", folder: rest[0], plugins: options.plugins, open: values.open };
|
|
135
|
+
}
|
|
136
|
+
default: {
|
|
137
|
+
throw new Error(`unknown command "${name}"`);
|
|
138
|
+
}
|
|
279
139
|
}
|
|
280
140
|
}
|
|
281
141
|
function integer(value, name, minimum) {
|
|
282
|
-
if (value ===
|
|
142
|
+
if (value === void 0) return void 0;
|
|
283
143
|
const number = Number(value);
|
|
284
144
|
if (!Number.isSafeInteger(number) || number < minimum) throw new Error(`${name} needs a whole number of at least ${minimum}, not "${value}"`);
|
|
285
145
|
return number;
|
|
286
146
|
}
|
|
287
147
|
function list(value) {
|
|
288
|
-
return value ===
|
|
148
|
+
return value === void 0 ? [] : value.split(",").map((entry) => entry.trim()).filter((entry) => entry !== "");
|
|
289
149
|
}
|
|
290
150
|
function nonEmpty(value) {
|
|
291
|
-
return value ===
|
|
151
|
+
return value === "" ? void 0 : value;
|
|
292
152
|
}
|
|
293
153
|
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
*
|
|
303
|
-
* @param diff - The diff.
|
|
304
|
-
* @param limit - Changes listed at most.
|
|
305
|
-
* @returns The lines.
|
|
306
|
-
*/
|
|
154
|
+
// src/diff/diff-files.use-case.ts
|
|
155
|
+
import { writeFile } from "node:fs/promises";
|
|
156
|
+
import { diffRecords, readRecordsFile } from "@opencraw/core";
|
|
157
|
+
|
|
158
|
+
// src/diff/diff-report.mapper.ts
|
|
159
|
+
import { readableKey } from "@opencraw/core";
|
|
160
|
+
var LIST_LIMIT = 50;
|
|
161
|
+
var VALUE_CHARS = 60;
|
|
307
162
|
function diffReport(diff, limit = LIST_LIMIT) {
|
|
308
163
|
const lines = [`${diff.added} added, ${diff.removed} removed, ${diff.changed} changed, ${diff.unchanged} unchanged (${diff.counts.previous} records before, ${diff.counts.current} now)`];
|
|
309
|
-
if (diff.shrunk !==
|
|
164
|
+
if (diff.shrunk !== void 0) lines.push(`! ${diff.shrunk.current} records where the previous run had ${diff.shrunk.previous}: if the source did not shrink, the site may have changed and the recipe no longer finds everything`);
|
|
310
165
|
if (diff.repeated > 0) lines.push(`! ${diff.repeated} records repeat a key already seen and were not compared: is the key complete?`);
|
|
311
166
|
for (const change of diff.changes.slice(0, limit)) {
|
|
312
167
|
const key = readableKey(change.key);
|
|
313
|
-
if (change.change ===
|
|
168
|
+
if (change.change === "added") lines.push(`+ ${key}`);
|
|
169
|
+
else if (change.change === "removed") lines.push(`- ${key}`);
|
|
170
|
+
else lines.push(`~ ${key} ${change.fields.map((field) => fieldLine(field)).join("; ")}`);
|
|
314
171
|
}
|
|
315
|
-
if (diff.changes.length > limit) lines.push(
|
|
172
|
+
if (diff.changes.length > limit) lines.push(`\u2026 and ${diff.changes.length - limit} more`);
|
|
316
173
|
return lines;
|
|
317
174
|
}
|
|
318
175
|
function fieldLine(change) {
|
|
319
|
-
return `${change.field}: ${shown(change.before)}
|
|
176
|
+
return `${change.field}: ${shown(change.before)} \u2192 ${shown(change.after)}`;
|
|
320
177
|
}
|
|
321
178
|
function shown(value) {
|
|
322
|
-
const text = JSON.stringify(value) ??
|
|
323
|
-
return text.length > VALUE_CHARS ? `${text.slice(0, VALUE_CHARS)}
|
|
179
|
+
const text = JSON.stringify(value) ?? "null";
|
|
180
|
+
return text.length > VALUE_CHARS ? `${text.slice(0, VALUE_CHARS)}\u2026` : text;
|
|
324
181
|
}
|
|
325
182
|
|
|
326
|
-
|
|
327
|
-
* Compares two runs' JSON Lines files and prints what changed; with
|
|
328
|
-
* `--changes`, writes every change as a JSON line too.
|
|
329
|
-
*
|
|
330
|
-
* @param command - The parsed `diff` command.
|
|
331
|
-
* @param terminal - Where the report goes.
|
|
332
|
-
* @returns The exit code: 0, or 1 when a file cannot be read or has no key.
|
|
333
|
-
*/
|
|
183
|
+
// src/diff/diff-files.use-case.ts
|
|
334
184
|
async function diffFiles(command, terminal) {
|
|
335
185
|
try {
|
|
336
186
|
const [previous, current] = await Promise.all([readRecordsFile(command.previous), readRecordsFile(command.current)]);
|
|
337
|
-
const diff = diffRecords(previous, current, {
|
|
338
|
-
key: command.key.length === 0 ? undefined : command.key,
|
|
339
|
-
ignore: command.ignore
|
|
340
|
-
});
|
|
187
|
+
const diff = diffRecords(previous, current, { key: command.key.length === 0 ? void 0 : command.key, ignore: command.ignore });
|
|
341
188
|
for (const line of diffReport(diff)) terminal.out(line);
|
|
342
|
-
if (command.changes !==
|
|
189
|
+
if (command.changes !== void 0) await writeChanges(command.changes, diff);
|
|
343
190
|
return 0;
|
|
344
191
|
} catch (error) {
|
|
345
192
|
terminal.err(error instanceof Error ? error.message : String(error));
|
|
346
193
|
return 1;
|
|
347
194
|
}
|
|
348
195
|
}
|
|
349
|
-
/**
|
|
350
|
-
* Writes a diff's changes as JSON Lines: `{ change, key, before?, after?, fields? }`.
|
|
351
|
-
*
|
|
352
|
-
* @param path - The file.
|
|
353
|
-
* @param diff - The diff.
|
|
354
|
-
*/
|
|
355
196
|
async function writeChanges(path, diff) {
|
|
356
|
-
await writeFile(path, diff.changes.map(change => `${JSON.stringify(change)}
|
|
197
|
+
await writeFile(path, diff.changes.map((change) => `${JSON.stringify(change)}
|
|
198
|
+
`).join(""));
|
|
357
199
|
}
|
|
358
200
|
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
*
|
|
363
|
-
* @param command - The parsed `export` command.
|
|
364
|
-
* @param terminal - Where the result and errors go.
|
|
365
|
-
* @returns The exit code: 0, or 1 when a file cannot be read, the recipe is not an output recipe, or the extension names no format.
|
|
366
|
-
*/
|
|
201
|
+
// src/export/export-records.use-case.ts
|
|
202
|
+
import { readFile } from "node:fs/promises";
|
|
203
|
+
import { RecipeValidationError, fileSink, outputRecordOf, parseOutputRecipe, readRecordsFile as readRecordsFile2 } from "@opencraw/core";
|
|
367
204
|
async function exportRecords(command, terminal) {
|
|
368
205
|
try {
|
|
369
|
-
const sink = fileSink(command.out, {
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
}
|
|
373
|
-
});
|
|
374
|
-
const output = parseOutputRecipe(JSON.parse(await readFile(command.output, 'utf8')), command.output);
|
|
375
|
-
const stored = await readRecordsFile(command.records);
|
|
206
|
+
const sink = fileSink(command.out, { browser: { executablePath: command.options.browserPath } });
|
|
207
|
+
const output = parseOutputRecipe(JSON.parse(await readFile(command.output, "utf8")), command.output);
|
|
208
|
+
const stored = await readRecordsFile2(command.records);
|
|
376
209
|
await sink.open(output);
|
|
377
210
|
for (const line of stored) await sink.write(outputRecordOf(line));
|
|
378
211
|
const summary = await sink.close();
|
|
379
212
|
terminal.err(`${String(summary.written)} records written to ${summary.location ?? command.out}`);
|
|
380
213
|
return 0;
|
|
381
214
|
} catch (error) {
|
|
382
|
-
if (error instanceof RecipeValidationError) for (const issue of error.issues) terminal.err(`${error.source}: ${issue.path ===
|
|
215
|
+
if (error instanceof RecipeValidationError) for (const issue of error.issues) terminal.err(`${error.source}: ${issue.path === "" ? "(root)" : issue.path}: ${issue.message}`);
|
|
216
|
+
else terminal.err(error instanceof Error ? error.message : String(error));
|
|
383
217
|
return 1;
|
|
384
218
|
}
|
|
385
219
|
}
|
|
386
220
|
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
221
|
+
// src/probe/probe-page.use-case.ts
|
|
222
|
+
import { resolve as resolve2 } from "node:path";
|
|
223
|
+
import { pathToFileURL as pathToFileURL2 } from "node:url";
|
|
224
|
+
import { AccessBroker, BrowserClient, HostAllowlist, HttpClient, parseXml } from "@opencraw/core";
|
|
225
|
+
import { describeDeck, describeHtml, describeJson, describePdf, describeWorkbook, describeXml, findData } from "@opencraw/probe";
|
|
226
|
+
|
|
227
|
+
// src/access/resolve-access.use-case.ts
|
|
228
|
+
import { loadAccessConfig } from "@opencraw/core";
|
|
395
229
|
async function resolveAccess(options) {
|
|
396
|
-
if (options.access ===
|
|
397
|
-
if (options.accessProfile !==
|
|
398
|
-
return
|
|
230
|
+
if (options.access === void 0) {
|
|
231
|
+
if (options.accessProfile !== void 0) throw new Error("--access-profile needs an access config: --access <file> or OPENCRAW_ACCESS");
|
|
232
|
+
return void 0;
|
|
399
233
|
}
|
|
400
234
|
const config = await loadAccessConfig(options.access);
|
|
401
|
-
if (options.accessProfile ===
|
|
235
|
+
if (options.accessProfile === void 0) return config;
|
|
402
236
|
if (!Object.hasOwn(config.profiles, options.accessProfile)) {
|
|
403
|
-
throw new Error(`access profile "${options.accessProfile}" is not in ${options.access} (profiles: ${Object.keys(config.profiles).join(
|
|
237
|
+
throw new Error(`access profile "${options.accessProfile}" is not in ${options.access} (profiles: ${Object.keys(config.profiles).join(", ") || "none"})`);
|
|
404
238
|
}
|
|
405
|
-
return {
|
|
406
|
-
...config,
|
|
407
|
-
default: options.accessProfile
|
|
408
|
-
};
|
|
239
|
+
return { ...config, default: options.accessProfile };
|
|
409
240
|
}
|
|
410
241
|
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
* (`{ name: function }`), `accessPlugins` (`[{ name, lease }]`) and
|
|
416
|
-
* `captchaSolvers` (`[{ name, solve }]`) give what recipes and access profiles
|
|
417
|
-
* refer to by name:
|
|
418
|
-
*
|
|
419
|
-
* ```js
|
|
420
|
-
* export const hooks = { positive: v => v > 0 }
|
|
421
|
-
* export const accessPlugins = [{ name: 'browserbase', lease: async () => ({ cdp: { endpoint } }) }]
|
|
422
|
-
* export const captchaSolvers = [{ name: 'capsolver', solve: async (challenge, { page }) => ({ status: 'solved' }) }]
|
|
423
|
-
* ```
|
|
424
|
-
*
|
|
425
|
-
* A module with none of those names is a bare hooks module, as before: its
|
|
426
|
-
* default export when that is an object of functions, else its named function
|
|
427
|
-
* exports.
|
|
428
|
-
*
|
|
429
|
-
* The module runs with the caller's privileges, like any code the caller
|
|
430
|
-
* imports: point this only at a file you trust.
|
|
431
|
-
*
|
|
432
|
-
* @param path - A `.mjs` / `.js` / `.cjs` file, or a `.ts` / `.mts` file on a Node that strips types (22.18+).
|
|
433
|
-
* @param importModule - How to import it; `import()` unless a test says otherwise.
|
|
434
|
-
* @returns The hooks and plugins.
|
|
435
|
-
* @throws Error naming the file when it cannot be imported, provides nothing,
|
|
436
|
-
* or provides something that is not what its name promises.
|
|
437
|
-
*/
|
|
242
|
+
// src/hooks-module/load-hooks.repository.ts
|
|
243
|
+
import { resolve } from "node:path";
|
|
244
|
+
import { pathToFileURL } from "node:url";
|
|
245
|
+
var PLUGIN_EXPORTS = ["hooks", "accessPlugins", "captchaSolvers"];
|
|
438
246
|
async function loadPlugins(path, importModule = importByUrl) {
|
|
439
247
|
const url = pathToFileURL(resolve(path)).href;
|
|
440
248
|
let exported;
|
|
441
249
|
try {
|
|
442
250
|
exported = await importModule(url);
|
|
443
251
|
} catch (error) {
|
|
444
|
-
throw new Error(`${path}: cannot load hooks (${error.message})`, {
|
|
445
|
-
cause: error
|
|
446
|
-
});
|
|
252
|
+
throw new Error(`${path}: cannot load hooks (${error.message})`, { cause: error });
|
|
447
253
|
}
|
|
448
|
-
if (PLUGIN_EXPORTS.every(name => !Object.hasOwn(exported, name))) {
|
|
254
|
+
if (PLUGIN_EXPORTS.every((name) => !Object.hasOwn(exported, name))) {
|
|
449
255
|
const hooks = hooksOf(hookExports(exported), path);
|
|
450
256
|
if (Object.keys(hooks).length === 0) throw new Error(`${path}: exports no hooks (export default { name: function } or named functions) and no plugins (export const hooks, accessPlugins, captchaSolvers)`);
|
|
451
|
-
return {
|
|
452
|
-
hooks,
|
|
453
|
-
accessPlugins: [],
|
|
454
|
-
captchaSolvers: []
|
|
455
|
-
};
|
|
257
|
+
return { hooks, accessPlugins: [], captchaSolvers: [] };
|
|
456
258
|
}
|
|
457
259
|
return {
|
|
458
|
-
hooks: hooksOf(objectExport(exported.hooks,
|
|
459
|
-
accessPlugins: pluginsOf(exported.accessPlugins,
|
|
460
|
-
captchaSolvers: pluginsOf(exported.captchaSolvers,
|
|
260
|
+
hooks: hooksOf(objectExport(exported.hooks, "hooks", path), path),
|
|
261
|
+
accessPlugins: pluginsOf(exported.accessPlugins, "accessPlugins", "lease", path),
|
|
262
|
+
captchaSolvers: pluginsOf(exported.captchaSolvers, "captchaSolvers", "solve", path)
|
|
461
263
|
};
|
|
462
264
|
}
|
|
463
|
-
/**
|
|
464
|
-
* Loads the hooks of a hooks or plugins module (see {@link loadPlugins}).
|
|
465
|
-
*
|
|
466
|
-
* @param path - A `.mjs` / `.js` / `.cjs` file.
|
|
467
|
-
* @param importModule - How to import it.
|
|
468
|
-
* @returns The hooks by name.
|
|
469
|
-
* @throws Error naming the file when it cannot be imported or exports no hooks.
|
|
470
|
-
*/
|
|
471
265
|
async function loadHooks(path, importModule = importByUrl) {
|
|
472
|
-
const {
|
|
473
|
-
hooks
|
|
474
|
-
} = await loadPlugins(path, importModule);
|
|
266
|
+
const { hooks } = await loadPlugins(path, importModule);
|
|
475
267
|
if (Object.keys(hooks).length === 0) throw new Error(`${path}: exports no hooks (export default { name: function } or named functions)`);
|
|
476
268
|
return hooks;
|
|
477
269
|
}
|
|
478
270
|
function hooksOf(candidates, path) {
|
|
479
271
|
const hooks = {};
|
|
480
272
|
for (const [name, value] of Object.entries(candidates)) {
|
|
481
|
-
if (typeof value !==
|
|
273
|
+
if (typeof value !== "function") throw new Error(`${path}: hook "${name}" is ${value === null ? "null" : typeof value}, not a function`);
|
|
482
274
|
hooks[name] = value;
|
|
483
275
|
}
|
|
484
276
|
return hooks;
|
|
485
277
|
}
|
|
486
278
|
function objectExport(value, name, path) {
|
|
487
|
-
if (value ===
|
|
488
|
-
if (typeof value !==
|
|
279
|
+
if (value === void 0) return {};
|
|
280
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${path}: "${name}" must be an object of functions`);
|
|
489
281
|
return value;
|
|
490
282
|
}
|
|
491
|
-
/** A list of `{ name, <method> }` plugins, checked. */
|
|
492
283
|
function pluginsOf(value, exportName, method, path) {
|
|
493
|
-
if (value ===
|
|
284
|
+
if (value === void 0) return [];
|
|
494
285
|
if (!Array.isArray(value)) throw new Error(`${path}: "${exportName}" must be an array of { name, ${method} }`);
|
|
495
|
-
const names = new Set();
|
|
286
|
+
const names = /* @__PURE__ */ new Set();
|
|
496
287
|
for (const [index, plugin] of value.entries()) {
|
|
497
288
|
const entry = plugin;
|
|
498
|
-
if (typeof entry?.name !==
|
|
289
|
+
if (typeof entry?.name !== "string" || entry.name === "" || typeof entry[method] !== "function") throw new Error(`${path}: ${exportName}[${index}] must be { name: string, ${method}: function }`);
|
|
499
290
|
if (names.has(entry.name)) throw new Error(`${path}: ${exportName} names "${entry.name}" twice`);
|
|
500
291
|
names.add(entry.name);
|
|
501
292
|
}
|
|
@@ -506,130 +297,99 @@ async function importByUrl(url) {
|
|
|
506
297
|
}
|
|
507
298
|
function hookExports(exported) {
|
|
508
299
|
const fallback = exported.default;
|
|
509
|
-
if (typeof fallback ===
|
|
510
|
-
const named = Object.entries(exported).filter(([name]) => name !==
|
|
300
|
+
if (typeof fallback === "object" && fallback !== null && !Array.isArray(fallback)) return fallback;
|
|
301
|
+
const named = Object.entries(exported).filter(([name]) => name !== "default" && name !== "module.exports");
|
|
511
302
|
return Object.fromEntries(named);
|
|
512
303
|
}
|
|
513
304
|
|
|
514
|
-
|
|
515
|
-
* Renders a probe's findings as text.
|
|
516
|
-
*
|
|
517
|
-
* @param url - The URL probed.
|
|
518
|
-
* @param status - The HTTP status.
|
|
519
|
-
* @param findings - What `findData` found.
|
|
520
|
-
* @param observed - JSON responses a browser saw, when `--browser` was used.
|
|
521
|
-
* @param html - The tables (and, for Markdown, the outline and front matter) found.
|
|
522
|
-
* @returns The report.
|
|
523
|
-
*/
|
|
305
|
+
// src/probe/probe-report.mapper.ts
|
|
524
306
|
function probeReport(url, status, findings, observed = [], html) {
|
|
525
|
-
const sections = [
|
|
526
|
-
|
|
307
|
+
const sections = [
|
|
308
|
+
section("Markdown front matter (in script[data-front-matter])", html?.frontMatter === void 0 || html.frontMatter.length === 0 ? [] : [` keys: ${html.frontMatter.join(", ")}`]),
|
|
309
|
+
section('Sections (a "css" selector for each)', (html?.outline ?? []).map((entry) => ` ${" ".repeat(entry.level - 1)}${entry.selector}`)),
|
|
310
|
+
section('HTML tables (a "table" extract selector for each)', (html?.tables ?? []).map((table) => ` ${table.table} ${table.selector}${table.hint === void 0 ? "" : ` (${table.hint})`}
|
|
311
|
+
${table.text}`)),
|
|
312
|
+
section("JSON-LD blocks", findings.jsonLd.map((block) => ` type=${block.types}, ${block.keys} keys`)),
|
|
313
|
+
section("Inline JSON (candidates)", findings.inlineJson.map((block) => ` ${block.where}, ${block.size} chars, keys: ${block.keys.slice(0, 8).join(", ")}`)),
|
|
314
|
+
section(".json URLs referenced", findings.jsonUrls.map((url_) => ` ${url_}`)),
|
|
315
|
+
section("Script hosts", findings.scriptHosts.map((host) => ` ${host}`)),
|
|
316
|
+
section("Links that look like an API", findings.apiLinks.map((link) => ` ${link}`)),
|
|
317
|
+
observed.length > 0 ? section("JSON responses observed in the browser", observed.map((entry) => ` ${entry}`)) : ""
|
|
318
|
+
];
|
|
319
|
+
return [`${url} (HTTP ${status})`, "", ...sections].filter((line) => line !== "").join("\n");
|
|
527
320
|
}
|
|
528
|
-
/**
|
|
529
|
-
* Renders what a probe found in a PDF as text.
|
|
530
|
-
*
|
|
531
|
-
* @param url - The URL or file probed.
|
|
532
|
-
* @param pdf - What `describePdf` found.
|
|
533
|
-
* @returns The report.
|
|
534
|
-
*/
|
|
535
321
|
function pdfReport(url, pdf) {
|
|
536
|
-
return [
|
|
322
|
+
return [
|
|
323
|
+
`${url} (PDF, ${pdf.pages} page${pdf.pages === 1 ? "" : "s"})`,
|
|
324
|
+
"",
|
|
325
|
+
section('Likely table headers (a "table" extract selector for each)', pdf.headers.map((header) => ` p${header.page} ${header.selector}
|
|
326
|
+
${header.text}`)),
|
|
327
|
+
section('First rows (cells separated by " | ")', pdf.rows.map((row) => ` p${row.page} ${row.text}`))
|
|
328
|
+
].filter((line) => line !== "").join("\n");
|
|
537
329
|
}
|
|
538
|
-
/**
|
|
539
|
-
* Formats what a probe found in a workbook (a CSV or a spreadsheet): its sheets,
|
|
540
|
-
* the rows that look like table headers with a selector for each, and the first
|
|
541
|
-
* rows.
|
|
542
|
-
*
|
|
543
|
-
* @param url - The workbook's URL.
|
|
544
|
-
* @param workbook - The findings.
|
|
545
|
-
* @returns The report.
|
|
546
|
-
*/
|
|
547
330
|
function workbookReport(url, workbook) {
|
|
548
|
-
const read = workbook.csv ===
|
|
549
|
-
return [
|
|
331
|
+
const read = workbook.csv === void 0 ? "workbook" : `CSV, ${workbook.csv.encoding}, delimited by ${delimiterName(workbook.csv.delimiter)}`;
|
|
332
|
+
return [
|
|
333
|
+
`${url} (${read})`,
|
|
334
|
+
"",
|
|
335
|
+
section("Sheets", workbook.sheets.map((sheet) => ` ${sheet.name}${sheet.hidden ? " (hidden)" : ""}: ${sheet.rows} rows \xD7 ${sheet.columns} columns`)),
|
|
336
|
+
section('Likely table headers (a "table" extract selector for each)', workbook.headers.map((header) => ` ${header.sheet} r${header.row} ${header.selector}${header.hint === void 0 ? "" : ` (${header.hint})`}
|
|
337
|
+
${header.text}`)),
|
|
338
|
+
section('First rows (cells separated by " | ")', workbook.rows.map((row) => ` ${row.sheet} r${row.row} ${row.text}`))
|
|
339
|
+
].filter((line) => line !== "").join("\n");
|
|
550
340
|
}
|
|
551
|
-
/**
|
|
552
|
-
* Formats what a probe found in a deck (a presentation): its slides, the native
|
|
553
|
-
* tables' headers with a selector for each, its charts, and the slides whose
|
|
554
|
-
* text boxes look like a table.
|
|
555
|
-
*
|
|
556
|
-
* @param url - The deck's URL.
|
|
557
|
-
* @param deck - The findings.
|
|
558
|
-
* @returns The report.
|
|
559
|
-
*/
|
|
560
341
|
function deckReport(url, deck) {
|
|
561
|
-
return [
|
|
342
|
+
return [
|
|
343
|
+
`${url} (presentation, ${deck.slides.length} slide${deck.slides.length === 1 ? "" : "s"}, ${deck.width} \xD7 ${deck.height} pt)`,
|
|
344
|
+
"",
|
|
345
|
+
section("Slides", deck.slides.map((slide) => ` ${slide.number} ${slide.title === "" ? "(no title)" : slide.title}${slide.hidden ? " (hidden)" : ""}: ${slide.shapes} text boxes, ${slide.tables} tables, ${slide.charts} charts`)),
|
|
346
|
+
section('Likely table headers (a "table" extract selector for each)', deck.headers.map((header) => ` slide ${header.slide} ${header.selector}${header.hint === void 0 ? "" : ` (${header.hint})`}
|
|
347
|
+
${header.text}`)),
|
|
348
|
+
section('Text boxes laid out as a table (a "table" extract with "shapes": true)', deck.grids.map((grid) => ` slide ${grid.slide} ${grid.title}: ${grid.boxes} short boxes`)),
|
|
349
|
+
section("Charts (read with jsonpath: $.slides[*].charts[*].series[*])", deck.charts.map((chart) => ` slide ${chart.slide} ${chart.type}${chart.title === void 0 ? "" : ` "${chart.title}"`}: ${chart.series.map((series) => `${series.name} (${series.points})`).join(", ")}`))
|
|
350
|
+
].filter((line) => line !== "").join("\n");
|
|
562
351
|
}
|
|
563
|
-
/**
|
|
564
|
-
* Formats what a probe found in JSON data: its record lists with the path a
|
|
565
|
-
* `jsonpath` extract needs, then its structure.
|
|
566
|
-
*
|
|
567
|
-
* @param url - The document's URL.
|
|
568
|
-
* @param json - The findings.
|
|
569
|
-
* @returns The report.
|
|
570
|
-
*/
|
|
571
352
|
function jsonReport(url, json) {
|
|
572
|
-
const read = {
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
353
|
+
const read = { json: "JSON", jsonl: "JSON Lines", yaml: "YAML" }[json.format] ?? json.format;
|
|
354
|
+
return [
|
|
355
|
+
`${url} (${read}: ${json.type})`,
|
|
356
|
+
"",
|
|
357
|
+
section('Record lists (a "jsonpath" selector for each, largest first)', json.lists.map((list2) => ` ${list2.path} ${list2.length} entries
|
|
358
|
+
keys: ${list2.keys.join(", ")}`)),
|
|
359
|
+
section("Structure", json.tree.map((line) => ` ${line}`))
|
|
360
|
+
].filter((line) => line !== "").join("\n");
|
|
578
361
|
}
|
|
579
|
-
/**
|
|
580
|
-
* Formats what a probe found in an XML document: its namespaces and the
|
|
581
|
-
* `namespaces` an `xpath` extract needs, the elements that repeat, and its structure.
|
|
582
|
-
*
|
|
583
|
-
* @param url - The document's URL.
|
|
584
|
-
* @param xml - The findings.
|
|
585
|
-
* @returns The report.
|
|
586
|
-
*/
|
|
587
362
|
function xmlReport(url, xml) {
|
|
588
|
-
const defaultNamespace = xml.namespaces[
|
|
589
|
-
const namespaceRows = Object.entries(xml.namespaces).map(([prefix, uri]) => ` ${prefix ===
|
|
590
|
-
const hint = defaultNamespace ===
|
|
591
|
-
return [
|
|
363
|
+
const defaultNamespace = xml.namespaces[""];
|
|
364
|
+
const namespaceRows = Object.entries(xml.namespaces).map(([prefix, uri]) => ` ${prefix === "" ? "(default)" : prefix} ${uri}`);
|
|
365
|
+
const hint = defaultNamespace === void 0 ? [] : ["", ` The elements are in a default namespace: query with "namespaces": { "x": "${defaultNamespace}" } and //x:${xml.root},`, ' or with "ignoreNamespaces": true and plain names, as below.'];
|
|
366
|
+
return [
|
|
367
|
+
`${url} (XML: ${xml.root}${xml.sitemap === void 0 ? "" : `, a sitemap of ${xml.sitemap.locations} ${xml.sitemap.kind === "urlset" ? "pages" : "sitemaps"}`})`,
|
|
368
|
+
"",
|
|
369
|
+
section("Namespaces", [...namespaceRows, ...hint]),
|
|
370
|
+
section('Repeated elements (an "xpath" selector for each, largest first)', xml.lists.map((list2) => ` ${list2.path} ${list2.count} entries${list2.children.length > 0 ? `
|
|
371
|
+
children: ${list2.children.join(", ")}` : ""}`)),
|
|
372
|
+
section("Structure", xml.tree.map((line) => ` ${line}`))
|
|
373
|
+
].filter((line) => line !== "").join("\n");
|
|
592
374
|
}
|
|
593
375
|
function delimiterName(delimiter) {
|
|
594
|
-
const names = {
|
|
595
|
-
'\t': 'tabs',
|
|
596
|
-
',': 'commas',
|
|
597
|
-
';': 'semicolons',
|
|
598
|
-
'|': 'pipes'
|
|
599
|
-
};
|
|
376
|
+
const names = { " ": "tabs", ",": "commas", ";": "semicolons", "|": "pipes" };
|
|
600
377
|
return names[delimiter] ?? `"${delimiter}"`;
|
|
601
378
|
}
|
|
602
379
|
function section(title, rows) {
|
|
603
|
-
if (rows.length === 0) return
|
|
604
|
-
return [`${title}:`, ...rows,
|
|
380
|
+
if (rows.length === 0) return "";
|
|
381
|
+
return [`${title}:`, ...rows, ""].join("\n");
|
|
605
382
|
}
|
|
606
383
|
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
* Fetches a page and finds where its data lives, through the access profile the
|
|
611
|
-
* options name (direct without one). With `browser: true` it also
|
|
612
|
-
* renders the page and lists the JSON responses seen while it settles, which
|
|
613
|
-
* finds endpoints a plain fetch of the initial HTML cannot. No `Terminal`
|
|
614
|
-
* involved: the cli's `probePage` and `@opencraw/mcp`'s probe tool both
|
|
615
|
-
* build on this, one printing the result, the other returning it as data.
|
|
616
|
-
*
|
|
617
|
-
* A PDF, a spreadsheet, a CSV or a presentation (by content type, or a local
|
|
618
|
-
* path) is read instead: its rows, and the rows that look like table headers.
|
|
619
|
-
*
|
|
620
|
-
* @param url - The page to probe, or a local file path.
|
|
621
|
-
* @param options - Browser path, TLS and user agent, plus whether to render.
|
|
622
|
-
* @returns What was found.
|
|
623
|
-
* @throws Error when the fetch itself fails.
|
|
624
|
-
*/
|
|
384
|
+
// src/probe/probe-page.use-case.ts
|
|
385
|
+
var BROWSER_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0 Safari/537.36";
|
|
386
|
+
var OBSERVE_MS = 4e3;
|
|
625
387
|
async function probeUrl(url, options) {
|
|
626
388
|
const allowedHosts = HostAllowlist.of(options.allowedHosts);
|
|
627
|
-
const plugins = options.accessPlugins ===
|
|
628
|
-
const access = options.accessConfig ===
|
|
629
|
-
const lease = await new AccessBroker(access, options.accessPlugins ?? plugins?.accessPlugins).lease({
|
|
630
|
-
|
|
631
|
-
});
|
|
632
|
-
if (lease.cdp !== undefined) throw new Error(`access profile "${lease.profile}" is a remote browser; probe fetches over HTTP and needs a proxy profile`);
|
|
389
|
+
const plugins = options.accessPlugins === void 0 && options.plugins !== void 0 ? await loadPlugins(options.plugins) : void 0;
|
|
390
|
+
const access = options.accessConfig === void 0 ? await resolveAccess(options) : withDefault(options.accessConfig, options.accessProfile);
|
|
391
|
+
const lease = await new AccessBroker(access, options.accessPlugins ?? plugins?.accessPlugins).lease({ recipeId: "probe" });
|
|
392
|
+
if (lease.cdp !== void 0) throw new Error(`access profile "${lease.profile}" is a remote browser; probe fetches over HTTP and needs a proxy profile`);
|
|
633
393
|
const client = await HttpClient.open({
|
|
634
394
|
userAgent: options.userAgent ?? BROWSER_USER_AGENT,
|
|
635
395
|
ignoreHTTPSErrors: options.insecureTls || lease.ignoreHTTPSErrors === true,
|
|
@@ -638,77 +398,23 @@ async function probeUrl(url, options) {
|
|
|
638
398
|
allowedHosts
|
|
639
399
|
});
|
|
640
400
|
try {
|
|
641
|
-
const target = /^[a-z][\w+.-]+:/i.test(url) ? url :
|
|
642
|
-
// GitHub raw and CDNs serve Markdown as text/plain: a .md URL is read as Markdown.
|
|
401
|
+
const target = /^[a-z][\w+.-]+:/i.test(url) ? url : pathToFileURL2(resolve2(url)).href;
|
|
643
402
|
const markdown = /\.(?:md|markdown)$/i.test(new URL(target).pathname);
|
|
644
|
-
const response = await client.send({
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
}
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
status: response.status,
|
|
656
|
-
findings: findData(''),
|
|
657
|
-
observed: [],
|
|
658
|
-
pdf: describePdf(body)
|
|
659
|
-
};
|
|
660
|
-
if (body.kind === 'workbook') return {
|
|
661
|
-
url: response.url,
|
|
662
|
-
status: response.status,
|
|
663
|
-
findings: findData(''),
|
|
664
|
-
observed: [],
|
|
665
|
-
workbook: describeWorkbook(body)
|
|
666
|
-
};
|
|
667
|
-
if (body.kind === 'deck') return {
|
|
668
|
-
url: response.url,
|
|
669
|
-
status: response.status,
|
|
670
|
-
findings: findData(''),
|
|
671
|
-
observed: [],
|
|
672
|
-
deck: describeDeck(body)
|
|
673
|
-
};
|
|
674
|
-
if (body.kind === 'json') return {
|
|
675
|
-
url: response.url,
|
|
676
|
-
status: response.status,
|
|
677
|
-
findings: findData(''),
|
|
678
|
-
observed: [],
|
|
679
|
-
json: describeJson(body.data, response.format ?? 'json')
|
|
680
|
-
};
|
|
681
|
-
if (body.kind === 'xml') return {
|
|
682
|
-
url: response.url,
|
|
683
|
-
status: response.status,
|
|
684
|
-
findings: findData(''),
|
|
685
|
-
observed: [],
|
|
686
|
-
xml: describeXml(parseXml(body.xml, response.url))
|
|
687
|
-
};
|
|
688
|
-
const text = body.kind === 'html' ? body.html : body.text;
|
|
689
|
-
const observed = options.browser && !target.startsWith('file:') ? await observeBrowserJson(url, options, lease, allowedHosts) : [];
|
|
690
|
-
const html = body.kind === 'html' ? {
|
|
691
|
-
html: describeHtml(body.html, response.format === 'markdown' || response.format === 'docx')
|
|
692
|
-
} : {};
|
|
693
|
-
return {
|
|
694
|
-
url: response.url,
|
|
695
|
-
status: response.status,
|
|
696
|
-
findings: findData(text),
|
|
697
|
-
observed,
|
|
698
|
-
...html
|
|
699
|
-
};
|
|
403
|
+
const response = await client.send({ url: target, ...markdown && { as: "markdown" } });
|
|
404
|
+
const { body } = response;
|
|
405
|
+
if (body.kind === "pdf") return { url: response.url, status: response.status, findings: findData(""), observed: [], pdf: describePdf(body) };
|
|
406
|
+
if (body.kind === "workbook") return { url: response.url, status: response.status, findings: findData(""), observed: [], workbook: describeWorkbook(body) };
|
|
407
|
+
if (body.kind === "deck") return { url: response.url, status: response.status, findings: findData(""), observed: [], deck: describeDeck(body) };
|
|
408
|
+
if (body.kind === "json") return { url: response.url, status: response.status, findings: findData(""), observed: [], json: describeJson(body.data, response.format ?? "json") };
|
|
409
|
+
if (body.kind === "xml") return { url: response.url, status: response.status, findings: findData(""), observed: [], xml: describeXml(parseXml(body.xml, response.url)) };
|
|
410
|
+
const text = body.kind === "html" ? body.html : body.text;
|
|
411
|
+
const observed = options.browser && !target.startsWith("file:") ? await observeBrowserJson(url, options, lease, allowedHosts) : [];
|
|
412
|
+
const html = body.kind === "html" ? { html: describeHtml(body.html, response.format === "markdown" || response.format === "docx") } : {};
|
|
413
|
+
return { url: response.url, status: response.status, findings: findData(text), observed, ...html };
|
|
700
414
|
} finally {
|
|
701
415
|
await client.dispose();
|
|
702
416
|
}
|
|
703
417
|
}
|
|
704
|
-
/**
|
|
705
|
-
* Fetches a page and reports where its data lives, to a `Terminal`.
|
|
706
|
-
*
|
|
707
|
-
* @param url - The page to probe.
|
|
708
|
-
* @param options - Browser path, TLS and user agent, plus whether to render.
|
|
709
|
-
* @param terminal - Where the report goes.
|
|
710
|
-
* @returns The exit code: 0, or 1 on a fetch failure.
|
|
711
|
-
*/
|
|
712
418
|
async function probePage(url, options, terminal) {
|
|
713
419
|
try {
|
|
714
420
|
const result = await probeUrl(url, options);
|
|
@@ -720,28 +426,16 @@ async function probePage(url, options, terminal) {
|
|
|
720
426
|
}
|
|
721
427
|
}
|
|
722
428
|
async function observeBrowserJson(url, options, lease, allowedHosts) {
|
|
723
|
-
const browser = await BrowserClient.launch({
|
|
724
|
-
executablePath: options.browserPath,
|
|
725
|
-
ignoreHTTPSErrors: options.insecureTls
|
|
726
|
-
});
|
|
429
|
+
const browser = await BrowserClient.launch({ executablePath: options.browserPath, ignoreHTTPSErrors: options.insecureTls });
|
|
727
430
|
try {
|
|
728
|
-
const session = await browser.newSession({
|
|
729
|
-
userAgent: options.userAgent,
|
|
730
|
-
proxy: lease.proxy,
|
|
731
|
-
headers: lease.headers,
|
|
732
|
-
ignoreHTTPSErrors: lease.ignoreHTTPSErrors,
|
|
733
|
-
allowedHosts
|
|
734
|
-
});
|
|
431
|
+
const session = await browser.newSession({ userAgent: options.userAgent, proxy: lease.proxy, headers: lease.headers, ignoreHTTPSErrors: lease.ignoreHTTPSErrors, allowedHosts });
|
|
735
432
|
const seen = [];
|
|
736
|
-
session.page.on(
|
|
737
|
-
if ((response.headers()[
|
|
433
|
+
session.page.on("response", (response) => {
|
|
434
|
+
if ((response.headers()["content-type"] ?? "").includes("json")) seen.push(`${response.status()} ${response.url()}`);
|
|
738
435
|
});
|
|
739
436
|
try {
|
|
740
|
-
await session.page.goto(url, {
|
|
741
|
-
waitUntil: 'networkidle'
|
|
742
|
-
});
|
|
437
|
+
await session.page.goto(url, { waitUntil: "networkidle" });
|
|
743
438
|
} catch {
|
|
744
|
-
// a page that never idles (long-poll, websocket) still yields whatever the browser observed
|
|
745
439
|
}
|
|
746
440
|
await session.page.waitForTimeout(OBSERVE_MS);
|
|
747
441
|
await session.close();
|
|
@@ -750,82 +444,56 @@ async function observeBrowserJson(url, options, lease, allowedHosts) {
|
|
|
750
444
|
await browser.close();
|
|
751
445
|
}
|
|
752
446
|
}
|
|
753
|
-
/** An access config with its default profile overridden, when one is named. */
|
|
754
447
|
function withDefault(config, profile) {
|
|
755
|
-
if (profile ===
|
|
756
|
-
if (!Object.hasOwn(config.profiles, profile)) throw new Error(`access profile "${profile}" is not in the access config (profiles: ${Object.keys(config.profiles).join(
|
|
757
|
-
return {
|
|
758
|
-
...config,
|
|
759
|
-
default: profile
|
|
760
|
-
};
|
|
448
|
+
if (profile === void 0) return config;
|
|
449
|
+
if (!Object.hasOwn(config.profiles, profile)) throw new Error(`access profile "${profile}" is not in the access config (profiles: ${Object.keys(config.profiles).join(", ") || "none"})`);
|
|
450
|
+
return { ...config, default: profile };
|
|
761
451
|
}
|
|
762
452
|
function reportOf(result) {
|
|
763
|
-
if (result.pdf !==
|
|
764
|
-
if (result.workbook !==
|
|
765
|
-
if (result.deck !==
|
|
766
|
-
if (result.json !==
|
|
767
|
-
if (result.xml !==
|
|
453
|
+
if (result.pdf !== void 0) return pdfReport(result.url, result.pdf);
|
|
454
|
+
if (result.workbook !== void 0) return workbookReport(result.url, result.workbook);
|
|
455
|
+
if (result.deck !== void 0) return deckReport(result.url, result.deck);
|
|
456
|
+
if (result.json !== void 0) return jsonReport(result.url, result.json);
|
|
457
|
+
if (result.xml !== void 0) return xmlReport(result.url, result.xml);
|
|
768
458
|
return probeReport(result.url, result.status, result.findings, result.observed, result.html);
|
|
769
459
|
}
|
|
770
460
|
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
461
|
+
// src/probe/index.ts
|
|
462
|
+
import { findData as findData2 } from "@opencraw/probe";
|
|
463
|
+
|
|
464
|
+
// src/main.ts
|
|
465
|
+
import { readFileSync } from "node:fs";
|
|
466
|
+
|
|
467
|
+
// src/run/run-recipes.use-case.ts
|
|
468
|
+
import { RecipeBindingError, RecipeSet, RecipeValidationError as RecipeValidationError2, createCrawler, diffOptionsFor, diffRecords as diffRecords2, fileSink as fileSink2, memorySink, readRecordsFile as readRecordsFile3, traceLine, variantLabel } from "@opencraw/core";
|
|
469
|
+
|
|
470
|
+
// src/run/load-for-run.use-case.ts
|
|
471
|
+
import { bindRecipeSet, parseInputRecipe, parseOutputRecipe as parseOutputRecipe2 } from "@opencraw/core";
|
|
472
|
+
|
|
473
|
+
// src/recipe-files/read-recipe-files.repository.ts
|
|
474
|
+
import { readRecipeSource } from "@opencraw/core";
|
|
780
475
|
async function readRecipeFiles(sources) {
|
|
781
|
-
const result = {
|
|
782
|
-
outputs: [],
|
|
783
|
-
inputs: [],
|
|
784
|
-
others: []
|
|
785
|
-
};
|
|
476
|
+
const result = { outputs: [], inputs: [], others: [] };
|
|
786
477
|
const documents = await readRecipeSource(sources);
|
|
787
478
|
for (const document of documents) {
|
|
788
479
|
const kind = document.content?.kind;
|
|
789
|
-
if (kind ===
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
});else if (kind === 'input') result.inputs.push({
|
|
793
|
-
path: document.source,
|
|
794
|
-
recipe: document.content
|
|
795
|
-
});else result.others.push(document.source);
|
|
480
|
+
if (kind === "output") result.outputs.push({ path: document.source, recipe: document.content });
|
|
481
|
+
else if (kind === "input") result.inputs.push({ path: document.source, recipe: document.content });
|
|
482
|
+
else result.others.push(document.source);
|
|
796
483
|
}
|
|
797
484
|
return result;
|
|
798
485
|
}
|
|
799
486
|
|
|
800
|
-
|
|
801
|
-
* Loads and binds the recipes `run` needs, filtered to the input ids in `only`
|
|
802
|
-
* when given.
|
|
803
|
-
*
|
|
804
|
-
* @param sources - Recipe files or directories (or recipes the MCP server was handed inline).
|
|
805
|
-
* @param only - Input recipe ids to keep; empty keeps all.
|
|
806
|
-
* @returns The bound set.
|
|
807
|
-
* @throws Error when there is not exactly one output recipe.
|
|
808
|
-
* @throws RecipeValidationError, RecipeBindingError on a bad recipe.
|
|
809
|
-
*/
|
|
487
|
+
// src/run/load-for-run.use-case.ts
|
|
810
488
|
async function loadForRun(sources, only) {
|
|
811
489
|
const files = await readRecipeFiles(sources);
|
|
812
490
|
if (files.outputs.length !== 1) throw new Error(`expected exactly one output recipe, found ${files.outputs.length}`);
|
|
813
|
-
const output =
|
|
814
|
-
const inputs = files.inputs.map(file => parseInputRecipe(file.recipe, file.path)).filter(input => only.length === 0 || only.includes(input.id));
|
|
491
|
+
const output = parseOutputRecipe2(files.outputs[0].recipe, files.outputs[0].path);
|
|
492
|
+
const inputs = files.inputs.map((file) => parseInputRecipe(file.recipe, file.path)).filter((input) => only.length === 0 || only.includes(input.id));
|
|
815
493
|
return bindRecipeSet(output, inputs);
|
|
816
494
|
}
|
|
817
495
|
|
|
818
|
-
|
|
819
|
-
* Crawls the recipes at `paths` and reports what happened.
|
|
820
|
-
*
|
|
821
|
-
* `--dry-run` caps every input at one record, attaches the scope each record
|
|
822
|
-
* was mapped from, and prints them side by side instead of writing them, so a
|
|
823
|
-
* recipe under construction can be checked without a full run.
|
|
824
|
-
*
|
|
825
|
-
* @param command - The parsed `run` command.
|
|
826
|
-
* @param terminal - Where records, the trace and errors go.
|
|
827
|
-
* @returns The exit code: 0, or 1 on a load failure or a recipe error.
|
|
828
|
-
*/
|
|
496
|
+
// src/run/run-recipes.use-case.ts
|
|
829
497
|
async function runRecipes(command, terminal) {
|
|
830
498
|
let set;
|
|
831
499
|
try {
|
|
@@ -835,18 +503,12 @@ async function runRecipes(command, terminal) {
|
|
|
835
503
|
return 1;
|
|
836
504
|
}
|
|
837
505
|
if (set.inputs.length === 0) {
|
|
838
|
-
terminal.err(command.only.length > 0 ? `no input recipe matches --only ${command.only.join(
|
|
506
|
+
terminal.err(command.only.length > 0 ? `no input recipe matches --only ${command.only.join(", ")}` : "no input recipes found");
|
|
839
507
|
return 1;
|
|
840
508
|
}
|
|
841
|
-
// The extension of --out names the format: one that names none is refused here, before the crawl, not after it.
|
|
842
509
|
let sink;
|
|
843
510
|
try {
|
|
844
|
-
sink = command.dryRun || command.out ===
|
|
845
|
-
append: command.append,
|
|
846
|
-
browser: {
|
|
847
|
-
executablePath: command.options.browserPath
|
|
848
|
-
}
|
|
849
|
-
});
|
|
511
|
+
sink = command.dryRun || command.out === void 0 ? memorySink() : fileSink2(command.out, { append: command.append, browser: { executablePath: command.options.browserPath } });
|
|
850
512
|
} catch (error) {
|
|
851
513
|
terminal.err(error instanceof Error ? error.message : String(error));
|
|
852
514
|
return 1;
|
|
@@ -857,9 +519,8 @@ async function runRecipes(command, terminal) {
|
|
|
857
519
|
const emitted = [];
|
|
858
520
|
try {
|
|
859
521
|
access = await resolveAccess(command.options);
|
|
860
|
-
plugins = command.options.plugins ===
|
|
861
|
-
|
|
862
|
-
previous = command.diff === undefined ? undefined : await readRecordsFile(command.diff);
|
|
522
|
+
plugins = command.options.plugins === void 0 ? void 0 : await loadPlugins(command.options.plugins);
|
|
523
|
+
previous = command.diff === void 0 ? void 0 : await readRecordsFile3(command.diff);
|
|
863
524
|
} catch (error) {
|
|
864
525
|
terminal.err(error instanceof Error ? error.message : String(error));
|
|
865
526
|
return 1;
|
|
@@ -870,9 +531,7 @@ async function runRecipes(command, terminal) {
|
|
|
870
531
|
throttle: throttleFor(command, access?.throttle),
|
|
871
532
|
profilesDir: command.profiles,
|
|
872
533
|
parallel: command.parallel,
|
|
873
|
-
retry: command.retries ===
|
|
874
|
-
attempts: command.retries
|
|
875
|
-
},
|
|
534
|
+
retry: command.retries === void 0 ? void 0 : { attempts: command.retries },
|
|
876
535
|
hooks: plugins?.hooks,
|
|
877
536
|
accessPlugins: plugins?.accessPlugins,
|
|
878
537
|
captchaSolvers: plugins?.captchaSolvers,
|
|
@@ -883,144 +542,100 @@ async function runRecipes(command, terminal) {
|
|
|
883
542
|
executablePath: command.options.browserPath,
|
|
884
543
|
ignoreHTTPSErrors: command.options.insecureTls
|
|
885
544
|
},
|
|
886
|
-
onEvent: event => {
|
|
887
|
-
if (previous !==
|
|
545
|
+
onEvent: (event) => {
|
|
546
|
+
if (previous !== void 0 && event.type === "record:emit") emitted.push(event.data);
|
|
888
547
|
onEvent(event, command, terminal);
|
|
889
548
|
}
|
|
890
549
|
});
|
|
891
550
|
try {
|
|
892
|
-
const inputs = command.dryRun ? set.inputs.map(input => ({
|
|
893
|
-
...input,
|
|
894
|
-
limits: {
|
|
895
|
-
...input.limits,
|
|
896
|
-
maxRecords: 1
|
|
897
|
-
}
|
|
898
|
-
})) : set.inputs;
|
|
551
|
+
const inputs = command.dryRun ? set.inputs.map((input) => ({ ...input, limits: { ...input.limits, maxRecords: 1 } })) : set.inputs;
|
|
899
552
|
const report = await crawler.run(new RecipeSet(set.output, inputs));
|
|
900
|
-
if (command.out ===
|
|
553
|
+
if (command.out === void 0 && !command.dryRun) for (const record of sink.records) terminal.out(JSON.stringify(record.data));
|
|
901
554
|
for (const recipe of report.recipes) {
|
|
902
|
-
terminal.err(`${recipe.recipeId}${recipe.variant ===
|
|
555
|
+
terminal.err(`${recipe.recipeId}${recipe.variant === void 0 ? "" : ` [${variantLabel(recipe.variant)}]`}: ${recipe.emitted} emitted, ${recipe.rejected} rejected, ${recipe.duplicates} duplicates, ${recipe.skipped} skipped, ${recipe.stepsSkipped > 0 ? `${recipe.stepsSkipped} steps skipped (see --trace), ` : ""}${recipe.pages} pages, ${recipe.durationMs} ms${recipe.error === void 0 ? "" : `, stopped: ${recipe.error}`}`);
|
|
903
556
|
}
|
|
904
|
-
if (!command.dryRun) terminal.err(`${report.records} records${report.sink.location ===
|
|
905
|
-
if (previous !==
|
|
906
|
-
const diff =
|
|
907
|
-
terminal.err(`compared with ${command.diff ??
|
|
557
|
+
if (!command.dryRun) terminal.err(`${report.records} records${report.sink.location === void 0 ? "" : ` written to ${report.sink.location}`}`);
|
|
558
|
+
if (previous !== void 0) {
|
|
559
|
+
const diff = diffRecords2(previous, emitted, diffOptionsFor(set.output));
|
|
560
|
+
terminal.err(`compared with ${command.diff ?? ""}:`);
|
|
908
561
|
for (const line of diffReport(diff)) terminal.err(` ${line}`);
|
|
909
|
-
if (command.changes !==
|
|
562
|
+
if (command.changes !== void 0) await writeChanges(command.changes, diff);
|
|
910
563
|
}
|
|
911
|
-
return report.recipes.some(recipe => recipe.error !==
|
|
564
|
+
return report.recipes.some((recipe) => recipe.error !== void 0) ? 1 : 0;
|
|
912
565
|
} finally {
|
|
913
566
|
await crawler.close();
|
|
914
567
|
}
|
|
915
568
|
}
|
|
916
|
-
/** The access config's throttle, with `--host-delay` / `--host-concurrency` over its defaults. */
|
|
917
569
|
function throttleFor(command, fromConfig) {
|
|
918
|
-
const {
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
} = command.throttle;
|
|
922
|
-
if (delayMs === undefined && concurrency === undefined) return fromConfig;
|
|
923
|
-
return {
|
|
924
|
-
...fromConfig,
|
|
925
|
-
...(delayMs !== undefined && {
|
|
926
|
-
delayMs
|
|
927
|
-
}),
|
|
928
|
-
...(concurrency !== undefined && {
|
|
929
|
-
concurrency
|
|
930
|
-
})
|
|
931
|
-
};
|
|
570
|
+
const { delayMs, concurrency } = command.throttle;
|
|
571
|
+
if (delayMs === void 0 && concurrency === void 0) return fromConfig;
|
|
572
|
+
return { ...fromConfig, ...delayMs !== void 0 && { delayMs }, ...concurrency !== void 0 && { concurrency } };
|
|
932
573
|
}
|
|
933
574
|
function onEvent(event, command, terminal) {
|
|
934
575
|
if (command.trace) {
|
|
935
576
|
const line = traceLine(event);
|
|
936
|
-
if (line !==
|
|
577
|
+
if (line !== void 0) terminal.err(line);
|
|
937
578
|
}
|
|
938
|
-
if (command.dryRun && event.type ===
|
|
579
|
+
if (command.dryRun && event.type === "record:emit") {
|
|
939
580
|
terminal.out(`--- ${event.recipeId} ---`);
|
|
940
581
|
terminal.out(`scope: ${JSON.stringify(event.scope, null, 2)}`);
|
|
941
582
|
terminal.out(`record: ${JSON.stringify(event.data, null, 2)}`);
|
|
942
583
|
}
|
|
943
|
-
if (command.dryRun && event.type ===
|
|
584
|
+
if (command.dryRun && event.type === "record:reject") terminal.out(`--- ${event.recipeId}: rejected (${event.field}: ${event.reason}) ---
|
|
585
|
+
scope: ${JSON.stringify(event.scope, null, 2)}`);
|
|
944
586
|
}
|
|
945
587
|
function loadErrorLines(error) {
|
|
946
|
-
if (error instanceof
|
|
947
|
-
if (error instanceof RecipeBindingError) return error.issues.map(issue => `${issue.recipeId}: ${issue.path}: ${issue.message}`);
|
|
588
|
+
if (error instanceof RecipeValidationError2) return error.issues.map((issue) => `${error.source}: ${issue.path === "" ? "(root)" : issue.path}: ${issue.message}`);
|
|
589
|
+
if (error instanceof RecipeBindingError) return error.issues.map((issue) => `${issue.recipeId}: ${issue.path}: ${issue.message}`);
|
|
948
590
|
return [error instanceof Error ? error.message : String(error)];
|
|
949
591
|
}
|
|
950
592
|
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
* delegates to its own `main`, which starts the server, prints the URL with
|
|
954
|
-
* the token, and keeps running until interrupted (Ctrl+C) — the cli's own
|
|
955
|
-
* call does not return before that. `@opencraw/studio` is an optional peer
|
|
956
|
-
* dependency (`peerDependenciesMeta`), never a hard one: the cli stays light
|
|
957
|
-
* for callers who only need to crawl. When it is not installed, this prints
|
|
958
|
-
* how to add it instead of letting a raw `ERR_MODULE_NOT_FOUND` stack trace
|
|
959
|
-
* through.
|
|
960
|
-
*
|
|
961
|
-
* @param folder - The recipe folder to open; `@opencraw/studio`'s own default (the current directory) when omitted.
|
|
962
|
-
* @param terminal - Where output goes.
|
|
963
|
-
* @param importStudio - How the module is loaded; the real dynamic `import()` by default, replaced in tests.
|
|
964
|
-
* @param pluginsFile - A hooks or plugins module (`--hooks`, issue #150), loaded here, on the command line's own authority, and handed to Studio: the server never loads code itself.
|
|
965
|
-
* @param load - How the module is read; `loadPlugins` unless a test says otherwise.
|
|
966
|
-
* @param open - A folder, recipe file or document (`--open`, issue #235): the printed link opens it straight away.
|
|
967
|
-
* @returns The exit code: 0 once the server has stopped, 1 when `@opencraw/studio` is missing.
|
|
968
|
-
*/
|
|
969
|
-
async function runStudio(folder, terminal, importStudio = () => import('@opencraw/studio'), pluginsFile, load = loadPlugins, open) {
|
|
593
|
+
// src/studio/run-studio.use-case.ts
|
|
594
|
+
async function runStudio(folder, terminal, importStudio = () => import("@opencraw/studio"), pluginsFile, load = loadPlugins, open) {
|
|
970
595
|
let studio;
|
|
971
596
|
try {
|
|
972
597
|
studio = await importStudio();
|
|
973
598
|
} catch (error) {
|
|
974
599
|
if (!isModuleNotFound(error)) throw error;
|
|
975
|
-
terminal.err(
|
|
600
|
+
terminal.err("the studio is not installed: run `npm install @opencraw/studio`, then try again");
|
|
976
601
|
return 1;
|
|
977
602
|
}
|
|
978
|
-
const plugins = pluginsFile ===
|
|
979
|
-
|
|
980
|
-
...(await load(pluginsFile))
|
|
981
|
-
};
|
|
982
|
-
await studio.main(folder === undefined ? [] : [folder], {
|
|
983
|
-
plugins,
|
|
984
|
-
open
|
|
985
|
-
});
|
|
603
|
+
const plugins = pluginsFile === void 0 ? void 0 : { source: pluginsFile, ...await load(pluginsFile) };
|
|
604
|
+
await studio.main(folder === void 0 ? [] : [folder], { plugins, open });
|
|
986
605
|
return 0;
|
|
987
606
|
}
|
|
988
607
|
function isModuleNotFound(error) {
|
|
989
|
-
return error instanceof Error &&
|
|
608
|
+
return error instanceof Error && "code" in error && error.code === "ERR_MODULE_NOT_FOUND";
|
|
990
609
|
}
|
|
991
610
|
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
*
|
|
995
|
-
* @param paths - Recipe files or directories.
|
|
996
|
-
* @param terminal - Where results go.
|
|
997
|
-
* @returns The exit code: 0 when the recipes bind cleanly, 1 otherwise.
|
|
998
|
-
*/
|
|
611
|
+
// src/validate/validate-recipes.use-case.ts
|
|
612
|
+
import { RecipeBindingError as RecipeBindingError2, RecipeValidationError as RecipeValidationError3, bindRecipeSet as bindRecipeSet2, parseInputRecipe as parseInputRecipe2, parseOutputRecipe as parseOutputRecipe3 } from "@opencraw/core";
|
|
999
613
|
async function validateRecipes(paths, terminal) {
|
|
1000
614
|
const files = await readRecipeFiles(paths);
|
|
1001
615
|
for (const path of files.others) terminal.err(`${path}: not a recipe ("kind" is missing or not "input"/"output")`);
|
|
1002
616
|
if (files.outputs.length === 0) {
|
|
1003
|
-
terminal.err(
|
|
617
|
+
terminal.err("no output recipe found");
|
|
1004
618
|
return 1;
|
|
1005
619
|
}
|
|
1006
620
|
if (files.outputs.length > 1) {
|
|
1007
|
-
terminal.err(`more than one output recipe: ${files.outputs.map(file => file.path).join(
|
|
621
|
+
terminal.err(`more than one output recipe: ${files.outputs.map((file) => file.path).join(", ")}`);
|
|
1008
622
|
return 1;
|
|
1009
623
|
}
|
|
1010
624
|
let ok = true;
|
|
1011
|
-
const output = parseOne(files.outputs[0].path, files.outputs[0].recipe,
|
|
1012
|
-
const inputs = files.inputs.flatMap(file => {
|
|
1013
|
-
const parsed = parseOne(file.path, file.recipe,
|
|
1014
|
-
return parsed ===
|
|
625
|
+
const output = parseOne(files.outputs[0].path, files.outputs[0].recipe, parseOutputRecipe3, terminal);
|
|
626
|
+
const inputs = files.inputs.flatMap((file) => {
|
|
627
|
+
const parsed = parseOne(file.path, file.recipe, parseInputRecipe2, terminal);
|
|
628
|
+
return parsed === void 0 ? [] : [parsed];
|
|
1015
629
|
});
|
|
1016
|
-
ok &&= output !==
|
|
1017
|
-
if (output ===
|
|
630
|
+
ok &&= output !== void 0 && inputs.length === files.inputs.length;
|
|
631
|
+
if (output === void 0) return 1;
|
|
1018
632
|
try {
|
|
1019
|
-
const set =
|
|
1020
|
-
terminal.out(`ok: ${set.output.id} <- ${set.inputs.map(input => input.id).join(
|
|
633
|
+
const set = bindRecipeSet2(output, inputs);
|
|
634
|
+
terminal.out(`ok: ${set.output.id} <- ${set.inputs.map((input) => input.id).join(", ") || "(no inputs)"}`);
|
|
1021
635
|
} catch (error) {
|
|
1022
636
|
ok = false;
|
|
1023
|
-
if (error instanceof
|
|
637
|
+
if (error instanceof RecipeBindingError2) for (const issue of error.issues) terminal.err(`${issue.recipeId}: ${issue.path}: ${issue.message}`);
|
|
638
|
+
else terminal.err(String(error));
|
|
1024
639
|
}
|
|
1025
640
|
return ok ? 0 : 1;
|
|
1026
641
|
}
|
|
@@ -1028,21 +643,14 @@ function parseOne(path, recipe, parse, terminal) {
|
|
|
1028
643
|
try {
|
|
1029
644
|
return parse(recipe, path);
|
|
1030
645
|
} catch (error) {
|
|
1031
|
-
if (error instanceof
|
|
1032
|
-
|
|
646
|
+
if (error instanceof RecipeValidationError3) for (const issue of error.issues) terminal.err(`${path}: ${issue.path}: ${issue.message}`);
|
|
647
|
+
else terminal.err(`${path}: ${String(error)}`);
|
|
648
|
+
return void 0;
|
|
1033
649
|
}
|
|
1034
650
|
}
|
|
1035
651
|
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
* Runs one cli invocation: parses argv, dispatches to a use case, writes
|
|
1039
|
-
* through `terminal`. Never throws; a bad command or a use case failure both
|
|
1040
|
-
* become a non-zero return.
|
|
1041
|
-
*
|
|
1042
|
-
* @param argv - Arguments after the program name.
|
|
1043
|
-
* @param terminal - Where output goes.
|
|
1044
|
-
* @returns The process exit code.
|
|
1045
|
-
*/
|
|
652
|
+
// src/main.ts
|
|
653
|
+
var VERSION = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
|
|
1046
654
|
async function main(argv, terminal) {
|
|
1047
655
|
let command;
|
|
1048
656
|
try {
|
|
@@ -1053,57 +661,63 @@ async function main(argv, terminal) {
|
|
|
1053
661
|
return 1;
|
|
1054
662
|
}
|
|
1055
663
|
switch (command.name) {
|
|
1056
|
-
case
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
case
|
|
1071
|
-
{
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
{
|
|
1083
|
-
return diffFiles(command, terminal);
|
|
1084
|
-
}
|
|
1085
|
-
case 'export':
|
|
1086
|
-
{
|
|
1087
|
-
return exportRecords(command, terminal);
|
|
1088
|
-
}
|
|
1089
|
-
case 'studio':
|
|
1090
|
-
{
|
|
1091
|
-
return runStudio(command.folder, terminal, undefined, command.plugins, undefined, command.open);
|
|
1092
|
-
}
|
|
664
|
+
case "help": {
|
|
665
|
+
terminal.out(USAGE);
|
|
666
|
+
return 0;
|
|
667
|
+
}
|
|
668
|
+
case "version": {
|
|
669
|
+
terminal.out(VERSION);
|
|
670
|
+
return 0;
|
|
671
|
+
}
|
|
672
|
+
case "validate": {
|
|
673
|
+
return validateRecipes(command.paths, terminal);
|
|
674
|
+
}
|
|
675
|
+
case "run": {
|
|
676
|
+
return runRecipes(command, terminal);
|
|
677
|
+
}
|
|
678
|
+
case "probe": {
|
|
679
|
+
return probePage(command.url, { browser: command.browser, ...command.options }, terminal);
|
|
680
|
+
}
|
|
681
|
+
case "diff": {
|
|
682
|
+
return diffFiles(command, terminal);
|
|
683
|
+
}
|
|
684
|
+
case "export": {
|
|
685
|
+
return exportRecords(command, terminal);
|
|
686
|
+
}
|
|
687
|
+
case "studio": {
|
|
688
|
+
return runStudio(command.folder, terminal, void 0, command.plugins, void 0, command.open);
|
|
689
|
+
}
|
|
1093
690
|
}
|
|
1094
691
|
}
|
|
1095
692
|
|
|
1096
|
-
|
|
693
|
+
// src/terminal/terminal.client.ts
|
|
1097
694
|
function processTerminal() {
|
|
1098
695
|
return {
|
|
1099
|
-
out: line => {
|
|
1100
|
-
process.stdout.write(`${line}
|
|
696
|
+
out: (line) => {
|
|
697
|
+
process.stdout.write(`${line}
|
|
698
|
+
`);
|
|
1101
699
|
},
|
|
1102
|
-
err: line => {
|
|
1103
|
-
process.stderr.write(`${line}
|
|
700
|
+
err: (line) => {
|
|
701
|
+
process.stderr.write(`${line}
|
|
702
|
+
`);
|
|
1104
703
|
}
|
|
1105
704
|
};
|
|
1106
705
|
}
|
|
1107
|
-
|
|
1108
|
-
|
|
706
|
+
export {
|
|
707
|
+
USAGE,
|
|
708
|
+
diffReport,
|
|
709
|
+
findData2 as findData,
|
|
710
|
+
loadForRun,
|
|
711
|
+
loadHooks,
|
|
712
|
+
loadPlugins,
|
|
713
|
+
main,
|
|
714
|
+
parseArguments,
|
|
715
|
+
probeReport,
|
|
716
|
+
probeUrl,
|
|
717
|
+
processTerminal,
|
|
718
|
+
readRecipeFiles,
|
|
719
|
+
resolveAccess,
|
|
720
|
+
runStudio,
|
|
721
|
+
writeChanges
|
|
722
|
+
};
|
|
1109
723
|
//# sourceMappingURL=index.esm.js.map
|