@scrappycoco/cli 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -2
- package/dist/index.js +9 -3
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -56,8 +56,9 @@ From a repository checkout, use `npm ci`, `npm run build`, and
|
|
|
56
56
|
`node dist/index.js <command>` instead.
|
|
57
57
|
|
|
58
58
|
Discover is optional. Use Run directly when the current capability, provider,
|
|
59
|
-
and native options are already clear. The CLI does not expose legacy
|
|
60
|
-
|
|
59
|
+
and native options are already clear. The CLI does not expose legacy assistant,
|
|
60
|
+
agent, general workflow, or scheduler commands. For repeated checks, callers
|
|
61
|
+
schedule ordinary Runs and pass each returned cursor into the next Run.
|
|
61
62
|
|
|
62
63
|
Node.js 20 or newer is required. Interactive use authenticates with Clerk OAuth Authorization Code + PKCE. CI can set `SCRAPPYCOCO_API_KEY`.
|
|
63
64
|
|
|
@@ -72,6 +73,20 @@ Use `--json` for machine-readable responses. Execution commands support
|
|
|
72
73
|
poll for completion. When `--output` is used, records go to the file and a
|
|
73
74
|
compact execution summary remains on stdout.
|
|
74
75
|
|
|
76
|
+
Canonical Run request files use `providers` as a top-level array and
|
|
77
|
+
`provider_options` keyed by provider ID:
|
|
78
|
+
|
|
79
|
+
```json
|
|
80
|
+
{
|
|
81
|
+
"input": {"query": "unified API for scraping"},
|
|
82
|
+
"providers": ["exa"],
|
|
83
|
+
"provider_options": {"exa": {}}
|
|
84
|
+
}
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Do not use a singular `provider` field. With flags, use repeatable
|
|
88
|
+
`--provider <id>` instead.
|
|
89
|
+
|
|
75
90
|
For a long run, add `--detach` to return a job ID immediately, then finish with
|
|
76
91
|
`scrappycoco jobs wait <job-id> --output results.json`. Use
|
|
77
92
|
`scrappycoco jobs get <job-id>` for a single status check or
|
package/dist/index.js
CHANGED
|
@@ -1119,6 +1119,12 @@ function splitScraperId(value) {
|
|
|
1119
1119
|
}
|
|
1120
1120
|
async function requestPayload(options, scraperId) {
|
|
1121
1121
|
const fromFile = options.file ? await readJsonFile(options.file) : {};
|
|
1122
|
+
if ("provider" in fromFile) {
|
|
1123
|
+
throw new CliError(
|
|
1124
|
+
'Run request files use "providers": ["provider_id"] (plural), or pass the repeatable --provider flag.',
|
|
1125
|
+
EXIT.usage
|
|
1126
|
+
);
|
|
1127
|
+
}
|
|
1122
1128
|
const { source, capability } = splitScraperId(scraperId);
|
|
1123
1129
|
const input = options.input ? parseJsonObject(options.input, "input JSON") : fromFile.input;
|
|
1124
1130
|
if (!input) throw new CliError("Provide input through --file or --input.", EXIT.usage);
|
|
@@ -1219,7 +1225,7 @@ function executionSummary(response) {
|
|
|
1219
1225
|
return route;
|
|
1220
1226
|
});
|
|
1221
1227
|
}
|
|
1222
|
-
for (const field of ["cursor", "
|
|
1228
|
+
for (const field of ["cursor", "snapshot"]) {
|
|
1223
1229
|
if (summary[field] === null) delete summary[field];
|
|
1224
1230
|
}
|
|
1225
1231
|
return summary;
|
|
@@ -1391,7 +1397,7 @@ program.command("doctor").description("Check the CLI, authentication, installed
|
|
|
1391
1397
|
if (!ok) process.exitCode = authentication.configured ? EXIT.api : EXIT.auth;
|
|
1392
1398
|
});
|
|
1393
1399
|
var scrapers = program.command("scrapers", { hidden: true }).description("Legacy scraper commands");
|
|
1394
|
-
scrapers.command("list").option("--source <source>", "filter by web, x, reddit, or filings").option("--provider <provider>", "filter by provider implementation").option("--available", "only include available providers").action(async (options, command) => {
|
|
1400
|
+
scrapers.command("list").option("--source <source>", "filter by web, x, reddit, instagram, tiktok, or filings").option("--provider <provider>", "filter by provider implementation").option("--available", "only include available providers").action(async (options, command) => {
|
|
1395
1401
|
const query = new URLSearchParams();
|
|
1396
1402
|
if (options.source) query.set("source", options.source);
|
|
1397
1403
|
if (options.provider) query.set("provider", options.provider);
|
|
@@ -1409,7 +1415,7 @@ scrapers.command("inspect <scraper-id>").action(async (scraperId, _options, comm
|
|
|
1409
1415
|
);
|
|
1410
1416
|
});
|
|
1411
1417
|
var catalog = program.command("catalog").description("Inspect capabilities, provider schemas, options, and pricing");
|
|
1412
|
-
catalog.command("list").option("--source <source>", "filter by web, x, reddit, or filings").option("--provider <provider>", "filter by provider implementation").option("--available", "only include configured providers; this is not a live health check").option("--full", "include input/output and provider option schemas in JSON output").action(async (options, command) => {
|
|
1418
|
+
catalog.command("list").option("--source <source>", "filter by web, x, reddit, instagram, tiktok, or filings").option("--provider <provider>", "filter by provider implementation").option("--available", "only include configured providers; this is not a live health check").option("--full", "include input/output and provider option schemas in JSON output").action(async (options, command) => {
|
|
1413
1419
|
const query = new URLSearchParams();
|
|
1414
1420
|
if (options.source) query.set("source", options.source);
|
|
1415
1421
|
if (options.provider) query.set("provider", options.provider);
|