@scrappycoco/cli 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -7
- package/dist/index.js +114 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -2,15 +2,24 @@
|
|
|
2
2
|
|
|
3
3
|
Discover, run, and compare scraper capabilities from a terminal or automation environment.
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
`npm view @scrappycoco/cli version`, then run:
|
|
5
|
+
Run the published package directly with `npx`:
|
|
7
6
|
|
|
8
7
|
```sh
|
|
9
|
-
npx @scrappycoco/cli
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
8
|
+
npx --yes @scrappycoco/cli setup
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
`setup` opens OAuth in the browser. New users can create an account in that
|
|
12
|
+
flow. After authentication, it installs the public Scrappycoco skill, verifies
|
|
13
|
+
the live catalog, and tells you when to reload or restart your agent.
|
|
14
|
+
|
|
15
|
+
Individual commands remain available:
|
|
16
|
+
|
|
17
|
+
```sh
|
|
18
|
+
npx --yes @scrappycoco/cli auth login
|
|
19
|
+
npx --yes @scrappycoco/cli scrapers list --available --json
|
|
20
|
+
npx --yes @scrappycoco/cli scrapers inspect web.extract_content --json
|
|
21
|
+
npx --yes @scrappycoco/cli scrapers run web.extract_content --file request.json --json
|
|
22
|
+
npx --yes @scrappycoco/cli scrapers compare web.extract_content --file request.json --json
|
|
14
23
|
```
|
|
15
24
|
|
|
16
25
|
From a repository checkout, use `npm ci`, `npm run build`, and
|
package/dist/index.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
// src/index.ts
|
|
4
4
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
5
|
+
import { readFileSync } from "fs";
|
|
5
6
|
import { Command, CommanderError, Option } from "commander";
|
|
6
7
|
|
|
7
8
|
// src/auth.ts
|
|
@@ -366,9 +367,114 @@ function errorPayload(error) {
|
|
|
366
367
|
return { error: error.message, details: error.details ?? null };
|
|
367
368
|
}
|
|
368
369
|
|
|
370
|
+
// src/setup.ts
|
|
371
|
+
import { spawn } from "child_process";
|
|
372
|
+
var SKILL_SOURCE = "https://scrappycoco.ai";
|
|
373
|
+
var SKILL_INSTALL_ARGS = [
|
|
374
|
+
"--yes",
|
|
375
|
+
"skills",
|
|
376
|
+
"add",
|
|
377
|
+
SKILL_SOURCE,
|
|
378
|
+
"--skill",
|
|
379
|
+
"scrappycoco",
|
|
380
|
+
"-g",
|
|
381
|
+
"-y"
|
|
382
|
+
];
|
|
383
|
+
function npxCommand() {
|
|
384
|
+
return process.platform === "win32" ? "npx.cmd" : "npx";
|
|
385
|
+
}
|
|
386
|
+
function skillInstallInvocation() {
|
|
387
|
+
return { command: npxCommand(), args: SKILL_INSTALL_ARGS };
|
|
388
|
+
}
|
|
389
|
+
function runCommand(command, args) {
|
|
390
|
+
return new Promise((resolve, reject) => {
|
|
391
|
+
const child = spawn(command, [...args], {
|
|
392
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
393
|
+
windowsHide: true
|
|
394
|
+
});
|
|
395
|
+
child.stdout.on("data", (chunk) => process.stderr.write(chunk));
|
|
396
|
+
child.stderr.on("data", (chunk) => process.stderr.write(chunk));
|
|
397
|
+
child.on("error", (error) => reject(
|
|
398
|
+
new CliError(`Could not start the skill installer: ${error.message}`, EXIT.api)
|
|
399
|
+
));
|
|
400
|
+
child.on("close", (code) => {
|
|
401
|
+
if (code === 0) {
|
|
402
|
+
resolve();
|
|
403
|
+
return;
|
|
404
|
+
}
|
|
405
|
+
reject(new CliError(
|
|
406
|
+
`Skill installation failed with exit code ${code ?? "unknown"}.`,
|
|
407
|
+
EXIT.api
|
|
408
|
+
));
|
|
409
|
+
});
|
|
410
|
+
});
|
|
411
|
+
}
|
|
412
|
+
async function installSkill(runner = runCommand) {
|
|
413
|
+
const invocation = skillInstallInvocation();
|
|
414
|
+
await runner(invocation.command, invocation.args);
|
|
415
|
+
}
|
|
416
|
+
function defaultDependencies(apiUrl) {
|
|
417
|
+
const client2 = new ApiClient(apiUrl);
|
|
418
|
+
return {
|
|
419
|
+
hasApiKey: () => Boolean(process.env.SCRAPPYCOCO_API_KEY),
|
|
420
|
+
clearRefreshToken,
|
|
421
|
+
loadRefreshToken,
|
|
422
|
+
login,
|
|
423
|
+
installSkill,
|
|
424
|
+
listAvailableScrapers: () => client2.get("/scrapers?available_only=true")
|
|
425
|
+
};
|
|
426
|
+
}
|
|
427
|
+
async function performSetup(options, dependencies = defaultDependencies(options.apiUrl)) {
|
|
428
|
+
const usingApiKey = dependencies.hasApiKey();
|
|
429
|
+
let authentication = usingApiKey ? "api_key" : "oauth";
|
|
430
|
+
let credentialStorage = usingApiKey ? "environment" : "existing";
|
|
431
|
+
const hadStoredToken = !usingApiKey && Boolean(await dependencies.loadRefreshToken());
|
|
432
|
+
if (!usingApiKey && !hadStoredToken) {
|
|
433
|
+
const authenticated = await dependencies.login({
|
|
434
|
+
apiUrl: options.apiUrl,
|
|
435
|
+
noBrowser: options.noBrowser
|
|
436
|
+
});
|
|
437
|
+
authentication = "oauth";
|
|
438
|
+
credentialStorage = authenticated.storage;
|
|
439
|
+
}
|
|
440
|
+
let catalog;
|
|
441
|
+
try {
|
|
442
|
+
catalog = await dependencies.listAvailableScrapers();
|
|
443
|
+
} catch (error) {
|
|
444
|
+
if (usingApiKey || !hadStoredToken || !(error instanceof CliError) || error.exitCode !== EXIT.auth) {
|
|
445
|
+
throw error;
|
|
446
|
+
}
|
|
447
|
+
await dependencies.clearRefreshToken();
|
|
448
|
+
const authenticated = await dependencies.login({
|
|
449
|
+
apiUrl: options.apiUrl,
|
|
450
|
+
noBrowser: options.noBrowser
|
|
451
|
+
});
|
|
452
|
+
credentialStorage = authenticated.storage;
|
|
453
|
+
catalog = await dependencies.listAvailableScrapers();
|
|
454
|
+
}
|
|
455
|
+
await dependencies.installSkill();
|
|
456
|
+
return {
|
|
457
|
+
authenticated: true,
|
|
458
|
+
authentication,
|
|
459
|
+
credential_storage: credentialStorage,
|
|
460
|
+
skill: {
|
|
461
|
+
installed: true,
|
|
462
|
+
source: SKILL_SOURCE
|
|
463
|
+
},
|
|
464
|
+
verification: {
|
|
465
|
+
catalog_reachable: true,
|
|
466
|
+
available_capabilities: catalog.length
|
|
467
|
+
},
|
|
468
|
+
next_step: "Reload or restart your agent, then ask it to list available Scrappycoco scrapers."
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
|
|
369
472
|
// src/index.ts
|
|
473
|
+
var packageMetadata = JSON.parse(
|
|
474
|
+
readFileSync(new URL("../package.json", import.meta.url), "utf8")
|
|
475
|
+
);
|
|
370
476
|
var program = new Command();
|
|
371
|
-
program.name("scrappycoco").description("Discover, run, and compare scraper capabilities through one API").version(
|
|
477
|
+
program.name("scrappycoco").description("Discover, run, and compare scraper capabilities through one API").version(packageMetadata.version).option("--json", "emit machine-readable JSON to stdout").option("--api-url <url>", "API base URL", process.env.SCRAPPYCOCO_API_URL || "https://api.scrappycoco.ai").showHelpAfterError().exitOverride();
|
|
372
478
|
function globals(command) {
|
|
373
479
|
return command.optsWithGlobals();
|
|
374
480
|
}
|
|
@@ -378,6 +484,13 @@ function client(command) {
|
|
|
378
484
|
function collect(value, previous) {
|
|
379
485
|
return [...previous, value];
|
|
380
486
|
}
|
|
487
|
+
program.command("setup").description("Authenticate, install the Scrappycoco skill, and verify the connection").option("--no-browser", "print the authorization URL without opening it").action(async (options, command) => {
|
|
488
|
+
const result = await performSetup({
|
|
489
|
+
apiUrl: globals(command).apiUrl,
|
|
490
|
+
noBrowser: options.browser === false
|
|
491
|
+
});
|
|
492
|
+
await emit(result, globals(command).json || false);
|
|
493
|
+
});
|
|
381
494
|
function splitScraperId(value) {
|
|
382
495
|
const separator = value.indexOf(".");
|
|
383
496
|
if (separator <= 0 || separator === value.length - 1) {
|