@scrappycoco/cli 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -5
- package/dist/index.js +109 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -5,11 +5,21 @@ Discover, run, and compare scraper capabilities from a terminal or automation en
|
|
|
5
5
|
Run the published package directly with `npx`:
|
|
6
6
|
|
|
7
7
|
```sh
|
|
8
|
-
npx @scrappycoco/cli
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
8
|
+
npx --yes @scrappycoco/cli setup
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
`setup` opens OAuth in the browser. New users can create an account in that
|
|
12
|
+
flow. After authentication, it installs the public Scrappycoco skill, verifies
|
|
13
|
+
the live catalog, and tells you when to reload or restart your agent.
|
|
14
|
+
|
|
15
|
+
Individual commands remain available:
|
|
16
|
+
|
|
17
|
+
```sh
|
|
18
|
+
npx --yes @scrappycoco/cli auth login
|
|
19
|
+
npx --yes @scrappycoco/cli scrapers list --available --json
|
|
20
|
+
npx --yes @scrappycoco/cli scrapers inspect web.extract_content --json
|
|
21
|
+
npx --yes @scrappycoco/cli scrapers run web.extract_content --file request.json --json
|
|
22
|
+
npx --yes @scrappycoco/cli scrapers compare web.extract_content --file request.json --json
|
|
13
23
|
```
|
|
14
24
|
|
|
15
25
|
From a repository checkout, use `npm ci`, `npm run build`, and
|
package/dist/index.js
CHANGED
|
@@ -367,6 +367,108 @@ function errorPayload(error) {
|
|
|
367
367
|
return { error: error.message, details: error.details ?? null };
|
|
368
368
|
}
|
|
369
369
|
|
|
370
|
+
// src/setup.ts
|
|
371
|
+
import { spawn } from "child_process";
|
|
372
|
+
var SKILL_SOURCE = "https://scrappycoco.ai";
|
|
373
|
+
var SKILL_INSTALL_ARGS = [
|
|
374
|
+
"--yes",
|
|
375
|
+
"skills",
|
|
376
|
+
"add",
|
|
377
|
+
SKILL_SOURCE,
|
|
378
|
+
"--skill",
|
|
379
|
+
"scrappycoco",
|
|
380
|
+
"-g",
|
|
381
|
+
"-y"
|
|
382
|
+
];
|
|
383
|
+
function npxCommand() {
|
|
384
|
+
return process.platform === "win32" ? "npx.cmd" : "npx";
|
|
385
|
+
}
|
|
386
|
+
function skillInstallInvocation() {
|
|
387
|
+
return { command: npxCommand(), args: SKILL_INSTALL_ARGS };
|
|
388
|
+
}
|
|
389
|
+
function runCommand(command, args) {
|
|
390
|
+
return new Promise((resolve, reject) => {
|
|
391
|
+
const child = spawn(command, [...args], {
|
|
392
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
393
|
+
windowsHide: true
|
|
394
|
+
});
|
|
395
|
+
child.stdout.on("data", (chunk) => process.stderr.write(chunk));
|
|
396
|
+
child.stderr.on("data", (chunk) => process.stderr.write(chunk));
|
|
397
|
+
child.on("error", (error) => reject(
|
|
398
|
+
new CliError(`Could not start the skill installer: ${error.message}`, EXIT.api)
|
|
399
|
+
));
|
|
400
|
+
child.on("close", (code) => {
|
|
401
|
+
if (code === 0) {
|
|
402
|
+
resolve();
|
|
403
|
+
return;
|
|
404
|
+
}
|
|
405
|
+
reject(new CliError(
|
|
406
|
+
`Skill installation failed with exit code ${code ?? "unknown"}.`,
|
|
407
|
+
EXIT.api
|
|
408
|
+
));
|
|
409
|
+
});
|
|
410
|
+
});
|
|
411
|
+
}
|
|
412
|
+
async function installSkill(runner = runCommand) {
|
|
413
|
+
const invocation = skillInstallInvocation();
|
|
414
|
+
await runner(invocation.command, invocation.args);
|
|
415
|
+
}
|
|
416
|
+
function defaultDependencies(apiUrl) {
|
|
417
|
+
const client2 = new ApiClient(apiUrl);
|
|
418
|
+
return {
|
|
419
|
+
hasApiKey: () => Boolean(process.env.SCRAPPYCOCO_API_KEY),
|
|
420
|
+
clearRefreshToken,
|
|
421
|
+
loadRefreshToken,
|
|
422
|
+
login,
|
|
423
|
+
installSkill,
|
|
424
|
+
listAvailableScrapers: () => client2.get("/scrapers?available_only=true")
|
|
425
|
+
};
|
|
426
|
+
}
|
|
427
|
+
async function performSetup(options, dependencies = defaultDependencies(options.apiUrl)) {
|
|
428
|
+
const usingApiKey = dependencies.hasApiKey();
|
|
429
|
+
let authentication = usingApiKey ? "api_key" : "oauth";
|
|
430
|
+
let credentialStorage = usingApiKey ? "environment" : "existing";
|
|
431
|
+
const hadStoredToken = !usingApiKey && Boolean(await dependencies.loadRefreshToken());
|
|
432
|
+
if (!usingApiKey && !hadStoredToken) {
|
|
433
|
+
const authenticated = await dependencies.login({
|
|
434
|
+
apiUrl: options.apiUrl,
|
|
435
|
+
noBrowser: options.noBrowser
|
|
436
|
+
});
|
|
437
|
+
authentication = "oauth";
|
|
438
|
+
credentialStorage = authenticated.storage;
|
|
439
|
+
}
|
|
440
|
+
let catalog;
|
|
441
|
+
try {
|
|
442
|
+
catalog = await dependencies.listAvailableScrapers();
|
|
443
|
+
} catch (error) {
|
|
444
|
+
if (usingApiKey || !hadStoredToken || !(error instanceof CliError) || error.exitCode !== EXIT.auth) {
|
|
445
|
+
throw error;
|
|
446
|
+
}
|
|
447
|
+
await dependencies.clearRefreshToken();
|
|
448
|
+
const authenticated = await dependencies.login({
|
|
449
|
+
apiUrl: options.apiUrl,
|
|
450
|
+
noBrowser: options.noBrowser
|
|
451
|
+
});
|
|
452
|
+
credentialStorage = authenticated.storage;
|
|
453
|
+
catalog = await dependencies.listAvailableScrapers();
|
|
454
|
+
}
|
|
455
|
+
await dependencies.installSkill();
|
|
456
|
+
return {
|
|
457
|
+
authenticated: true,
|
|
458
|
+
authentication,
|
|
459
|
+
credential_storage: credentialStorage,
|
|
460
|
+
skill: {
|
|
461
|
+
installed: true,
|
|
462
|
+
source: SKILL_SOURCE
|
|
463
|
+
},
|
|
464
|
+
verification: {
|
|
465
|
+
catalog_reachable: true,
|
|
466
|
+
available_capabilities: catalog.length
|
|
467
|
+
},
|
|
468
|
+
next_step: "Reload or restart your agent, then ask it to list available Scrappycoco scrapers."
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
|
|
370
472
|
// src/index.ts
|
|
371
473
|
var packageMetadata = JSON.parse(
|
|
372
474
|
readFileSync(new URL("../package.json", import.meta.url), "utf8")
|
|
@@ -382,6 +484,13 @@ function client(command) {
|
|
|
382
484
|
function collect(value, previous) {
|
|
383
485
|
return [...previous, value];
|
|
384
486
|
}
|
|
487
|
+
program.command("setup").description("Authenticate, install the Scrappycoco skill, and verify the connection").option("--no-browser", "print the authorization URL without opening it").action(async (options, command) => {
|
|
488
|
+
const result = await performSetup({
|
|
489
|
+
apiUrl: globals(command).apiUrl,
|
|
490
|
+
noBrowser: options.browser === false
|
|
491
|
+
});
|
|
492
|
+
await emit(result, globals(command).json || false);
|
|
493
|
+
});
|
|
385
494
|
function splitScraperId(value) {
|
|
386
495
|
const separator = value.indexOf(".");
|
|
387
496
|
if (separator <= 0 || separator === value.length - 1) {
|