@scrappycoco/cli 0.4.2 → 0.5.0

package/README.md

@@ -1,8 +1,9 @@
 # Scrappycoco CLI
 
-Discover configurations and run scraper capabilities from a terminal or
-automation environment. The calling AI agent owns all provider and result
-judgment.
+Run external-data capabilities from a terminal or automation environment.
+Scrappycoco is deterministic infrastructure, not an AI agent: all LLM
+reasoning, provider decisions, and result judgment stay in the user's agentic
+client.
 
 Run the published package directly with `npx`:
 
@@ -41,9 +42,10 @@ Individual commands remain available:
 
 ```sh
 npx --yes @scrappycoco/cli@latest auth login
+npx --yes @scrappycoco/cli@latest doctor --json
 npx --yes @scrappycoco/cli@latest catalog list --available --json
 npx --yes @scrappycoco/cli@latest catalog inspect web.extract_content --json
-npx --yes @scrappycoco/cli@latest run web.extract_content --file request.json --json
+npx --yes @scrappycoco/cli@latest run web.extract_content --file request.json --output results.json --json
 npx --yes @scrappycoco/cli@latest discover --file discovery.json --json
 npx --yes @scrappycoco/cli@latest discover --id DISCOVERY_ID --test --input '{"url":"https://example.com"}' --json
 npx --yes @scrappycoco/cli@latest discover --id DISCOVERY_ID --finalize --json
@@ -67,13 +69,22 @@ Use `--json` for machine-readable responses. Execution commands support
 `--format json|jsonl|csv` with `--output`, provider-native
 `--provider-options`, batch `--concurrency`, and an explicit
 `--idempotency-key` for safe identical retries. They submit durable jobs and
-poll for completion; set `SCRAPPYCOCO_JOB_TIMEOUT_MS` to change the 20-minute
-local wait. If a command times out while its job continues, inspect it with
-`scrappycoco jobs get <job-id>`.
-
-When provider or configuration choice is uncertain, define 2 to 5 named
-`candidates` in a Discover route. Each candidate has its own `id`, `provider`,
-and `options`, so one discovery can test both `zyte-http` with
+poll for completion. When `--output` is used, records go to the file and a
+compact execution summary remains on stdout.
+
+For a long run, add `--detach` to return a job ID immediately, then finish with
+`scrappycoco jobs wait <job-id> --output results.json`. Use
+`scrappycoco jobs get <job-id>` for a single status check. Set
+`SCRAPPYCOCO_JOB_TIMEOUT_MS` to change the default 20-minute foreground wait.
+Run `scrappycoco doctor --json` only when authentication, connectivity, or
+installation is unclear.
+
+When provider or configuration choice is uncertain, define at least one named
+candidate for each special provider configuration under `candidates` in a
+Discover route. Scrappycoco automatically adds one default candidate for every
+available provider omitted from the draft, so the comparison remains
+exhaustive. Each candidate has its own `id`, `provider`, and `options`, so one
+discovery can test both `zyte-http` with
 `{"browser_html":false}` and `zyte-browser` with
 `{"browser_html":true}` against the same input.
 

package/dist/index.js

@@ -554,8 +554,11 @@ var ApiClient = class {
   post(path, body, key = randomUUID2()) {
     return this.request("POST", path, body, { "Idempotency-Key": key });
   }
-  async postJob(path, body, key = randomUUID2()) {
-    const submitted = await this.post(path, body, key);
+  submitJob(path, body, key = randomUUID2()) {
+    return this.post(path, body, key);
+  }
+  async waitForJob(submittedOrJobId) {
+    const submitted = typeof submittedOrJobId === "string" ? await this.get(`/jobs/${encodeURIComponent(submittedOrJobId)}`) : submittedOrJobId;
     const timeoutMs = positiveInteger2(process.env.SCRAPPYCOCO_JOB_TIMEOUT_MS, DEFAULT_JOB_TIMEOUT_MS);
     const initialDelayMs = positiveInteger2(
       process.env.SCRAPPYCOCO_JOB_POLL_INITIAL_MS,
@@ -594,7 +597,10 @@ var ApiClient = class {
     if (!job.result || typeof job.result !== "object" || Array.isArray(job.result)) {
       throw new CliError(`Job ${job.job_id} completed without a result.`, EXIT.api, job);
     }
-    return job.result;
+    return { ...job.result, job_id: job.job_id };
+  }
+  async postJob(path, body, key = randomUUID2()) {
+    return this.waitForJob(await this.submitJob(path, body, key));
   }
   patch(path, body) {
     return this.request("PATCH", path, body);
@@ -883,6 +889,12 @@ async function loadInstalledDigest() {
     return null;
   }
 }
+async function installedSkillDiagnostics() {
+  return {
+    installed: await isSkillInstalled(),
+    digest: await loadInstalledDigest()
+  };
+}
 async function writePrivateJson2(path, value) {
   await mkdir2(dirname2(path), { recursive: true, mode: 448 });
   const temporary = `${path}.${process.pid}.${randomUUID3()}.tmp`;
@@ -1033,6 +1045,12 @@ function client(command) {
 function collect(value, previous) {
   return [...previous, value];
 }
+function selectFields(value, fields) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return {};
+  return Object.fromEntries(
+    fields.filter((field) => field in value).map((field) => [field, value[field]])
+  );
+}
 program.command("setup").description("Authenticate, install the Scrappycoco skill, and verify the connection").option("--no-browser", "print the authorization URL without opening it").option("--manual-callback", "paste the final callback URL in this terminal for remote/headless login").action(async (options, command) => {
   const result = await performSetup({
     apiUrl: globals(command).apiUrl,
@@ -1074,12 +1092,141 @@ async function emitExecution(response, options, command) {
   const jsonMode = globals(command).json || false;
   if (options.output) {
     await emit(response, jsonMode, options.output, formatRecords(records, options.format));
+    const summary = { ...response };
+    delete summary.records;
+    delete summary.items;
+    delete summary.normalized_schema;
+    summary.truncated_count = records.filter((record) => {
+      const metadata = record.metadata;
+      return metadata && typeof metadata === "object" && !Array.isArray(metadata) && metadata.truncated === true;
+    }).length;
+    if (summary.usage) {
+      summary.usage = selectFields(summary.usage, [
+        "billing_status",
+        "payg_charge_usd_exact",
+        "provider_cost_usd_exact",
+        "unresolved_cost_count"
+      ]);
+    }
+    if (Array.isArray(summary.attempts)) {
+      summary.attempts = summary.attempts.map((attempt) => selectFields(attempt, [
+        "provider",
+        "status",
+        "result_count",
+        "latency_ms",
+        "estimated_cost_usd",
+        "error"
+      ]));
+    }
+    const providerResults = summary.provider_results;
+    if (providerResults && typeof providerResults === "object" && !Array.isArray(providerResults)) {
+      const counts = Object.fromEntries(
+        Object.entries(providerResults).map(([provider, values]) => [
+          provider,
+          Array.isArray(values) ? values.length : 0
+        ])
+      );
+      if (Object.keys(counts).length) summary.provider_result_counts = counts;
+      delete summary.provider_results;
+    }
+    if (Array.isArray(response.items)) {
+      summary.item_count = response.items.length;
+      summary.failed_item_count = response.items.filter(
+        (item) => item && typeof item === "object" && item.status === "failed"
+      ).length;
+    }
+    if (Array.isArray(summary.routes)) {
+      summary.routes = summary.routes.map((value) => {
+        if (!value || typeof value !== "object" || Array.isArray(value)) return value;
+        const route = { ...value };
+        const routeResults = route.provider_results;
+        if (routeResults && typeof routeResults === "object" && !Array.isArray(routeResults)) {
+          const counts = Object.fromEntries(
+            Object.entries(routeResults).map(([provider, values]) => [
+              provider,
+              Array.isArray(values) ? values.length : 0
+            ])
+          );
+          if (Object.keys(counts).length) route.provider_result_counts = counts;
+          delete route.provider_results;
+        }
+        if (Array.isArray(route.attempts)) {
+          route.attempts = route.attempts.map((attempt) => selectFields(attempt, [
+            "provider",
+            "status",
+            "result_count",
+            "latency_ms",
+            "estimated_cost_usd",
+            "error"
+          ]));
+        }
+        return route;
+      });
+    }
+    for (const field of ["cursor", "monitor"]) {
+      if (summary[field] === null) delete summary[field];
+    }
+    summary.output = {
+      path: options.output,
+      format: options.format,
+      record_count: records.length
+    };
+    await emit(summary, jsonMode);
     process.stderr.write(`Saved ${records.length} records to ${options.output}
 `);
     return;
   }
   await emit(response, jsonMode);
 }
+async function executeQueuedJob(path, payload, options, command) {
+  if (options.detach && options.output) {
+    throw new CliError(
+      "Do not combine --detach with --output. Use `scrappycoco jobs wait JOB_ID --output PATH`.",
+      EXIT.usage
+    );
+  }
+  const apiClient = client(command);
+  const idempotencyKey = options.idempotencyKey || randomUUID4();
+  if (options.detach) {
+    const job = await apiClient.submitJob(path, payload, idempotencyKey);
+    await emit(
+      {
+        ...job,
+        next_command: `scrappycoco jobs wait ${job.job_id}`
+      },
+      globals(command).json || false
+    );
+    return;
+  }
+  await emitExecution(
+    await apiClient.postJob(
+      path,
+      payload,
+      idempotencyKey
+    ),
+    options,
+    command
+  );
+}
+function formatCatalog(items) {
+  const grouped = /* @__PURE__ */ new Map();
+  for (const item of items) {
+    const source = String(item.source || item.id.split(".", 1)[0] || "other");
+    grouped.set(source, [...grouped.get(source) || [], item]);
+  }
+  const lines = [];
+  for (const [source, capabilities] of grouped) {
+    lines.push(`${source.toUpperCase()} (${capabilities.length})`);
+    for (const capability of capabilities) {
+      const label = typeof capability.label === "string" ? capability.label : capability.id;
+      const description = typeof capability.description === "string" ? ` \u2014 ${capability.description}` : "";
+      lines.push(`  ${capability.id}  ${label}${description}`);
+    }
+    lines.push("");
+  }
+  return `${lines.join("\n").trimEnd()}
+`;
+}
 var auth = program.command("auth").description("Manage Clerk OAuth credentials");
 auth.command("login").option("--no-browser", "print the authorization URL without opening it").option("--manual-callback", "paste the final callback URL in this terminal for remote/headless login").action(async (options, command) => {
   const result = await login({
@@ -1124,6 +1271,48 @@ auth.command("logout").action(async (_options, command) => {
   await clearRefreshToken2();
   await emit({ authenticated: false }, globals(command).json || false);
 });
+program.command("doctor").description("Check the CLI, authentication, installed skill, API, and live catalog").action(async (_options, command) => {
+  const usingApiKey = Boolean(process.env.SCRAPPYCOCO_API_KEY);
+  const usingOAuth = !usingApiKey && Boolean(await loadRefreshToken());
+  const authentication = {
+    configured: usingApiKey || usingOAuth,
+    method: usingApiKey ? "api_key" : usingOAuth ? "oauth" : null
+  };
+  let catalog2 = [];
+  let catalogError = null;
+  if (authentication.configured) {
+    try {
+      catalog2 = await client(command).get("/scrapers?available_only=true");
+    } catch (error) {
+      catalogError = error instanceof Error ? error.message : String(error);
+    }
+  }
+  const skill = await installedSkillDiagnostics();
+  const ok = authentication.configured && catalogError === null;
+  await emit(
+    {
+      ok,
+      cli: {
+        version: packageMetadata.version,
+        node: process.version
+      },
+      authentication,
+      api: {
+        url: globals(command).apiUrl,
+        reachable: authentication.configured && catalogError === null
+      },
+      catalog: {
+        reachable: authentication.configured && catalogError === null,
+        available_capabilities: catalog2.length,
+        error: catalogError
+      },
+      skill,
+      next_action: ok ? "Scrappycoco is ready." : authentication.configured ? "Check the API connection, then run doctor again." : "Run `scrappycoco setup`."
+    },
+    globals(command).json || false
+  );
+  if (!ok) process.exitCode = authentication.configured ? EXIT.api : EXIT.auth;
+});
 var scrapers = program.command("scrapers", { hidden: true }).description("Legacy scraper commands");
 scrapers.command("list").option("--source <source>", "filter by web, x, reddit, or filings").option("--provider <provider>", "filter by provider implementation").option("--available", "only include available providers").action(async (options, command) => {
   const query = new URLSearchParams();
@@ -1148,10 +1337,9 @@ catalog.command("list").option("--source <source>", "filter by web, x, reddit, o
   if (options.source) query.set("source", options.source);
   if (options.provider) query.set("provider", options.provider);
   if (options.available) query.set("available_only", "true");
-  await emit(
-    await client(command).get(`/scrapers${query.size ? `?${query}` : ""}`),
-    globals(command).json || false
-  );
+  const items = await client(command).get(`/scrapers${query.size ? `?${query}` : ""}`);
+  if (globals(command).json) await emit(items, true);
+  else process.stdout.write(formatCatalog(items));
 });
 catalog.command("inspect <capability-id>").action(async (capabilityId, _options, command) => {
   const { source, capability } = splitScraperId(capabilityId);
@@ -1176,17 +1364,17 @@ function executionCommand(name) {
 }
 executionCommand("run");
 executionCommand("compare");
-program.command("run [capability-id]").description("Run a capability directly; discovery is optional").option("--config <discovery-id>", "run a finalized multi-step configuration").option("-f, --file <path>", "canonical request JSON file").option("--input <json>", "capability input JSON; use url or urls for web.extract_content").option("--provider <id>", "provider ID; repeat for an ordered fallback waterfall", collect, []).option("--provider-options <json>", "provider-native options keyed by provider ID").option("--concurrency <number>", "batch concurrency (default 3, maximum 10)").option("--limit <number>", "maximum records").option("--idempotency-key <key>", "stable retry key").option("--retry-failed <run-id>", "retry only failed URLs from a partial batch run").addOption(new Option("--format <format>", "output record format").choices(["json", "jsonl", "csv"]).default("json")).option("-o, --output <path>", "write records to a file").action(async (capabilityId, options, command) => {
+program.command("run [capability-id]").description("Run a capability directly; use Discover only when configuration is uncertain").option("--config <discovery-id>", "run a finalized multi-step configuration").option("-f, --file <path>", "canonical request JSON file").option("--input <json>", "capability input JSON; use url or urls for web.extract_content").option("--provider <id>", "provider ID; repeat for an ordered fallback waterfall", collect, []).option("--provider-options <json>", "provider-native options keyed by provider ID").option("--concurrency <number>", "batch concurrency (default 3, maximum 10)").option("--limit <number>", "maximum records").option("--idempotency-key <key>", "stable retry key").option("--retry-failed <run-id>", "retry only failed URLs from a partial batch run").option("--detach", "queue the job and return immediately").addOption(new Option("--format <format>", "output record format").choices(["json", "jsonl", "csv"]).default("json")).option("-o, --output <path>", "write records to a file").action(async (capabilityId, options, command) => {
   if (options.retryFailed) {
     if (capabilityId || options.config) {
       throw new CliError("Do not combine --retry-failed with a capability ID or --config.", EXIT.usage);
     }
-    const response2 = await client(command).postJob(
+    await executeQueuedJob(
       `/runs/${encodeURIComponent(options.retryFailed)}/retry-failed`,
       {},
-      options.idempotencyKey || randomUUID4()
+      options,
+      command
     );
-    await emitExecution(response2, options, command);
     return;
   }
   if (options.config) {
@@ -1195,16 +1383,16 @@ program.command("run [capability-id]").description("Run a capability directly; d
     }
     const fromFile = options.file ? await readJsonFile(options.file) : {};
     const input = options.input ? parseJsonObject(options.input, "runtime input JSON") : fromFile.input || {};
-    const response2 = await client(command).postJob(
+    await executeQueuedJob(
       `/discoveries/${encodeURIComponent(options.config)}/jobs`,
       {
         ...fromFile,
         input,
         limit: Number(options.limit ?? fromFile.limit ?? 25)
       },
-      options.idempotencyKey || randomUUID4()
+      options,
+      command
     );
-    await emitExecution(response2, options, command);
     return;
   }
   if (!capabilityId) {
@@ -1212,12 +1400,12 @@ program.command("run [capability-id]").description("Run a capability directly; d
   }
   const payload = await requestPayload(options, capabilityId);
   if (payload.limit === void 0) payload.limit = 10;
-  const response = await client(command).postJob(
+  await executeQueuedJob(
     "/scrapers/jobs",
     payload,
-    options.idempotencyKey || randomUUID4()
+    options,
+    command
   );
-  await emitExecution(response, options, command);
 });
 var jobs = program.command("jobs").description("Inspect durable queued jobs");
 jobs.command("get <job-id>").action(async (jobId, _options, command) => {
@@ -1226,6 +1414,13 @@ jobs.command("get <job-id>").action(async (jobId, _options, command) => {
     globals(command).json || false
   );
 });
+jobs.command("wait <job-id>").description("Wait for a queued job and return its result").addOption(new Option("--format <format>", "output record format").choices(["json", "jsonl", "csv"]).default("json")).option("-o, --output <path>", "write records to a file").action(async (jobId, options, command) => {
+  await emitExecution(
+    await client(command).waitForJob(jobId),
+    options,
+    command
+  );
+});
 var providers = program.command("providers", { hidden: true }).description("Legacy provider commands");
 providers.command("list").option("--available", "only include available provider-capability routes").action(async (options, command) => {
   await emit(
@@ -1278,7 +1473,7 @@ discoveries.command("delete <discovery-id>").requiredOption("--yes", "confirm pe
   await client(command).delete(`/discoveries/${encodeURIComponent(discoveryId)}`);
   await emit({ deleted: true, discovery_id: discoveryId }, globals(command).json || false);
 });
-program.command("discover").description("Save, sample-test, or finalize an agent-authored configuration").option("-f, --file <path>", "create from agent-authored discovery JSON").option("--id <discovery-id>", "existing discovery ID").option("--test", "approve and run a paid sample test").option("--input <json>", "sample runtime input JSON").option("--update <path>", "replace fields or configuration from agent-authored JSON").option("--finalize", "mark the current explicit configuration finalized").option("--idempotency-key <key>", "stable sample retry key").action(async (options, command) => {
+program.command("discover").description("Save, sample-test, or finalize an agent-authored configuration").option("-f, --file <path>", "create from agent-authored discovery JSON").option("--id <discovery-id>", "existing discovery ID").option("--test", "run a representative provider sample test").option("--input <json>", "sample runtime input JSON").option("--update <path>", "replace fields or configuration from agent-authored JSON").option("--finalize", "mark the current explicit configuration finalized").option("--idempotency-key <key>", "stable sample retry key").action(async (options, command) => {
   const selected = Number(Boolean(options.file)) + Number(Boolean(options.test)) + Number(Boolean(options.update)) + Number(Boolean(options.finalize));
   if (selected !== 1) {
     throw new CliError(

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scrappycoco/cli",
-  "version": "0.4.2",
+  "version": "0.5.0",
   "description": "CLI for Scrappycoco scraper discovery and execution",
   "type": "module",
   "bin": {