@scrappycoco/cli 0.4.1 → 0.5.0

package/README.md

@@ -1,13 +1,14 @@
 # Scrappycoco CLI
 
-Discover configurations and run scraper capabilities from a terminal or
-automation environment. The calling AI agent owns all provider and result
-judgment.
+Run external-data capabilities from a terminal or automation environment.
+Scrappycoco is deterministic infrastructure, not an AI agent: all LLM
+reasoning, provider decisions, and result judgment stay in the user's agentic
+client.
 
 Run the published package directly with `npx`:
 
 ```sh
-npx --yes @scrappycoco/cli setup
+npx --yes @scrappycoco/cli@latest setup
 ```
 
 `setup` opens OAuth in the browser. New users can create an account in that
@@ -16,12 +17,19 @@ the live catalog, and tells you when to reload or restart your agent. Browser
 authorization is not complete until the terminal confirms that the credential
 was saved and the catalog check succeeded.
 
+Before every ordinary command, the CLI compares the installed Scrappycoco skill
+with the digest in the hash-verified public feed. When the digest changes, it
+automatically refreshes the skill and prints a reload/restart notice to stderr.
+Release-check or installer failures do not block the requested CLI command.
+CLI-only users without an installed skill are not modified. Set
+`SCRAPPYCOCO_DISABLE_SKILL_AUTO_UPDATE=1` to opt out.
+
 When the CLI runs on a remote or headless host whose `127.0.0.1` is not the
 browser's localhost, keep the command running and use the manual callback
 flow:
 
 ```sh
-npx --yes @scrappycoco/cli setup --no-browser --manual-callback
+npx --yes @scrappycoco/cli@latest setup --no-browser --manual-callback
 ```
 
 Open the printed authorization URL, then paste the browser's final callback
@@ -33,14 +41,15 @@ URL.
 Individual commands remain available:
 
 ```sh
-npx --yes @scrappycoco/cli auth login
-npx --yes @scrappycoco/cli catalog list --available --json
-npx --yes @scrappycoco/cli catalog inspect web.extract_content --json
-npx --yes @scrappycoco/cli run web.extract_content --file request.json --json
-npx --yes @scrappycoco/cli discover --file discovery.json --json
-npx --yes @scrappycoco/cli discover --id DISCOVERY_ID --test --input '{"url":"https://example.com"}' --json
-npx --yes @scrappycoco/cli discover --id DISCOVERY_ID --finalize --json
-npx --yes @scrappycoco/cli run --config DISCOVERY_ID --input '{}' --json
+npx --yes @scrappycoco/cli@latest auth login
+npx --yes @scrappycoco/cli@latest doctor --json
+npx --yes @scrappycoco/cli@latest catalog list --available --json
+npx --yes @scrappycoco/cli@latest catalog inspect web.extract_content --json
+npx --yes @scrappycoco/cli@latest run web.extract_content --file request.json --output results.json --json
+npx --yes @scrappycoco/cli@latest discover --file discovery.json --json
+npx --yes @scrappycoco/cli@latest discover --id DISCOVERY_ID --test --input '{"url":"https://example.com"}' --json
+npx --yes @scrappycoco/cli@latest discover --id DISCOVERY_ID --finalize --json
+npx --yes @scrappycoco/cli@latest run --config DISCOVERY_ID --input '{}' --json
 ```
 
 From a repository checkout, use `npm ci`, `npm run build`, and
@@ -60,16 +69,28 @@ Use `--json` for machine-readable responses. Execution commands support
 `--format json|jsonl|csv` with `--output`, provider-native
 `--provider-options`, batch `--concurrency`, and an explicit
 `--idempotency-key` for safe identical retries. They submit durable jobs and
-poll for completion; set `SCRAPPYCOCO_JOB_TIMEOUT_MS` to change the 20-minute
-local wait. If a command times out while its job continues, inspect it with
-`scrappycoco jobs get <job-id>`.
-
-When provider or configuration choice is uncertain, define 2 to 5 named
-`candidates` in a Discover route. Each candidate has its own `id`, `provider`,
-and `options`, so one discovery can test both `zyte-http` with
+poll for completion. When `--output` is used, records go to the file and a
+compact execution summary remains on stdout.
+
+For a long run, add `--detach` to return a job ID immediately, then finish with
+`scrappycoco jobs wait <job-id> --output results.json`. Use
+`scrappycoco jobs get <job-id>` for a single status check. Set
+`SCRAPPYCOCO_JOB_TIMEOUT_MS` to change the default 20-minute foreground wait.
+Run `scrappycoco doctor --json` only when authentication, connectivity, or
+installation is unclear.
+
+When provider or configuration choice is uncertain, define at least one named
+candidate for each special provider configuration under `candidates` in a
+Discover route. Scrappycoco automatically adds one default candidate for every
+available provider omitted from the draft, so the comparison remains
+exhaustive. Each candidate has its own `id`, `provider`, and `options`, so one
+discovery can test both `zyte-http` with
 `{"browser_html":false}` and `zyte-browser` with
 `{"browser_html":true}` against the same input.
 
+The distributed skill includes a Discovery workbook with complete direct-Run,
+comparison, update, finalization, fallback, and audit-handoff examples.
+
 Use `scrappycoco --help` for the complete command reference. See the
 [Scrappycoco API documentation](https://scrappycoco.ai/docs) for the public
 contract.

package/dist/index.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 
 // src/index.ts
-import { randomUUID as randomUUID3 } from "crypto";
+import { randomUUID as randomUUID4 } from "crypto";
 import { readFileSync } from "fs";
 import { Command, CommanderError, Option } from "commander";
 
@@ -554,8 +554,11 @@ var ApiClient = class {
   post(path, body, key = randomUUID2()) {
     return this.request("POST", path, body, { "Idempotency-Key": key });
   }
-  async postJob(path, body, key = randomUUID2()) {
-    const submitted = await this.post(path, body, key);
+  submitJob(path, body, key = randomUUID2()) {
+    return this.post(path, body, key);
+  }
+  async waitForJob(submittedOrJobId) {
+    const submitted = typeof submittedOrJobId === "string" ? await this.get(`/jobs/${encodeURIComponent(submittedOrJobId)}`) : submittedOrJobId;
     const timeoutMs = positiveInteger2(process.env.SCRAPPYCOCO_JOB_TIMEOUT_MS, DEFAULT_JOB_TIMEOUT_MS);
     const initialDelayMs = positiveInteger2(
       process.env.SCRAPPYCOCO_JOB_POLL_INITIAL_MS,
@@ -594,7 +597,10 @@ var ApiClient = class {
     if (!job.result || typeof job.result !== "object" || Array.isArray(job.result)) {
       throw new CliError(`Job ${job.job_id} completed without a result.`, EXIT.api, job);
     }
-    return job.result;
+    return { ...job.result, job_id: job.job_id };
+  }
+  async postJob(path, body, key = randomUUID2()) {
+    return this.waitForJob(await this.submitJob(path, body, key));
   }
   patch(path, body) {
     return this.request("PATCH", path, body);
@@ -671,7 +677,7 @@ import { determineAgent } from "@vercel/detect-agent";
 var SKILL_SOURCE = "https://scrappycoco.ai";
 var SKILL_INSTALL_ARGS = [
   "--yes",
-  "skills",
+  "skills@latest",
   "add",
   SKILL_SOURCE,
   "--skill",
@@ -710,7 +716,7 @@ function skillInstallInvocation(agent = "universal", npmExecPath = process.env.n
       args: [
         npmExecPath,
         "dlx",
-        "skills",
+        "skills@latest",
         ...SKILL_INSTALL_ARGS.slice(2),
         "--agent",
         agent
@@ -839,6 +845,191 @@ async function performSetup(options, dependencies = defaultDependencies(options.
   };
 }
 
+// src/skill-update.ts
+import { randomUUID as randomUUID3 } from "crypto";
+import { access, chmod as chmod2, mkdir as mkdir2, open as open3, readFile as readFile3, rename as rename2, rm as rm2, stat as stat2, writeFile as writeFile3 } from "fs/promises";
+import { homedir as homedir2 } from "os";
+import { dirname as dirname2, join as join2 } from "path";
+var SKILL_NAME = "scrappycoco";
+var SKILL_INDEX_URL = `${SKILL_SOURCE}/.well-known/agent-skills/index.json`;
+var SKILL_UPDATE_TIMEOUT_MS = 3e3;
+var SKILL_UPDATE_LOCK_STALE_MS = 5 * 6e4;
+var SHA256_DIGEST = /^sha256:[a-f0-9]{64}$/i;
+function skillReleaseStatePath() {
+  return join2(dirname2(fallbackCredentialPath()), "skill-release.json");
+}
+function skillUpdateLockPath() {
+  return `${skillReleaseStatePath()}.lock`;
+}
+function installedSkillCandidates() {
+  return [
+    join2(homedir2(), ".agents", "skills", SKILL_NAME, "SKILL.md"),
+    join2(homedir2(), ".codex", "skills", SKILL_NAME, "SKILL.md"),
+    join2(homedir2(), ".claude", "skills", SKILL_NAME, "SKILL.md"),
+    join2(homedir2(), ".cursor", "skills", SKILL_NAME, "SKILL.md")
+  ];
+}
+async function isSkillInstalled() {
+  for (const path of installedSkillCandidates()) {
+    try {
+      await access(path);
+      return true;
+    } catch {
+    }
+  }
+  return false;
+}
+async function loadInstalledDigest() {
+  try {
+    const state = JSON.parse(
+      await readFile3(skillReleaseStatePath(), "utf8")
+    );
+    return typeof state.digest === "string" && SHA256_DIGEST.test(state.digest) ? state.digest : null;
+  } catch {
+    return null;
+  }
+}
+async function installedSkillDiagnostics() {
+  return {
+    installed: await isSkillInstalled(),
+    digest: await loadInstalledDigest()
+  };
+}
+async function writePrivateJson2(path, value) {
+  await mkdir2(dirname2(path), { recursive: true, mode: 448 });
+  const temporary = `${path}.${process.pid}.${randomUUID3()}.tmp`;
+  try {
+    await writeFile3(temporary, JSON.stringify(value), { mode: 384 });
+    if (process.platform !== "win32") await chmod2(temporary, 384);
+    await rename2(temporary, path);
+    if (process.platform !== "win32") await chmod2(path, 384);
+  } finally {
+    await rm2(temporary, { force: true });
+  }
+}
+async function saveInstalledDigest(digest) {
+  await writePrivateJson2(skillReleaseStatePath(), {
+    digest,
+    updated_at: (/* @__PURE__ */ new Date()).toISOString()
+  });
+}
+async function fetchPublishedSkillDigest(fetcher = fetch) {
+  const response = await fetcher(SKILL_INDEX_URL, {
+    headers: { accept: "application/json" },
+    signal: AbortSignal.timeout(SKILL_UPDATE_TIMEOUT_MS)
+  });
+  if (!response.ok) {
+    throw new Error(`Skill release check returned HTTP ${response.status}.`);
+  }
+  const index = await response.json();
+  const entry = index.skills?.find((skill) => skill.name === SKILL_NAME);
+  if (!entry || typeof entry.digest !== "string" || !SHA256_DIGEST.test(entry.digest)) {
+    throw new Error("Skill release index did not contain a valid Scrappycoco digest.");
+  }
+  return entry.digest.toLowerCase();
+}
+function defaultDependencies2() {
+  return {
+    isInstalled: isSkillInstalled,
+    loadInstalledDigest,
+    fetchPublishedDigest: fetchPublishedSkillDigest,
+    install: installSkill,
+    saveInstalledDigest
+  };
+}
+async function checkAndInstallSkillUpdate(dependencies = defaultDependencies2()) {
+  if (!await dependencies.isInstalled()) return { status: "not_installed" };
+  let publishedDigest;
+  try {
+    publishedDigest = await dependencies.fetchPublishedDigest();
+  } catch {
+    return { status: "unavailable" };
+  }
+  if (await dependencies.loadInstalledDigest() === publishedDigest) {
+    return { status: "current" };
+  }
+  try {
+    await dependencies.install();
+  } catch (error) {
+    return {
+      status: "failed",
+      error: error instanceof Error ? error : new Error(String(error))
+    };
+  }
+  let stateSaved = true;
+  try {
+    await dependencies.saveInstalledDigest(publishedDigest);
+  } catch {
+    stateSaved = false;
+  }
+  return { status: "updated", digest: publishedDigest, state_saved: stateSaved };
+}
+async function withSkillUpdateLock(operation) {
+  const path = skillUpdateLockPath();
+  await mkdir2(dirname2(path), { recursive: true, mode: 448 });
+  let handle;
+  try {
+    handle = await open3(path, "wx", 384);
+  } catch (error) {
+    if (error.code !== "EEXIST") {
+      return { status: "unavailable" };
+    }
+    try {
+      const lock = await stat2(path);
+      if (Date.now() - lock.mtimeMs <= SKILL_UPDATE_LOCK_STALE_MS) {
+        return { status: "busy" };
+      }
+      await rm2(path, { force: true });
+      handle = await open3(path, "wx", 384);
+    } catch {
+      return { status: "busy" };
+    }
+  }
+  try {
+    await handle.writeFile(JSON.stringify({ pid: process.pid, created_at: Date.now() }));
+    return await operation();
+  } finally {
+    await handle.close();
+    await rm2(path, { force: true });
+  }
+}
+function autoUpdateDisabled() {
+  return ["1", "true", "yes"].includes(
+    (process.env.SCRAPPYCOCO_DISABLE_SKILL_AUTO_UPDATE || "").toLowerCase()
+  );
+}
+async function autoUpdateInstalledSkill() {
+  if (autoUpdateDisabled()) return { status: "disabled" };
+  let result;
+  try {
+    result = await withSkillUpdateLock(() => checkAndInstallSkillUpdate());
+  } catch {
+    return { status: "unavailable" };
+  }
+  if (result.status === "updated") {
+    process.stderr.write(
+      "Updated the Scrappycoco skill. Reload or restart the agent to use the new instructions.\n"
+    );
+    if (!result.state_saved) {
+      process.stderr.write(
+        "Warning: the skill update succeeded, but its release state could not be saved.\n"
+      );
+    }
+  } else if (result.status === "failed") {
+    process.stderr.write(
+      `Warning: could not update the Scrappycoco skill; continuing with the installed copy: ${result.error.message}
+`
+    );
+  }
+  return result;
+}
+async function rememberInstalledSkillRelease() {
+  try {
+    await saveInstalledDigest(await fetchPublishedSkillDigest());
+  } catch {
+  }
+}
+
 // src/index.ts
 var packageMetadata = JSON.parse(
   readFileSync(new URL("../package.json", import.meta.url), "utf8")
@@ -854,14 +1045,25 @@ function client(command) {
 function collect(value, previous) {
   return [...previous, value];
 }
+function selectFields(value, fields) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return {};
+  return Object.fromEntries(
+    fields.filter((field) => field in value).map((field) => [field, value[field]])
+  );
+}
 program.command("setup").description("Authenticate, install the Scrappycoco skill, and verify the connection").option("--no-browser", "print the authorization URL without opening it").option("--manual-callback", "paste the final callback URL in this terminal for remote/headless login").action(async (options, command) => {
   const result = await performSetup({
     apiUrl: globals(command).apiUrl,
     noBrowser: options.browser === false,
     manualCallback: options.manualCallback
   });
+  await rememberInstalledSkillRelease();
   await emit(result, globals(command).json || false);
 });
+program.hook("preAction", async (_thisCommand, actionCommand) => {
+  if (actionCommand.name() === "setup") return;
+  await autoUpdateInstalledSkill();
+});
 function splitScraperId(value) {
   const separator = value.indexOf(".");
   if (separator <= 0 || separator === value.length - 1) {
@@ -890,12 +1092,141 @@ async function emitExecution(response, options, command) {
   const jsonMode = globals(command).json || false;
   if (options.output) {
     await emit(response, jsonMode, options.output, formatRecords(records, options.format));
+    const summary = { ...response };
+    delete summary.records;
+    delete summary.items;
+    delete summary.normalized_schema;
+    summary.truncated_count = records.filter((record) => {
+      const metadata = record.metadata;
+      return metadata && typeof metadata === "object" && !Array.isArray(metadata) && metadata.truncated === true;
+    }).length;
+    if (summary.usage) {
+      summary.usage = selectFields(summary.usage, [
+        "billing_status",
+        "payg_charge_usd_exact",
+        "provider_cost_usd_exact",
+        "unresolved_cost_count"
+      ]);
+    }
+    if (Array.isArray(summary.attempts)) {
+      summary.attempts = summary.attempts.map((attempt) => selectFields(attempt, [
+        "provider",
+        "status",
+        "result_count",
+        "latency_ms",
+        "estimated_cost_usd",
+        "error"
+      ]));
+    }
+    const providerResults = summary.provider_results;
+    if (providerResults && typeof providerResults === "object" && !Array.isArray(providerResults)) {
+      const counts = Object.fromEntries(
+        Object.entries(providerResults).map(([provider, values]) => [
+          provider,
+          Array.isArray(values) ? values.length : 0
+        ])
+      );
+      if (Object.keys(counts).length) summary.provider_result_counts = counts;
+      delete summary.provider_results;
+    }
+    if (Array.isArray(response.items)) {
+      summary.item_count = response.items.length;
+      summary.failed_item_count = response.items.filter(
+        (item) => item && typeof item === "object" && item.status === "failed"
+      ).length;
+    }
+    if (Array.isArray(summary.routes)) {
+      summary.routes = summary.routes.map((value) => {
+        if (!value || typeof value !== "object" || Array.isArray(value)) return value;
+        const route = { ...value };
+        const routeResults = route.provider_results;
+        if (routeResults && typeof routeResults === "object" && !Array.isArray(routeResults)) {
+          const counts = Object.fromEntries(
+            Object.entries(routeResults).map(([provider, values]) => [
+              provider,
+              Array.isArray(values) ? values.length : 0
+            ])
+          );
+          if (Object.keys(counts).length) route.provider_result_counts = counts;
+          delete route.provider_results;
+        }
+        if (Array.isArray(route.attempts)) {
+          route.attempts = route.attempts.map((attempt) => selectFields(attempt, [
+            "provider",
+            "status",
+            "result_count",
+            "latency_ms",
+            "estimated_cost_usd",
+            "error"
+          ]));
+        }
+        return route;
+      });
+    }
+    for (const field of ["cursor", "monitor"]) {
+      if (summary[field] === null) delete summary[field];
+    }
+    summary.output = {
+      path: options.output,
+      format: options.format,
+      record_count: records.length
+    };
+    await emit(summary, jsonMode);
     process.stderr.write(`Saved ${records.length} records to ${options.output}
 `);
     return;
   }
   await emit(response, jsonMode);
 }
+async function executeQueuedJob(path, payload, options, command) {
+  if (options.detach && options.output) {
+    throw new CliError(
+      "Do not combine --detach with --output. Use `scrappycoco jobs wait JOB_ID --output PATH`.",
+      EXIT.usage
+    );
+  }
+  const apiClient = client(command);
+  const idempotencyKey = options.idempotencyKey || randomUUID4();
+  if (options.detach) {
+    const job = await apiClient.submitJob(path, payload, idempotencyKey);
+    await emit(
+      {
+        ...job,
+        next_command: `scrappycoco jobs wait ${job.job_id}`
+      },
+      globals(command).json || false
+    );
+    return;
+  }
+  await emitExecution(
+    await apiClient.postJob(
+      path,
+      payload,
+      idempotencyKey
+    ),
+    options,
+    command
+  );
+}
+function formatCatalog(items) {
+  const grouped = /* @__PURE__ */ new Map();
+  for (const item of items) {
+    const source = String(item.source || item.id.split(".", 1)[0] || "other");
+    grouped.set(source, [...grouped.get(source) || [], item]);
+  }
+  const lines = [];
+  for (const [source, capabilities] of grouped) {
+    lines.push(`${source.toUpperCase()} (${capabilities.length})`);
+    for (const capability of capabilities) {
+      const label = typeof capability.label === "string" ? capability.label : capability.id;
+      const description = typeof capability.description === "string" ? ` \u2014 ${capability.description}` : "";
+      lines.push(`  ${capability.id}  ${label}${description}`);
+    }
+    lines.push("");
+  }
+  return `${lines.join("\n").trimEnd()}
+`;
+}
 var auth = program.command("auth").description("Manage Clerk OAuth credentials");
 auth.command("login").option("--no-browser", "print the authorization URL without opening it").option("--manual-callback", "paste the final callback URL in this terminal for remote/headless login").action(async (options, command) => {
   const result = await login({
@@ -940,6 +1271,48 @@ auth.command("logout").action(async (_options, command) => {
   await clearRefreshToken2();
   await emit({ authenticated: false }, globals(command).json || false);
 });
+program.command("doctor").description("Check the CLI, authentication, installed skill, API, and live catalog").action(async (_options, command) => {
+  const usingApiKey = Boolean(process.env.SCRAPPYCOCO_API_KEY);
+  const usingOAuth = !usingApiKey && Boolean(await loadRefreshToken());
+  const authentication = {
+    configured: usingApiKey || usingOAuth,
+    method: usingApiKey ? "api_key" : usingOAuth ? "oauth" : null
+  };
+  let catalog2 = [];
+  let catalogError = null;
+  if (authentication.configured) {
+    try {
+      catalog2 = await client(command).get("/scrapers?available_only=true");
+    } catch (error) {
+      catalogError = error instanceof Error ? error.message : String(error);
+    }
+  }
+  const skill = await installedSkillDiagnostics();
+  const ok = authentication.configured && catalogError === null;
+  await emit(
+    {
+      ok,
+      cli: {
+        version: packageMetadata.version,
+        node: process.version
+      },
+      authentication,
+      api: {
+        url: globals(command).apiUrl,
+        reachable: authentication.configured && catalogError === null
+      },
+      catalog: {
+        reachable: authentication.configured && catalogError === null,
+        available_capabilities: catalog2.length,
+        error: catalogError
+      },
+      skill,
+      next_action: ok ? "Scrappycoco is ready." : authentication.configured ? "Check the API connection, then run doctor again." : "Run `scrappycoco setup`."
+    },
+    globals(command).json || false
+  );
+  if (!ok) process.exitCode = authentication.configured ? EXIT.api : EXIT.auth;
+});
 var scrapers = program.command("scrapers", { hidden: true }).description("Legacy scraper commands");
 scrapers.command("list").option("--source <source>", "filter by web, x, reddit, or filings").option("--provider <provider>", "filter by provider implementation").option("--available", "only include available providers").action(async (options, command) => {
   const query = new URLSearchParams();
@@ -964,10 +1337,9 @@ catalog.command("list").option("--source <source>", "filter by web, x, reddit, o
   if (options.source) query.set("source", options.source);
   if (options.provider) query.set("provider", options.provider);
   if (options.available) query.set("available_only", "true");
-  await emit(
-    await client(command).get(`/scrapers${query.size ? `?${query}` : ""}`),
-    globals(command).json || false
-  );
+  const items = await client(command).get(`/scrapers${query.size ? `?${query}` : ""}`);
+  if (globals(command).json) await emit(items, true);
+  else process.stdout.write(formatCatalog(items));
 });
 catalog.command("inspect <capability-id>").action(async (capabilityId, _options, command) => {
   const { source, capability } = splitScraperId(capabilityId);
@@ -985,24 +1357,24 @@ function executionCommand(name) {
     const response = await client(command).postJob(
       name === "run" ? "/scrapers/jobs" : "/scrapers/compare/jobs",
       payload,
-      options.idempotencyKey || randomUUID3()
+      options.idempotencyKey || randomUUID4()
     );
     await emitExecution(response, options, command);
   });
 }
 executionCommand("run");
 executionCommand("compare");
-program.command("run [capability-id]").description("Run a capability directly; discovery is optional").option("--config <discovery-id>", "run a finalized multi-step configuration").option("-f, --file <path>", "canonical request JSON file").option("--input <json>", "capability input JSON; use url or urls for web.extract_content").option("--provider <id>", "provider ID; repeat for an ordered fallback waterfall", collect, []).option("--provider-options <json>", "provider-native options keyed by provider ID").option("--concurrency <number>", "batch concurrency (default 3, maximum 10)").option("--limit <number>", "maximum records").option("--idempotency-key <key>", "stable retry key").option("--retry-failed <run-id>", "retry only failed URLs from a partial batch run").addOption(new Option("--format <format>", "output record format").choices(["json", "jsonl", "csv"]).default("json")).option("-o, --output <path>", "write records to a file").action(async (capabilityId, options, command) => {
+program.command("run [capability-id]").description("Run a capability directly; use Discover only when configuration is uncertain").option("--config <discovery-id>", "run a finalized multi-step configuration").option("-f, --file <path>", "canonical request JSON file").option("--input <json>", "capability input JSON; use url or urls for web.extract_content").option("--provider <id>", "provider ID; repeat for an ordered fallback waterfall", collect, []).option("--provider-options <json>", "provider-native options keyed by provider ID").option("--concurrency <number>", "batch concurrency (default 3, maximum 10)").option("--limit <number>", "maximum records").option("--idempotency-key <key>", "stable retry key").option("--retry-failed <run-id>", "retry only failed URLs from a partial batch run").option("--detach", "queue the job and return immediately").addOption(new Option("--format <format>", "output record format").choices(["json", "jsonl", "csv"]).default("json")).option("-o, --output <path>", "write records to a file").action(async (capabilityId, options, command) => {
   if (options.retryFailed) {
     if (capabilityId || options.config) {
       throw new CliError("Do not combine --retry-failed with a capability ID or --config.", EXIT.usage);
     }
-    const response2 = await client(command).postJob(
+    await executeQueuedJob(
       `/runs/${encodeURIComponent(options.retryFailed)}/retry-failed`,
       {},
-      options.idempotencyKey || randomUUID3()
+      options,
+      command
     );
-    await emitExecution(response2, options, command);
     return;
   }
   if (options.config) {
@@ -1011,16 +1383,16 @@ program.command("run [capability-id]").description("Run a capability directly; d
     }
     const fromFile = options.file ? await readJsonFile(options.file) : {};
     const input = options.input ? parseJsonObject(options.input, "runtime input JSON") : fromFile.input || {};
-    const response2 = await client(command).postJob(
+    await executeQueuedJob(
       `/discoveries/${encodeURIComponent(options.config)}/jobs`,
       {
         ...fromFile,
         input,
         limit: Number(options.limit ?? fromFile.limit ?? 25)
       },
-      options.idempotencyKey || randomUUID3()
+      options,
+      command
     );
-    await emitExecution(response2, options, command);
     return;
   }
   if (!capabilityId) {
@@ -1028,12 +1400,12 @@ program.command("run [capability-id]").description("Run a capability directly; d
   }
   const payload = await requestPayload(options, capabilityId);
   if (payload.limit === void 0) payload.limit = 10;
-  const response = await client(command).postJob(
+  await executeQueuedJob(
     "/scrapers/jobs",
     payload,
-    options.idempotencyKey || randomUUID3()
+    options,
+    command
   );
-  await emitExecution(response, options, command);
 });
 var jobs = program.command("jobs").description("Inspect durable queued jobs");
 jobs.command("get <job-id>").action(async (jobId, _options, command) => {
@@ -1042,6 +1414,13 @@ jobs.command("get <job-id>").action(async (jobId, _options, command) => {
     globals(command).json || false
   );
 });
+jobs.command("wait <job-id>").description("Wait for a queued job and return its result").addOption(new Option("--format <format>", "output record format").choices(["json", "jsonl", "csv"]).default("json")).option("-o, --output <path>", "write records to a file").action(async (jobId, options, command) => {
+  await emitExecution(
+    await client(command).waitForJob(jobId),
+    options,
+    command
+  );
+});
 var providers = program.command("providers", { hidden: true }).description("Legacy provider commands");
 providers.command("list").option("--available", "only include available provider-capability routes").action(async (options, command) => {
   await emit(
@@ -1086,7 +1465,7 @@ discoveries.command("run <discovery-id>").option("-f, --file <path>", "request J
   const response = await client(command).postJob(
     `/discoveries/${encodeURIComponent(discoveryId)}/jobs`,
     payload,
-    options.idempotencyKey || randomUUID3()
+    options.idempotencyKey || randomUUID4()
   );
   await emitExecution(response, options, command);
 });
@@ -1094,7 +1473,7 @@ discoveries.command("delete <discovery-id>").requiredOption("--yes", "confirm pe
   await client(command).delete(`/discoveries/${encodeURIComponent(discoveryId)}`);
   await emit({ deleted: true, discovery_id: discoveryId }, globals(command).json || false);
 });
-program.command("discover").description("Save, sample-test, or finalize an agent-authored configuration").option("-f, --file <path>", "create from agent-authored discovery JSON").option("--id <discovery-id>", "existing discovery ID").option("--test", "approve and run a paid sample test").option("--input <json>", "sample runtime input JSON").option("--update <path>", "replace fields or configuration from agent-authored JSON").option("--finalize", "mark the current explicit configuration finalized").option("--idempotency-key <key>", "stable sample retry key").action(async (options, command) => {
+program.command("discover").description("Save, sample-test, or finalize an agent-authored configuration").option("-f, --file <path>", "create from agent-authored discovery JSON").option("--id <discovery-id>", "existing discovery ID").option("--test", "run a representative provider sample test").option("--input <json>", "sample runtime input JSON").option("--update <path>", "replace fields or configuration from agent-authored JSON").option("--finalize", "mark the current explicit configuration finalized").option("--idempotency-key <key>", "stable sample retry key").action(async (options, command) => {
   const selected = Number(Boolean(options.file)) + Number(Boolean(options.test)) + Number(Boolean(options.update)) + Number(Boolean(options.finalize));
   if (selected !== 1) {
     throw new CliError(
@@ -1134,7 +1513,7 @@ program.command("discover").description("Save, sample-test, or finalize an agent
   const response = await client(command).postJob(
     `/discoveries/${encodeURIComponent(options.id)}/jobs`,
     { input, limit: 25 },
-    options.idempotencyKey || randomUUID3()
+    options.idempotencyKey || randomUUID4()
   );
   await emit(response, globals(command).json || false);
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scrappycoco/cli",
-  "version": "0.4.1",
+  "version": "0.5.0",
   "description": "CLI for Scrappycoco scraper discovery and execution",
   "type": "module",
   "bin": {