@scrappycoco/cli 0.2.1 → 0.4.0

package/README.md

@@ -1,33 +1,47 @@
 # Scrappycoco CLI
 
-Discover, run, and compare scraper capabilities from a terminal or automation environment.
+Discover configurations and run scraper capabilities from a terminal or
+automation environment. The calling AI agent owns all provider and result
+judgment.
 
 Run the published package directly with `npx`:
 
 ```sh
-npx @scrappycoco/cli auth login
-npx @scrappycoco/cli scrapers list --available --json
-npx @scrappycoco/cli scrapers inspect web.extract_content --json
-npx @scrappycoco/cli scrapers run web.extract_content --file request.json --json
-npx @scrappycoco/cli scrapers compare web.extract_content --file request.json --json
+npx --yes @scrappycoco/cli setup
+```
+
+`setup` opens OAuth in the browser. New users can create an account in that
+flow. After authentication, it installs the public Scrappycoco skill, verifies
+the live catalog, and tells you when to reload or restart your agent.
+
+Individual commands remain available:
+
+```sh
+npx --yes @scrappycoco/cli auth login
+npx --yes @scrappycoco/cli catalog list --available --json
+npx --yes @scrappycoco/cli catalog inspect web.extract_content --json
+npx --yes @scrappycoco/cli run web.extract_content --file request.json --json
+npx --yes @scrappycoco/cli discover --file discovery.json --json
+npx --yes @scrappycoco/cli discover --id DISCOVERY_ID --finalize --json
+npx --yes @scrappycoco/cli run --config DISCOVERY_ID --input '{}' --json
 ```
 
 From a repository checkout, use `npm ci`, `npm run build`, and
 `node dist/index.js <command>` instead.
 
-Use `providers list` to inspect routes and
-`discoveries create|list|get|update|run|delete` for saved multi-capability
-configurations. The CLI does not expose legacy assistant, agent, workflow, or
-schedule commands.
+Discover is optional. Use Run directly when the current capability, provider,
+and native options are already clear. The CLI does not expose legacy
+assistant, agent, workflow, or schedule commands.
 
 Node.js 20 or newer is required. Interactive use authenticates with Clerk OAuth Authorization Code + PKCE. CI can set `SCRAPPYCOCO_API_KEY`.
 
 Use `--json` for machine-readable responses. Execution commands support
-`--format json|jsonl|csv` with `--output`, repeatable `--provider` flags, and an
-explicit `--idempotency-key` for safe identical retries. They submit durable
-jobs and poll for completion; set `SCRAPPYCOCO_JOB_TIMEOUT_MS` to change the
-20-minute local wait. If a command times out while its job continues, inspect
-it with `scrappycoco jobs get <job-id>`.
+`--format json|jsonl|csv` with `--output`, provider-native
+`--provider-options`, batch `--concurrency`, and an explicit
+`--idempotency-key` for safe identical retries. They submit durable jobs and
+poll for completion; set `SCRAPPYCOCO_JOB_TIMEOUT_MS` to change the 20-minute
+local wait. If a command times out while its job continues, inspect it with
+`scrappycoco jobs get <job-id>`.
 
 Use `scrappycoco --help` for the complete command reference. See the
 [Scrappycoco API documentation](https://scrappycoco.ai/docs) for the public

package/dist/index.js

@@ -8,7 +8,7 @@ import { Command, CommanderError, Option } from "commander";
 // src/auth.ts
 import { createHash, randomBytes, timingSafeEqual } from "crypto";
 import { createServer } from "http";
-import open from "open";
+import open2 from "open";
 
 // src/errors.ts
 var EXIT = {
@@ -32,7 +32,7 @@ var CliError = class extends Error {
 };
 
 // src/storage.ts
-import { chmod, mkdir, readFile, rm, writeFile } from "fs/promises";
+import { chmod, mkdir, open, readFile, rm, stat, writeFile } from "fs/promises";
 import { homedir } from "os";
 import { dirname, join } from "path";
 import { deletePassword, getPassword, setPassword } from "cross-keychain";
@@ -42,6 +42,49 @@ function fallbackCredentialPath() {
   const base = process.platform === "win32" ? process.env.APPDATA || join(homedir(), "AppData", "Roaming") : process.env.XDG_CONFIG_HOME || join(homedir(), ".config");
   return join(base, "scrappycoco", "credentials.json");
 }
+function credentialRefreshLockPath() {
+  return `${fallbackCredentialPath()}.refresh.lock`;
+}
+var LOCK_RETRY_MS = 100;
+var LOCK_TIMEOUT_MS = 3e4;
+var LOCK_STALE_MS = 6e4;
+function wait(milliseconds) {
+  return new Promise((resolve) => setTimeout(resolve, milliseconds));
+}
+async function withCredentialRefreshLock(operation) {
+  const path = credentialRefreshLockPath();
+  await mkdir(dirname(path), { recursive: true, mode: 448 });
+  const deadline = Date.now() + LOCK_TIMEOUT_MS;
+  while (true) {
+    try {
+      const handle = await open(path, "wx", 384);
+      try {
+        await handle.writeFile(JSON.stringify({ pid: process.pid, created_at: Date.now() }));
+        return await operation();
+      } finally {
+        await handle.close();
+        await rm(path, { force: true });
+      }
+    } catch (error) {
+      const code = error.code;
+      if (code !== "EEXIST") throw error;
+      try {
+        const lock = await stat(path);
+        if (Date.now() - lock.mtimeMs > LOCK_STALE_MS) {
+          await rm(path, { force: true });
+          continue;
+        }
+      } catch (statError) {
+        if (statError.code === "ENOENT") continue;
+        throw statError;
+      }
+      if (Date.now() >= deadline) {
+        throw new Error("Timed out waiting for the Scrappycoco credential refresh lock.");
+      }
+      await wait(LOCK_RETRY_MS);
+    }
+  }
+}
 async function readFallback() {
   try {
     const value = JSON.parse(await readFile(fallbackCredentialPath(), "utf8"));
@@ -79,6 +122,8 @@ async function clearRefreshToken() {
 }
 
 // src/auth.ts
+var cachedAccessToken;
+var refreshesInFlight = /* @__PURE__ */ new Map();
 async function oauthConfig(apiUrl) {
   const clientId = process.env.SCRAPPYCOCO_OAUTH_CLIENT_ID;
   if (clientId) {
@@ -168,7 +213,7 @@ async function login(options) {
       process.stderr.write(`Open this URL to authenticate:
 ${authorize.toString()}
 `);
-      if (!options.noBrowser) await open(authorize.toString());
+      if (!options.noBrowser) await open2(authorize.toString());
     });
     timeout = setTimeout(() => {
       server.close();
@@ -184,20 +229,53 @@ ${authorize.toString()}
     redirect_uri: redirectUri
   }));
   if (!tokens.refresh_token) throw new CliError("OAuth response did not include a refresh token.", EXIT.auth);
+  invalidateAccessToken();
   return { storage: await saveRefreshToken(tokens.refresh_token) };
 }
+function invalidateAccessToken() {
+  cachedAccessToken = void 0;
+}
+async function clearRefreshToken2() {
+  invalidateAccessToken();
+  await clearRefreshToken();
+}
 async function accessToken(apiUrl) {
-  const config = await oauthConfig(apiUrl);
-  const clientId = config.client_id;
-  const refreshToken = await loadRefreshToken();
-  if (!refreshToken) throw new CliError("Not logged in. Run `scrappycoco auth login`.", EXIT.auth);
-  const tokens = await tokenRequest(config.issuer, new URLSearchParams({
-    grant_type: "refresh_token",
-    client_id: clientId,
-    refresh_token: refreshToken
-  }));
-  if (tokens.refresh_token && tokens.refresh_token !== refreshToken) await saveRefreshToken(tokens.refresh_token);
-  return tokens.access_token;
+  const normalizedApiUrl = apiUrl.replace(/\/$/, "");
+  if (cachedAccessToken && cachedAccessToken.apiUrl === normalizedApiUrl && cachedAccessToken.expiresAt - 3e4 > Date.now()) {
+    return cachedAccessToken.value;
+  }
+  const existingRefresh = refreshesInFlight.get(normalizedApiUrl);
+  if (existingRefresh) return existingRefresh;
+  const refresh = withCredentialRefreshLock(async () => {
+    if (cachedAccessToken && cachedAccessToken.apiUrl === normalizedApiUrl && cachedAccessToken.expiresAt - 3e4 > Date.now()) {
+      return cachedAccessToken.value;
+    }
+    const config = await oauthConfig(normalizedApiUrl);
+    const refreshToken = await loadRefreshToken();
+    if (!refreshToken) throw new CliError("Not logged in. Run `scrappycoco auth login`.", EXIT.auth);
+    const tokens = await tokenRequest(config.issuer, new URLSearchParams({
+      grant_type: "refresh_token",
+      client_id: config.client_id,
+      refresh_token: refreshToken
+    }));
+    if (tokens.refresh_token && tokens.refresh_token !== refreshToken) {
+      await saveRefreshToken(tokens.refresh_token);
+    }
+    cachedAccessToken = {
+      apiUrl: normalizedApiUrl,
+      value: tokens.access_token,
+      expiresAt: Date.now() + Math.max(tokens.expires_in ?? 300, 60) * 1e3
+    };
+    return tokens.access_token;
+  });
+  refreshesInFlight.set(normalizedApiUrl, refresh);
+  try {
+    return await refresh;
+  } finally {
+    if (refreshesInFlight.get(normalizedApiUrl) === refresh) {
+      refreshesInFlight.delete(normalizedApiUrl);
+    }
+  }
 }
 
 // src/client.ts
@@ -235,12 +313,19 @@ var ApiClient = class {
     };
   }
   async request(method, path, body, headers, signal) {
-    const response = await fetch(`${this.baseUrl.replace(/\/$/, "")}/api/v1${path}`, {
+    const url = `${this.baseUrl.replace(/\/$/, "")}/api/v1${path}`;
+    const requestBody = body === void 0 ? void 0 : JSON.stringify(body);
+    const send = async () => fetch(url, {
       method,
       headers: await this.headers(headers),
-      body: body === void 0 ? void 0 : JSON.stringify(body),
+      body: requestBody,
       signal
     });
+    let response = await send();
+    if (response.status === 401 && !process.env.SCRAPPYCOCO_API_KEY) {
+      invalidateAccessToken();
+      response = await send();
+    }
     if (response.status === 204) return void 0;
     const payload = await response.json().catch(() => ({}));
     if (!response.ok) {
@@ -367,12 +452,167 @@ function errorPayload(error) {
   return { error: error.message, details: error.details ?? null };
 }
 
+// src/setup.ts
+import { spawn } from "child_process";
+import { determineAgent } from "@vercel/detect-agent";
+var SKILL_SOURCE = "https://scrappycoco.ai";
+var SKILL_INSTALL_ARGS = [
+  "--yes",
+  "skills",
+  "add",
+  SKILL_SOURCE,
+  "--skill",
+  "scrappycoco",
+  "-g",
+  "-y"
+];
+var SKILL_INSTALLER_AGENTS = {
+  antigravity: "antigravity",
+  "augment-cli": "augment",
+  claude: "claude-code",
+  codex: "codex",
+  cowork: "claude-code",
+  cursor: "cursor",
+  "cursor-cli": "cursor",
+  devin: "universal",
+  gemini: "gemini-cli",
+  "github-copilot": "github-copilot",
+  opencode: "opencode",
+  replit: "replit"
+};
+var MAX_INSTALLER_OUTPUT_LENGTH = 512 * 1024;
+var ANSI_ESCAPE = /\u001B\[[0-?]*[ -/]*[@-~]/g;
+function npxCommand() {
+  return process.platform === "win32" ? "npx.cmd" : "npx";
+}
+async function detectSkillInstallerAgent(detector = determineAgent) {
+  const detected = await detector();
+  if (!detected.isAgent || !detected.agent) return "universal";
+  return SKILL_INSTALLER_AGENTS[detected.agent.name] || "universal";
+}
+function skillInstallInvocation(agent = "universal") {
+  return {
+    command: npxCommand(),
+    args: [...SKILL_INSTALL_ARGS, "--agent", agent]
+  };
+}
+function skillInstallerReportedFailures(output) {
+  const plainOutput = output.replace(ANSI_ESCAPE, "");
+  if (/Failed to install\s+1/i.test(plainOutput) && /PromptScript does not support global installation/i.test(plainOutput) && !/Failed to install\s+[2-9]\d*/i.test(plainOutput)) {
+    return false;
+  }
+  return /Failed to install\s+[1-9]\d*/i.test(plainOutput);
+}
+function runCommand(command, args) {
+  return new Promise((resolve, reject) => {
+    let installerOutput = "";
+    const capture = (chunk) => {
+      installerOutput += chunk.toString();
+      if (installerOutput.length > MAX_INSTALLER_OUTPUT_LENGTH) {
+        installerOutput = installerOutput.slice(-MAX_INSTALLER_OUTPUT_LENGTH);
+      }
+    };
+    const child = spawn(command, [...args], {
+      stdio: ["ignore", "pipe", "pipe"],
+      windowsHide: true
+    });
+    child.stdout.on("data", (chunk) => {
+      capture(chunk);
+      process.stderr.write(chunk);
+    });
+    child.stderr.on("data", (chunk) => {
+      capture(chunk);
+      process.stderr.write(chunk);
+    });
+    child.on("error", (error) => reject(
+      new CliError(`Could not start the skill installer: ${error.message}`, EXIT.api)
+    ));
+    child.on("close", (code) => {
+      if (code === 0 && !skillInstallerReportedFailures(installerOutput)) {
+        resolve();
+        return;
+      }
+      if (code === 0) {
+        reject(new CliError(
+          "Skill installer reported one or more failed targets.",
+          EXIT.api
+        ));
+        return;
+      }
+      reject(new CliError(
+        `Skill installation failed with exit code ${code ?? "unknown"}.`,
+        EXIT.api
+      ));
+    });
+  });
+}
+async function installSkill(runner = runCommand, detector = determineAgent) {
+  const agent = await detectSkillInstallerAgent(detector);
+  const invocation = skillInstallInvocation(agent);
+  await runner(invocation.command, invocation.args);
+}
+function defaultDependencies(apiUrl) {
+  const client2 = new ApiClient(apiUrl);
+  return {
+    hasApiKey: () => Boolean(process.env.SCRAPPYCOCO_API_KEY),
+    clearRefreshToken: clearRefreshToken2,
+    loadRefreshToken,
+    login,
+    installSkill,
+    listAvailableScrapers: () => client2.get("/scrapers?available_only=true")
+  };
+}
+async function performSetup(options, dependencies = defaultDependencies(options.apiUrl)) {
+  const usingApiKey = dependencies.hasApiKey();
+  let authentication = usingApiKey ? "api_key" : "oauth";
+  let credentialStorage = usingApiKey ? "environment" : "existing";
+  const hadStoredToken = !usingApiKey && Boolean(await dependencies.loadRefreshToken());
+  if (!usingApiKey && !hadStoredToken) {
+    const authenticated = await dependencies.login({
+      apiUrl: options.apiUrl,
+      noBrowser: options.noBrowser
+    });
+    authentication = "oauth";
+    credentialStorage = authenticated.storage;
+  }
+  let catalog2;
+  try {
+    catalog2 = await dependencies.listAvailableScrapers();
+  } catch (error) {
+    if (usingApiKey || !hadStoredToken || !(error instanceof CliError) || error.exitCode !== EXIT.auth) {
+      throw error;
+    }
+    await dependencies.clearRefreshToken();
+    const authenticated = await dependencies.login({
+      apiUrl: options.apiUrl,
+      noBrowser: options.noBrowser
+    });
+    credentialStorage = authenticated.storage;
+    catalog2 = await dependencies.listAvailableScrapers();
+  }
+  await dependencies.installSkill();
+  return {
+    authenticated: true,
+    authentication,
+    credential_storage: credentialStorage,
+    skill: {
+      installed: true,
+      source: SKILL_SOURCE
+    },
+    verification: {
+      catalog_reachable: true,
+      available_capabilities: catalog2.length
+    },
+    next_step: "Reload or restart your agent, then ask it to inspect the Scrappycoco catalog."
+  };
+}
+
 // src/index.ts
 var packageMetadata = JSON.parse(
   readFileSync(new URL("../package.json", import.meta.url), "utf8")
 );
 var program = new Command();
-program.name("scrappycoco").description("Discover, run, and compare scraper capabilities through one API").version(packageMetadata.version).option("--json", "emit machine-readable JSON to stdout").option("--api-url <url>", "API base URL", process.env.SCRAPPYCOCO_API_URL || "https://api.scrappycoco.ai").showHelpAfterError().exitOverride();
+program.name("scrappycoco").description("Discover configurations and run external-data capabilities through one deterministic API").version(packageMetadata.version).option("--json", "emit machine-readable JSON to stdout").option("--api-url <url>", "API base URL", process.env.SCRAPPYCOCO_API_URL || "https://api.scrappycoco.ai").showHelpAfterError().exitOverride();
 function globals(command) {
   return command.optsWithGlobals();
 }
@@ -382,6 +622,13 @@ function client(command) {
 function collect(value, previous) {
   return [...previous, value];
 }
+program.command("setup").description("Authenticate, install the Scrappycoco skill, and verify the connection").option("--no-browser", "print the authorization URL without opening it").action(async (options, command) => {
+  const result = await performSetup({
+    apiUrl: globals(command).apiUrl,
+    noBrowser: options.browser === false
+  });
+  await emit(result, globals(command).json || false);
+});
 function splitScraperId(value) {
   const separator = value.indexOf(".");
   if (separator <= 0 || separator === value.length - 1) {
@@ -400,6 +647,8 @@ async function requestPayload(options, scraperId) {
     capability,
     input,
     ...options.provider?.length ? { providers: options.provider } : {},
+    ...options.providerOptions ? { provider_options: parseJsonObject(options.providerOptions, "provider options JSON") } : {},
+    ...options.concurrency !== void 0 ? { concurrency: Number(options.concurrency) } : {},
     ...options.limit !== void 0 ? { limit: Number(options.limit) } : {}
   };
 }
@@ -434,10 +683,10 @@ auth.command("status").action(async (_options, command) => {
   );
 });
 auth.command("logout").action(async (_options, command) => {
-  await clearRefreshToken();
+  await clearRefreshToken2();
   await emit({ authenticated: false }, globals(command).json || false);
 });
-var scrapers = program.command("scrapers").description("Inspect and execute scraper capabilities");
+var scrapers = program.command("scrapers", { hidden: true }).description("Legacy scraper commands");
 scrapers.command("list").option("--source <source>", "filter by web, x, reddit, or filings").option("--provider <provider>", "filter by provider implementation").option("--available", "only include available providers").action(async (options, command) => {
   const query = new URLSearchParams();
   if (options.source) query.set("source", options.source);
@@ -455,8 +704,28 @@ scrapers.command("inspect <scraper-id>").action(async (scraperId, _options, comm
     globals(command).json || false
   );
 });
+var catalog = program.command("catalog").description("Inspect capabilities, provider schemas, options, and pricing");
+catalog.command("list").option("--source <source>", "filter by web, x, reddit, or filings").option("--provider <provider>", "filter by provider implementation").option("--available", "only include available providers").action(async (options, command) => {
+  const query = new URLSearchParams();
+  if (options.source) query.set("source", options.source);
+  if (options.provider) query.set("provider", options.provider);
+  if (options.available) query.set("available_only", "true");
+  await emit(
+    await client(command).get(`/scrapers${query.size ? `?${query}` : ""}`),
+    globals(command).json || false
+  );
+});
+catalog.command("inspect <capability-id>").action(async (capabilityId, _options, command) => {
+  const { source, capability } = splitScraperId(capabilityId);
+  await emit(
+    await client(command).get(
+      `/scrapers/${encodeURIComponent(source)}/${encodeURIComponent(capability)}`
+    ),
+    globals(command).json || false
+  );
+});
 function executionCommand(name) {
-  return scrapers.command(`${name} <scraper-id>`).description(name === "run" ? "Run one capability through a provider waterfall" : "Compare providers for one capability").option("-f, --file <path>", "canonical request JSON file").option("--input <json>", "capability input JSON").option("--provider <id>", "provider ID; repeat to choose and order providers", collect, []).option("--limit <number>", "maximum records").option("--idempotency-key <key>", "stable retry key").addOption(new Option("--format <format>", "output record format").choices(["json", "jsonl", "csv"]).default("json")).option("-o, --output <path>", "write records to a file").action(async (scraperId, options, command) => {
+  return scrapers.command(`${name} <scraper-id>`).description(name === "run" ? "Run one capability through a provider waterfall" : "Compare providers for one capability").option("-f, --file <path>", "canonical request JSON file").option("--input <json>", "capability input JSON").option("--provider <id>", "provider ID; repeat to choose and order providers", collect, []).option("--provider-options <json>", "provider-native options keyed by provider ID").option("--concurrency <number>", "batch concurrency (default 3, maximum 10)").option("--limit <number>", "maximum records").option("--idempotency-key <key>", "stable retry key").addOption(new Option("--format <format>", "output record format").choices(["json", "jsonl", "csv"]).default("json")).option("-o, --output <path>", "write records to a file").action(async (scraperId, options, command) => {
     const payload = await requestPayload(options, scraperId);
     if (payload.limit === void 0) payload.limit = 10;
     const response = await client(command).postJob(
@@ -469,6 +738,49 @@ function executionCommand(name) {
 }
 executionCommand("run");
 executionCommand("compare");
+program.command("run [capability-id]").description("Run a capability directly; discovery is optional").option("--config <discovery-id>", "run a finalized multi-step configuration").option("-f, --file <path>", "canonical request JSON file").option("--input <json>", "capability input JSON; use url or urls for web.extract_content").option("--provider <id>", "one provider selected by the calling agent", collect, []).option("--provider-options <json>", "provider-native options keyed by provider ID").option("--concurrency <number>", "batch concurrency (default 3, maximum 10)").option("--limit <number>", "maximum records").option("--idempotency-key <key>", "stable retry key").option("--retry-failed <run-id>", "retry only failed URLs from a partial batch run").addOption(new Option("--format <format>", "output record format").choices(["json", "jsonl", "csv"]).default("json")).option("-o, --output <path>", "write records to a file").action(async (capabilityId, options, command) => {
+  if (options.retryFailed) {
+    if (capabilityId || options.config) {
+      throw new CliError("Do not combine --retry-failed with a capability ID or --config.", EXIT.usage);
+    }
+    const response2 = await client(command).postJob(
+      `/runs/${encodeURIComponent(options.retryFailed)}/retry-failed`,
+      {},
+      options.idempotencyKey || randomUUID2()
+    );
+    await emitExecution(response2, options, command);
+    return;
+  }
+  if (options.config) {
+    if (capabilityId) {
+      throw new CliError("Choose either a capability ID or --config.", EXIT.usage);
+    }
+    const fromFile = options.file ? await readJsonFile(options.file) : {};
+    const input = options.input ? parseJsonObject(options.input, "runtime input JSON") : fromFile.input || {};
+    const response2 = await client(command).postJob(
+      `/discoveries/${encodeURIComponent(options.config)}/jobs`,
+      {
+        ...fromFile,
+        input,
+        limit: Number(options.limit ?? fromFile.limit ?? 25)
+      },
+      options.idempotencyKey || randomUUID2()
+    );
+    await emitExecution(response2, options, command);
+    return;
+  }
+  if (!capabilityId) {
+    throw new CliError("Provide a capability ID or --retry-failed <run-id>.", EXIT.usage);
+  }
+  const payload = await requestPayload(options, capabilityId);
+  if (payload.limit === void 0) payload.limit = 10;
+  const response = await client(command).postJob(
+    "/scrapers/jobs",
+    payload,
+    options.idempotencyKey || randomUUID2()
+  );
+  await emitExecution(response, options, command);
+});
 var jobs = program.command("jobs").description("Inspect durable queued jobs");
 jobs.command("get <job-id>").action(async (jobId, _options, command) => {
   await emit(
@@ -476,17 +788,17 @@ jobs.command("get <job-id>").action(async (jobId, _options, command) => {
     globals(command).json || false
   );
 });
-var providers = program.command("providers").description("Inspect integrated scraper providers");
+var providers = program.command("providers", { hidden: true }).description("Legacy provider commands");
 providers.command("list").option("--available", "only include available provider-capability routes").action(async (options, command) => {
   await emit(
     await client(command).get(`/providers${options.available ? "?available_only=true" : ""}`),
     globals(command).json || false
   );
 });
-var discoveries = program.command("discoveries").description("Design and run saved multi-capability scraper configurations");
-discoveries.command("create").requiredOption("--goal <text>", "natural-language data goal").addOption(new Option("--priority <priority>", "comparison priority").choices(["balanced", "quality", "coverage", "cost", "speed"]).default("balanced")).action(async (options, command) => {
+var discoveries = program.command("discoveries", { hidden: true }).description("Legacy discovery commands");
+discoveries.command("create").requiredOption("-f, --file <path>", "agent-authored discovery JSON with goal and configuration").action(async (options, command) => {
   await emit(
-    await client(command).post("/discoveries", { goal: options.goal, priority: options.priority }),
+    await client(command).post("/discoveries", await readJsonFile(options.file)),
     globals(command).json || false
   );
 });
@@ -528,6 +840,50 @@ discoveries.command("delete <discovery-id>").requiredOption("--yes", "confirm pe
   await client(command).delete(`/discoveries/${encodeURIComponent(discoveryId)}`);
   await emit({ deleted: true, discovery_id: discoveryId }, globals(command).json || false);
 });
+program.command("discover").description("Save, sample-test, or finalize an agent-authored configuration").option("-f, --file <path>", "create from agent-authored discovery JSON").option("--id <discovery-id>", "existing discovery ID").option("--test", "approve and run a paid sample test").option("--input <json>", "sample runtime input JSON").option("--update <path>", "replace fields or configuration from agent-authored JSON").option("--finalize", "mark the current explicit configuration finalized").option("--idempotency-key <key>", "stable sample retry key").action(async (options, command) => {
+  const selected = Number(Boolean(options.file)) + Number(Boolean(options.test)) + Number(Boolean(options.update)) + Number(Boolean(options.finalize));
+  if (selected !== 1) {
+    throw new CliError(
+      "Choose exactly one action: --file, --id --update, --id --test, or --id --finalize.",
+      EXIT.usage
+    );
+  }
+  if (options.file) {
+    await emit(
+      await client(command).post("/discoveries", await readJsonFile(options.file)),
+      globals(command).json || false
+    );
+    return;
+  }
+  if (!options.id) throw new CliError("--id is required for --update, --test, and --finalize.", EXIT.usage);
+  if (options.update) {
+    await emit(
+      await client(command).patch(
+        `/discoveries/${encodeURIComponent(options.id)}`,
+        await readJsonFile(options.update)
+      ),
+      globals(command).json || false
+    );
+    return;
+  }
+  if (options.finalize) {
+    await emit(
+      await client(command).post(
+        `/discoveries/${encodeURIComponent(options.id)}/finalize`,
+        {}
+      ),
+      globals(command).json || false
+    );
+    return;
+  }
+  const input = options.input ? parseJsonObject(options.input, "sample input JSON") : {};
+  const response = await client(command).postJob(
+    `/discoveries/${encodeURIComponent(options.id)}/jobs`,
+    { input, limit: 25 },
+    options.idempotencyKey || randomUUID2()
+  );
+  await emit(response, globals(command).json || false);
+});
 program.configureOutput({ writeErr: (text) => process.stderr.write(text) });
 program.parseAsync(process.argv).catch(async (error) => {
   if (error instanceof CommanderError) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scrappycoco/cli",
-  "version": "0.2.1",
+  "version": "0.4.0",
   "description": "CLI for Scrappycoco scraper discovery, execution, and provider comparison",
   "type": "module",
   "bin": {
@@ -22,6 +22,7 @@
     "prepack": "npm run build"
   },
   "dependencies": {
+    "@vercel/detect-agent": "^1.2.3",
     "commander": "^14.0.0",
     "cross-keychain": "^1.1.0",
     "open": "^10.2.0"