@scrappycoco/cli 0.3.0 → 0.4.1

package/README.md

@@ -1,6 +1,8 @@
 # Scrappycoco CLI
 
-Discover, run, and compare scraper capabilities from a terminal or automation environment.
+Discover configurations and run scraper capabilities from a terminal or
+automation environment. The calling AI agent owns all provider and result
+judgment.
 
 Run the published package directly with `npx`:
 
@@ -10,34 +12,63 @@ npx --yes @scrappycoco/cli setup
 
 `setup` opens OAuth in the browser. New users can create an account in that
 flow. After authentication, it installs the public Scrappycoco skill, verifies
-the live catalog, and tells you when to reload or restart your agent.
+the live catalog, and tells you when to reload or restart your agent. Browser
+authorization is not complete until the terminal confirms that the credential
+was saved and the catalog check succeeded.
+
+When the CLI runs on a remote or headless host whose `127.0.0.1` is not the
+browser's localhost, keep the command running and use the manual callback
+flow:
+
+```sh
+npx --yes @scrappycoco/cli setup --no-browser --manual-callback
+```
+
+Open the printed authorization URL, then paste the browser's final callback
+URL into that same terminal. The callback contains a short-lived
+authorization code: never paste it into chat, logs, or a saved file. A timed
+out or exited login cannot be resumed; start a new command and use its new
+URL.
 
 Individual commands remain available:
 
 ```sh
 npx --yes @scrappycoco/cli auth login
-npx --yes @scrappycoco/cli scrapers list --available --json
-npx --yes @scrappycoco/cli scrapers inspect web.extract_content --json
-npx --yes @scrappycoco/cli scrapers run web.extract_content --file request.json --json
-npx --yes @scrappycoco/cli scrapers compare web.extract_content --file request.json --json
+npx --yes @scrappycoco/cli catalog list --available --json
+npx --yes @scrappycoco/cli catalog inspect web.extract_content --json
+npx --yes @scrappycoco/cli run web.extract_content --file request.json --json
+npx --yes @scrappycoco/cli discover --file discovery.json --json
+npx --yes @scrappycoco/cli discover --id DISCOVERY_ID --test --input '{"url":"https://example.com"}' --json
+npx --yes @scrappycoco/cli discover --id DISCOVERY_ID --finalize --json
+npx --yes @scrappycoco/cli run --config DISCOVERY_ID --input '{}' --json
 ```
 
 From a repository checkout, use `npm ci`, `npm run build`, and
 `node dist/index.js <command>` instead.
 
-Use `providers list` to inspect routes and
-`discoveries create|list|get|update|run|delete` for saved multi-capability
-configurations. The CLI does not expose legacy assistant, agent, workflow, or
-schedule commands.
+Discover is optional. Use Run directly when the current capability, provider,
+and native options are already clear. The CLI does not expose legacy
+assistant, agent, workflow, or schedule commands.
 
 Node.js 20 or newer is required. Interactive use authenticates with Clerk OAuth Authorization Code + PKCE. CI can set `SCRAPPYCOCO_API_KEY`.
 
+OAuth and API requests have bounded timeouts. Set
+`SCRAPPYCOCO_AUTH_TIMEOUT_MS` or `SCRAPPYCOCO_API_TIMEOUT_MS` only when a slow
+network requires a longer window.
+
 Use `--json` for machine-readable responses. Execution commands support
-`--format json|jsonl|csv` with `--output`, repeatable `--provider` flags, and an
-explicit `--idempotency-key` for safe identical retries. They submit durable
-jobs and poll for completion; set `SCRAPPYCOCO_JOB_TIMEOUT_MS` to change the
-20-minute local wait. If a command times out while its job continues, inspect
-it with `scrappycoco jobs get <job-id>`.
+`--format json|jsonl|csv` with `--output`, provider-native
+`--provider-options`, batch `--concurrency`, and an explicit
+`--idempotency-key` for safe identical retries. They submit durable jobs and
+poll for completion; set `SCRAPPYCOCO_JOB_TIMEOUT_MS` to change the 20-minute
+local wait. If a command times out while its job continues, inspect it with
+`scrappycoco jobs get <job-id>`.
+
+When provider or configuration choice is uncertain, define 2 to 5 named
+`candidates` in a Discover route. Each candidate has its own `id`, `provider`,
+and `options`, so one discovery can test both `zyte-http` with
+`{"browser_html":false}` and `zyte-browser` with
+`{"browser_html":true}` against the same input.
 
 Use `scrappycoco --help` for the complete command reference. See the
 [Scrappycoco API documentation](https://scrappycoco.ai/docs) for the public

package/dist/index.js

@@ -1,14 +1,15 @@
 #!/usr/bin/env node
 
 // src/index.ts
-import { randomUUID as randomUUID2 } from "crypto";
+import { randomUUID as randomUUID3 } from "crypto";
 import { readFileSync } from "fs";
 import { Command, CommanderError, Option } from "commander";
 
 // src/auth.ts
 import { createHash, randomBytes, timingSafeEqual } from "crypto";
 import { createServer } from "http";
-import open from "open";
+import { createInterface } from "readline/promises";
+import open2 from "open";
 
 // src/errors.ts
 var EXIT = {
@@ -32,16 +33,65 @@ var CliError = class extends Error {
 };
 
 // src/storage.ts
-import { chmod, mkdir, readFile, rm, writeFile } from "fs/promises";
+import { randomUUID } from "crypto";
+import { chmod, mkdir, open, readFile, rename, rm, stat, writeFile } from "fs/promises";
 import { homedir } from "os";
 import { dirname, join } from "path";
 import { deletePassword, getPassword, setPassword } from "cross-keychain";
 var SERVICE = "scrappycoco-cli";
 var ACCOUNT = "refresh-token";
 function fallbackCredentialPath() {
+  const configured = process.env.SCRAPPYCOCO_CONFIG_HOME;
+  if (configured) return join(configured, "credentials.json");
   const base = process.platform === "win32" ? process.env.APPDATA || join(homedir(), "AppData", "Roaming") : process.env.XDG_CONFIG_HOME || join(homedir(), ".config");
   return join(base, "scrappycoco", "credentials.json");
 }
+function credentialRefreshLockPath() {
+  return `${fallbackCredentialPath()}.refresh.lock`;
+}
+function credentialStatePath() {
+  return `${fallbackCredentialPath()}.state`;
+}
+var LOCK_RETRY_MS = 100;
+var LOCK_TIMEOUT_MS = 3e4;
+var LOCK_STALE_MS = 6e4;
+function wait(milliseconds) {
+  return new Promise((resolve) => setTimeout(resolve, milliseconds));
+}
+async function withCredentialRefreshLock(operation) {
+  const path = credentialRefreshLockPath();
+  await mkdir(dirname(path), { recursive: true, mode: 448 });
+  const deadline = Date.now() + LOCK_TIMEOUT_MS;
+  while (true) {
+    try {
+      const handle = await open(path, "wx", 384);
+      try {
+        await handle.writeFile(JSON.stringify({ pid: process.pid, created_at: Date.now() }));
+        return await operation();
+      } finally {
+        await handle.close();
+        await rm(path, { force: true });
+      }
+    } catch (error) {
+      const code = error.code;
+      if (code !== "EEXIST") throw error;
+      try {
+        const lock = await stat(path);
+        if (Date.now() - lock.mtimeMs > LOCK_STALE_MS) {
+          await rm(path, { force: true });
+          continue;
+        }
+      } catch (statError) {
+        if (statError.code === "ENOENT") continue;
+        throw statError;
+      }
+      if (Date.now() >= deadline) {
+        throw new Error("Timed out waiting for the Scrappycoco credential refresh lock.");
+      }
+      await wait(LOCK_RETRY_MS);
+    }
+  }
+}
 async function readFallback() {
   try {
     const value = JSON.parse(await readFile(fallbackCredentialPath(), "utf8"));
@@ -50,23 +100,89 @@ async function readFallback() {
     return null;
   }
 }
-async function loadRefreshToken() {
+async function readKeychain() {
   try {
     const value = await getPassword(SERVICE, ACCOUNT);
-    if (value) return value;
+    return value || null;
+  } catch {
+    return null;
+  }
+}
+async function readCredentialState() {
+  try {
+    const value = JSON.parse(await readFile(credentialStatePath(), "utf8"));
+    return value.storage === "keychain" || value.storage === "file" || value.storage === "none" ? value.storage : null;
   } catch {
+    return null;
   }
-  return readFallback();
+}
+async function writePrivateJson(path, value) {
+  await mkdir(dirname(path), { recursive: true, mode: 448 });
+  const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
+  try {
+    await writeFile(temporary, JSON.stringify(value), { mode: 384 });
+    if (process.platform !== "win32") await chmod(temporary, 384);
+    await rename(temporary, path);
+    if (process.platform !== "win32") await chmod(path, 384);
+  } finally {
+    await rm(temporary, { force: true });
+  }
+}
+async function writeCredentialState(storage) {
+  await writePrivateJson(credentialStatePath(), { storage });
+}
+async function loadRefreshTokenCandidates() {
+  const state = await readCredentialState();
+  if (state === "none") return [];
+  if (state === "file") {
+    const value = await readFallback();
+    return value ? [{ value, storage: "file" }] : [];
+  }
+  if (state === "keychain") {
+    const value = await readKeychain();
+    return value ? [{ value, storage: "keychain" }] : [];
+  }
+  const [keychain, file] = await Promise.all([readKeychain(), readFallback()]);
+  const candidates = [];
+  if (keychain) candidates.push({ value: keychain, storage: "keychain" });
+  if (file && file !== keychain) candidates.push({ value: file, storage: "file" });
+  return candidates;
+}
+async function prepareCredentialStorage() {
+  const state = await readCredentialState();
+  if (state === "file") {
+    await assertFallbackWritable();
+    return "file";
+  }
+  try {
+    await getPassword(SERVICE, ACCOUNT);
+    return "keychain";
+  } catch {
+    await assertFallbackWritable();
+    return "file";
+  }
+}
+async function assertFallbackWritable() {
+  const path = fallbackCredentialPath();
+  await mkdir(dirname(path), { recursive: true, mode: 448 });
+  const probe = `${path}.write-test-${process.pid}-${randomUUID()}`;
+  const handle = await open(probe, "w", 384);
+  await handle.close();
+  await rm(probe, { force: true });
 }
 async function saveRefreshToken(refreshToken) {
   try {
     await setPassword(SERVICE, ACCOUNT, refreshToken);
+    await rm(fallbackCredentialPath(), { force: true });
+    await writeCredentialState("keychain");
     return "keychain";
   } catch {
-    const path = fallbackCredentialPath();
-    await mkdir(dirname(path), { recursive: true, mode: 448 });
-    await writeFile(path, JSON.stringify({ refresh_token: refreshToken }), { mode: 384 });
-    if (process.platform !== "win32") await chmod(path, 384);
+    await writePrivateJson(fallbackCredentialPath(), { refresh_token: refreshToken });
+    await writeCredentialState("file");
+    try {
+      await deletePassword(SERVICE, ACCOUNT);
+    } catch {
+    }
     return "file";
   }
 }
@@ -76,9 +192,42 @@ async function clearRefreshToken() {
   } catch {
   }
   await rm(fallbackCredentialPath(), { force: true });
+  await writeCredentialState("none");
 }
 
 // src/auth.ts
+var cachedAccessToken;
+var refreshesInFlight = /* @__PURE__ */ new Map();
+var DEFAULT_AUTH_HTTP_TIMEOUT_MS = 15e3;
+function positiveInteger(value, fallback) {
+  if (!value) return fallback;
+  const parsed = Number(value);
+  return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback;
+}
+function authHttpTimeoutMs() {
+  return positiveInteger(process.env.SCRAPPYCOCO_AUTH_TIMEOUT_MS, DEFAULT_AUTH_HTTP_TIMEOUT_MS);
+}
+function networkMessage(error) {
+  if (error instanceof DOMException && (error.name === "AbortError" || error.name === "TimeoutError")) {
+    return `timed out after ${Math.ceil(authHttpTimeoutMs() / 1e3)} seconds`;
+  }
+  return "failed to reach the authentication service";
+}
+async function oauthFetch(url, init, phase, exitCode) {
+  try {
+    return await fetch(url, {
+      ...init,
+      signal: AbortSignal.timeout(authHttpTimeoutMs())
+    });
+  } catch (error) {
+    const timeout = error instanceof DOMException && (error.name === "AbortError" || error.name === "TimeoutError");
+    throw new CliError(
+      `OAuth ${phase} ${networkMessage(error)}. ${phase === "authorization-code exchange" ? "Authorization was received but was not saved; run `scrappycoco auth login` again." : "Check the connection and try again."}`,
+      timeout ? EXIT.timeout : exitCode,
+      { phase, network_error: true }
+    );
+  }
+}
 async function oauthConfig(apiUrl) {
   const clientId = process.env.SCRAPPYCOCO_OAUTH_CLIENT_ID;
   if (clientId) {
@@ -88,7 +237,12 @@ async function oauthConfig(apiUrl) {
       scopes: ["openid", "profile", "email"]
     };
   }
-  const response = await fetch(`${apiUrl.replace(/\/$/, "")}/api/v1/oauth/cli-config`);
+  const response = await oauthFetch(
+    `${apiUrl.replace(/\/$/, "")}/api/v1/oauth/cli-config`,
+    {},
+    "configuration request",
+    EXIT.api
+  );
   const payload = await response.json().catch(() => ({}));
   if (!response.ok || !payload.issuer || !payload.client_id || !Array.isArray(payload.scopes)) {
     throw new CliError(payload.detail || "Scrappycoco CLI OAuth is not configured.", EXIT.auth);
@@ -111,49 +265,99 @@ function statesMatch(expected, received) {
   const right = Buffer.from(received);
   return left.length === right.length && timingSafeEqual(left, right);
 }
-async function tokenRequest(issuer, body) {
-  const response = await fetch(`${issuer.replace(/\/$/, "")}/oauth/token`, {
-    method: "POST",
-    headers: { "content-type": "application/x-www-form-urlencoded" },
-    body
-  });
+async function tokenRequest(issuer, body, phase) {
+  const response = await oauthFetch(
+    `${issuer.replace(/\/$/, "")}/oauth/token`,
+    {
+      method: "POST",
+      headers: { "content-type": "application/x-www-form-urlencoded" },
+      body
+    },
+    phase,
+    EXIT.auth
+  );
   const payload = await response.json().catch(() => ({}));
   if (!response.ok || typeof payload.access_token !== "string") {
-    throw new CliError(String(payload.error_description || payload.error || "OAuth token exchange failed."), EXIT.auth);
+    throw new CliError(
+      String(payload.error_description || payload.error || `OAuth ${phase} failed.`),
+      EXIT.auth,
+      {
+        phase,
+        status: response.status,
+        ...typeof payload.error === "string" ? { oauth_error: payload.error } : {}
+      }
+    );
   }
   return payload;
 }
+function callbackResult(rawUrl, expectedState, redirectUri) {
+  const current = new URL(rawUrl, redirectUri);
+  const expected = new URL(redirectUri);
+  if (current.protocol !== expected.protocol || current.hostname !== expected.hostname || current.port !== expected.port || current.pathname !== expected.pathname) {
+    throw new CliError("OAuth callback URL did not match this login attempt.", EXIT.auth);
+  }
+  const state = current.searchParams.get("state") || "";
+  const providerError = current.searchParams.get("error");
+  if (!statesMatch(expectedState, state)) {
+    throw new CliError("OAuth state validation failed.", EXIT.auth);
+  }
+  if (providerError) {
+    const description = current.searchParams.get("error_description") || providerError;
+    throw new CliError(`OAuth authorization was denied: ${description}`, EXIT.auth);
+  }
+  const code = current.searchParams.get("code") || "";
+  if (!code) throw new CliError("OAuth callback did not include an authorization code.", EXIT.auth);
+  return { code, redirectUri };
+}
 async function login(options) {
   const config = await oauthConfig(options.apiUrl);
   const clientId = config.client_id;
   const pkce = createPkce();
   let timeout;
+  let manualReader;
   const callback = new Promise((resolve, reject) => {
+    let settled = false;
+    const finish = (result, error) => {
+      if (settled) return;
+      settled = true;
+      if (timeout) clearTimeout(timeout);
+      manualReader?.close();
+      server.close();
+      if (error) reject(error);
+      else if (result) resolve(result);
+    };
     const server = createServer((request, response) => {
-      const current = new URL(request.url || "/", "http://127.0.0.1");
-      if (current.pathname !== "/callback") {
-        response.writeHead(404).end("Not found");
+      const address = server.address();
+      if (!address || typeof address === "string") {
+        response.writeHead(500, { "content-type": "text/plain" }).end("OAuth callback failed.");
+        finish(void 0, new CliError("OAuth callback failed.", EXIT.auth));
         return;
       }
-      const state = current.searchParams.get("state") || "";
-      const code2 = current.searchParams.get("code") || "";
-      if (!statesMatch(pkce.state, state) || !code2) {
-        response.writeHead(400, { "content-type": "text/plain" }).end("Invalid OAuth callback. You may close this window.");
-        clearTimeout(timeout);
-        server.close();
-        reject(new CliError("OAuth state validation failed.", EXIT.auth));
+      const redirectUri2 = `http://127.0.0.1:${address.port}/callback`;
+      const current = new URL(request.url || "/", redirectUri2);
+      if (current.pathname !== "/callback") {
+        response.writeHead(404, { "content-type": "text/plain" }).end("Not found");
         return;
       }
-      const address = server.address();
-      if (!address || typeof address === "string") return reject(new CliError("OAuth callback failed.", EXIT.auth));
-      response.writeHead(200, { "content-type": "text/plain" }).end("Scrappycoco login complete. You may close this window.");
-      clearTimeout(timeout);
-      server.close();
-      resolve({ code: code2, redirectUri: `http://127.0.0.1:${address.port}/callback` });
+      try {
+        const result = callbackResult(request.url || "/", pkce.state, redirectUri2);
+        response.writeHead(200, { "content-type": "text/plain" }).end(
+          "Scrappycoco authorization received. Keep the terminal open while it securely saves the login; the terminal will report when it is complete."
+        );
+        finish(result);
+      } catch (error) {
+        response.writeHead(400, { "content-type": "text/plain" }).end(
+          error instanceof Error ? error.message : "Invalid OAuth callback."
+        );
+        finish(void 0, error);
+      }
     });
     server.listen(0, "127.0.0.1", async () => {
       const address = server.address();
-      if (!address || typeof address === "string") return reject(new CliError("Could not start OAuth callback.", EXIT.auth));
+      if (!address || typeof address === "string") {
+        finish(void 0, new CliError("Could not start OAuth callback.", EXIT.auth));
+        return;
+      }
       const redirectUri2 = `http://127.0.0.1:${address.port}/callback`;
       const authorize = new URL(`${config.issuer.replace(/\/$/, "")}/oauth/authorize`);
       authorize.search = new URLSearchParams({
@@ -168,11 +372,32 @@ async function login(options) {
       process.stderr.write(`Open this URL to authenticate:
 ${authorize.toString()}
 `);
-      if (!options.noBrowser) await open(authorize.toString());
+      if (options.manualCallback) {
+        process.stderr.write(
+          "After signing in, paste the final callback URL into this terminal. Never paste it into chat.\n"
+        );
+        manualReader = createInterface({ input: process.stdin, output: process.stderr });
+        void manualReader.question("Callback URL: ").then((rawUrl) => {
+          try {
+            finish(callbackResult(rawUrl.trim(), pkce.state, redirectUri2));
+          } catch (error) {
+            finish(void 0, error);
+          }
+        }).catch((error) => finish(void 0, error));
+      }
+      if (!options.noBrowser) {
+        try {
+          await open2(authorize.toString());
+        } catch {
+          process.stderr.write("Could not open a browser automatically; use the URL above.\n");
+        }
+      }
     });
     timeout = setTimeout(() => {
-      server.close();
-      reject(new CliError("OAuth login timed out.", EXIT.timeout));
+      finish(void 0, new CliError(
+        "OAuth login timed out. The printed URL and callback port are no longer valid; start a new login.",
+        EXIT.timeout
+      ));
     }, 5 * 6e4);
   });
   const { code, redirectUri } = await callback;
@@ -182,31 +407,81 @@ ${authorize.toString()}
     code,
     code_verifier: pkce.verifier,
     redirect_uri: redirectUri
-  }));
+  }), "authorization-code exchange");
   if (!tokens.refresh_token) throw new CliError("OAuth response did not include a refresh token.", EXIT.auth);
+  invalidateAccessToken();
   return { storage: await saveRefreshToken(tokens.refresh_token) };
 }
+function invalidateAccessToken() {
+  cachedAccessToken = void 0;
+}
+async function clearRefreshToken2() {
+  invalidateAccessToken();
+  await clearRefreshToken();
+}
 async function accessToken(apiUrl) {
-  const config = await oauthConfig(apiUrl);
-  const clientId = config.client_id;
-  const refreshToken = await loadRefreshToken();
-  if (!refreshToken) throw new CliError("Not logged in. Run `scrappycoco auth login`.", EXIT.auth);
-  const tokens = await tokenRequest(config.issuer, new URLSearchParams({
-    grant_type: "refresh_token",
-    client_id: clientId,
-    refresh_token: refreshToken
-  }));
-  if (tokens.refresh_token && tokens.refresh_token !== refreshToken) await saveRefreshToken(tokens.refresh_token);
-  return tokens.access_token;
+  const normalizedApiUrl = apiUrl.replace(/\/$/, "");
+  if (cachedAccessToken && cachedAccessToken.apiUrl === normalizedApiUrl && cachedAccessToken.expiresAt - 3e4 > Date.now()) {
+    return cachedAccessToken.value;
+  }
+  const existingRefresh = refreshesInFlight.get(normalizedApiUrl);
+  if (existingRefresh) return existingRefresh;
+  const refresh = withCredentialRefreshLock(async () => {
+    if (cachedAccessToken && cachedAccessToken.apiUrl === normalizedApiUrl && cachedAccessToken.expiresAt - 3e4 > Date.now()) {
+      return cachedAccessToken.value;
+    }
+    const config = await oauthConfig(normalizedApiUrl);
+    const candidates = await loadRefreshTokenCandidates();
+    if (!candidates.length) throw new CliError("Not logged in. Run `scrappycoco auth login`.", EXIT.auth);
+    let tokens;
+    let acceptedRefreshToken;
+    let lastAuthError;
+    for (const candidate of candidates) {
+      try {
+        tokens = await tokenRequest(config.issuer, new URLSearchParams({
+          grant_type: "refresh_token",
+          client_id: config.client_id,
+          refresh_token: candidate.value
+        }), "token refresh");
+        acceptedRefreshToken = candidate.value;
+        break;
+      } catch (error) {
+        if (!(error instanceof CliError) || error.exitCode !== EXIT.auth || typeof error.details === "object" && error.details !== null && "network_error" in error.details) {
+          throw error;
+        }
+        lastAuthError = error;
+      }
+    }
+    if (!tokens || !acceptedRefreshToken) throw lastAuthError || new CliError("Stored OAuth login is no longer valid.", EXIT.auth);
+    await saveRefreshToken(tokens.refresh_token || acceptedRefreshToken);
+    cachedAccessToken = {
+      apiUrl: normalizedApiUrl,
+      value: tokens.access_token,
+      expiresAt: Date.now() + Math.max(tokens.expires_in ?? 300, 60) * 1e3
+    };
+    return tokens.access_token;
+  });
+  refreshesInFlight.set(normalizedApiUrl, refresh);
+  try {
+    return await refresh;
+  } finally {
+    if (refreshesInFlight.get(normalizedApiUrl) === refresh) {
+      refreshesInFlight.delete(normalizedApiUrl);
+    }
+  }
+}
+async function loadRefreshToken() {
+  return (await loadRefreshTokenCandidates())[0]?.value || null;
 }
 
 // src/client.ts
-import { randomUUID } from "crypto";
+import { randomUUID as randomUUID2 } from "crypto";
 var DEFAULT_API_URL = process.env.SCRAPPYCOCO_API_URL || "https://api.scrappycoco.ai";
 var DEFAULT_JOB_TIMEOUT_MS = 20 * 60 * 1e3;
 var DEFAULT_JOB_POLL_INITIAL_MS = 250;
 var DEFAULT_JOB_POLL_MAX_MS = 5e3;
-function positiveInteger(value, fallback) {
+var DEFAULT_API_REQUEST_TIMEOUT_MS = 3e4;
+function positiveInteger2(value, fallback) {
   if (!value) return fallback;
   const parsed = Number(value);
   return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback;
@@ -235,12 +510,35 @@ var ApiClient = class {
     };
   }
   async request(method, path, body, headers, signal) {
-    const response = await fetch(`${this.baseUrl.replace(/\/$/, "")}/api/v1${path}`, {
-      method,
-      headers: await this.headers(headers),
-      body: body === void 0 ? void 0 : JSON.stringify(body),
-      signal
-    });
+    const url = `${this.baseUrl.replace(/\/$/, "")}/api/v1${path}`;
+    const requestBody = body === void 0 ? void 0 : JSON.stringify(body);
+    const send = async () => {
+      const timeoutSignal = AbortSignal.timeout(
+        positiveInteger2(process.env.SCRAPPYCOCO_API_TIMEOUT_MS, DEFAULT_API_REQUEST_TIMEOUT_MS)
+      );
+      const requestSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
+      const requestHeaders = await this.headers(headers);
+      try {
+        return await fetch(url, {
+          method,
+          headers: requestHeaders,
+          body: requestBody,
+          signal: requestSignal
+        });
+      } catch (error) {
+        const timeout = error instanceof DOMException && (error.name === "AbortError" || error.name === "TimeoutError");
+        throw new CliError(
+          timeout ? `Scrappycoco API request timed out: ${method} ${path}.` : `Could not reach the Scrappycoco API: ${method} ${path}.`,
+          timeout ? EXIT.timeout : EXIT.api,
+          { method, path }
+        );
+      }
+    };
+    let response = await send();
+    if (response.status === 401 && !process.env.SCRAPPYCOCO_API_KEY) {
+      invalidateAccessToken();
+      response = await send();
+    }
     if (response.status === 204) return void 0;
     const payload = await response.json().catch(() => ({}));
     if (!response.ok) {
@@ -253,17 +551,17 @@ var ApiClient = class {
   get(path, signal) {
     return this.request("GET", path, void 0, void 0, signal);
   }
-  post(path, body, key = randomUUID()) {
+  post(path, body, key = randomUUID2()) {
     return this.request("POST", path, body, { "Idempotency-Key": key });
   }
-  async postJob(path, body, key = randomUUID()) {
+  async postJob(path, body, key = randomUUID2()) {
     const submitted = await this.post(path, body, key);
-    const timeoutMs = positiveInteger(process.env.SCRAPPYCOCO_JOB_TIMEOUT_MS, DEFAULT_JOB_TIMEOUT_MS);
-    const initialDelayMs = positiveInteger(
+    const timeoutMs = positiveInteger2(process.env.SCRAPPYCOCO_JOB_TIMEOUT_MS, DEFAULT_JOB_TIMEOUT_MS);
+    const initialDelayMs = positiveInteger2(
       process.env.SCRAPPYCOCO_JOB_POLL_INITIAL_MS,
       DEFAULT_JOB_POLL_INITIAL_MS
     );
-    const maxDelayMs = positiveInteger(process.env.SCRAPPYCOCO_JOB_POLL_MAX_MS, DEFAULT_JOB_POLL_MAX_MS);
+    const maxDelayMs = positiveInteger2(process.env.SCRAPPYCOCO_JOB_POLL_MAX_MS, DEFAULT_JOB_POLL_MAX_MS);
     const deadline = Date.now() + timeoutMs;
     let delayMs = Math.min(initialDelayMs, maxDelayMs);
     let job = submitted;
@@ -369,6 +667,7 @@ function errorPayload(error) {
 
 // src/setup.ts
 import { spawn } from "child_process";
+import { determineAgent } from "@vercel/detect-agent";
 var SKILL_SOURCE = "https://scrappycoco.ai";
 var SKILL_INSTALL_ARGS = [
   "--yes",
@@ -380,28 +679,92 @@ var SKILL_INSTALL_ARGS = [
   "-g",
   "-y"
 ];
+var SKILL_INSTALLER_AGENTS = {
+  antigravity: "antigravity",
+  "augment-cli": "augment",
+  claude: "claude-code",
+  codex: "codex",
+  cowork: "claude-code",
+  cursor: "cursor",
+  "cursor-cli": "cursor",
+  devin: "universal",
+  gemini: "gemini-cli",
+  "github-copilot": "github-copilot",
+  opencode: "opencode",
+  replit: "replit"
+};
+var MAX_INSTALLER_OUTPUT_LENGTH = 512 * 1024;
+var ANSI_ESCAPE = /\u001B\[[0-?]*[ -/]*[@-~]/g;
 function npxCommand() {
   return process.platform === "win32" ? "npx.cmd" : "npx";
 }
-function skillInstallInvocation() {
-  return { command: npxCommand(), args: SKILL_INSTALL_ARGS };
+async function detectSkillInstallerAgent(detector = determineAgent) {
+  const detected = await detector();
+  if (!detected.isAgent || !detected.agent) return "universal";
+  return SKILL_INSTALLER_AGENTS[detected.agent.name] || "universal";
+}
+function skillInstallInvocation(agent = "universal", npmExecPath = process.env.npm_execpath) {
+  if (npmExecPath && /(?:^|[/\\])pnpm(?:\.c?js)?$/i.test(npmExecPath)) {
+    return {
+      command: process.execPath,
+      args: [
+        npmExecPath,
+        "dlx",
+        "skills",
+        ...SKILL_INSTALL_ARGS.slice(2),
+        "--agent",
+        agent
+      ]
+    };
+  }
+  return {
+    command: npxCommand(),
+    args: [...SKILL_INSTALL_ARGS, "--agent", agent]
+  };
+}
+function skillInstallerReportedFailures(output) {
+  const plainOutput = output.replace(ANSI_ESCAPE, "");
+  if (/Failed to install\s+1/i.test(plainOutput) && /PromptScript does not support global installation/i.test(plainOutput) && !/Failed to install\s+[2-9]\d*/i.test(plainOutput)) {
+    return false;
+  }
+  return /Failed to install\s+[1-9]\d*/i.test(plainOutput);
 }
 function runCommand(command, args) {
   return new Promise((resolve, reject) => {
+    let installerOutput = "";
+    const capture = (chunk) => {
+      installerOutput += chunk.toString();
+      if (installerOutput.length > MAX_INSTALLER_OUTPUT_LENGTH) {
+        installerOutput = installerOutput.slice(-MAX_INSTALLER_OUTPUT_LENGTH);
+      }
+    };
     const child = spawn(command, [...args], {
       stdio: ["ignore", "pipe", "pipe"],
       windowsHide: true
     });
-    child.stdout.on("data", (chunk) => process.stderr.write(chunk));
-    child.stderr.on("data", (chunk) => process.stderr.write(chunk));
+    child.stdout.on("data", (chunk) => {
+      capture(chunk);
+      process.stderr.write(chunk);
+    });
+    child.stderr.on("data", (chunk) => {
+      capture(chunk);
+      process.stderr.write(chunk);
+    });
     child.on("error", (error) => reject(
       new CliError(`Could not start the skill installer: ${error.message}`, EXIT.api)
     ));
     child.on("close", (code) => {
-      if (code === 0) {
+      if (code === 0 && !skillInstallerReportedFailures(installerOutput)) {
         resolve();
         return;
       }
+      if (code === 0) {
+        reject(new CliError(
+          "Skill installer reported one or more failed targets.",
+          EXIT.api
+        ));
+        return;
+      }
       reject(new CliError(
         `Skill installation failed with exit code ${code ?? "unknown"}.`,
         EXIT.api
@@ -409,17 +772,19 @@ function runCommand(command, args) {
     });
   });
 }
-async function installSkill(runner = runCommand) {
-  const invocation = skillInstallInvocation();
+async function installSkill(runner = runCommand, detector = determineAgent) {
+  const agent = await detectSkillInstallerAgent(detector);
+  const invocation = skillInstallInvocation(agent);
   await runner(invocation.command, invocation.args);
 }
 function defaultDependencies(apiUrl) {
   const client2 = new ApiClient(apiUrl);
   return {
     hasApiKey: () => Boolean(process.env.SCRAPPYCOCO_API_KEY),
-    clearRefreshToken,
+    clearRefreshToken: clearRefreshToken2,
     loadRefreshToken,
     login,
+    prepareCredentialStorage,
     installSkill,
     listAvailableScrapers: () => client2.get("/scrapers?available_only=true")
   };
@@ -429,17 +794,22 @@ async function performSetup(options, dependencies = defaultDependencies(options.
   let authentication = usingApiKey ? "api_key" : "oauth";
   let credentialStorage = usingApiKey ? "environment" : "existing";
   const hadStoredToken = !usingApiKey && Boolean(await dependencies.loadRefreshToken());
+  if (!usingApiKey && !hadStoredToken) {
+    await dependencies.prepareCredentialStorage();
+  }
+  await dependencies.installSkill();
   if (!usingApiKey && !hadStoredToken) {
     const authenticated = await dependencies.login({
       apiUrl: options.apiUrl,
-      noBrowser: options.noBrowser
+      noBrowser: options.noBrowser,
+      manualCallback: options.manualCallback
     });
     authentication = "oauth";
     credentialStorage = authenticated.storage;
   }
-  let catalog;
+  let catalog2;
   try {
-    catalog = await dependencies.listAvailableScrapers();
+    catalog2 = await dependencies.listAvailableScrapers();
   } catch (error) {
     if (usingApiKey || !hadStoredToken || !(error instanceof CliError) || error.exitCode !== EXIT.auth) {
       throw error;
@@ -447,12 +817,12 @@ async function performSetup(options, dependencies = defaultDependencies(options.
     await dependencies.clearRefreshToken();
     const authenticated = await dependencies.login({
       apiUrl: options.apiUrl,
-      noBrowser: options.noBrowser
+      noBrowser: options.noBrowser,
+      manualCallback: options.manualCallback
     });
     credentialStorage = authenticated.storage;
-    catalog = await dependencies.listAvailableScrapers();
+    catalog2 = await dependencies.listAvailableScrapers();
   }
-  await dependencies.installSkill();
   return {
     authenticated: true,
     authentication,
@@ -463,9 +833,9 @@ async function performSetup(options, dependencies = defaultDependencies(options.
     },
     verification: {
       catalog_reachable: true,
-      available_capabilities: catalog.length
+      available_capabilities: catalog2.length
     },
-    next_step: "Reload or restart your agent, then ask it to list available Scrappycoco scrapers."
+    next_step: "Reload or restart your agent, then ask it to inspect the Scrappycoco catalog."
   };
 }
 
@@ -474,7 +844,7 @@ var packageMetadata = JSON.parse(
   readFileSync(new URL("../package.json", import.meta.url), "utf8")
 );
 var program = new Command();
-program.name("scrappycoco").description("Discover, run, and compare scraper capabilities through one API").version(packageMetadata.version).option("--json", "emit machine-readable JSON to stdout").option("--api-url <url>", "API base URL", process.env.SCRAPPYCOCO_API_URL || "https://api.scrappycoco.ai").showHelpAfterError().exitOverride();
+program.name("scrappycoco").description("Discover configurations and run external-data capabilities through one deterministic API").version(packageMetadata.version).option("--json", "emit machine-readable JSON to stdout").option("--api-url <url>", "API base URL", process.env.SCRAPPYCOCO_API_URL || "https://api.scrappycoco.ai").showHelpAfterError().exitOverride();
 function globals(command) {
   return command.optsWithGlobals();
 }
@@ -484,10 +854,11 @@ function client(command) {
 function collect(value, previous) {
   return [...previous, value];
 }
-program.command("setup").description("Authenticate, install the Scrappycoco skill, and verify the connection").option("--no-browser", "print the authorization URL without opening it").action(async (options, command) => {
+program.command("setup").description("Authenticate, install the Scrappycoco skill, and verify the connection").option("--no-browser", "print the authorization URL without opening it").option("--manual-callback", "paste the final callback URL in this terminal for remote/headless login").action(async (options, command) => {
   const result = await performSetup({
     apiUrl: globals(command).apiUrl,
-    noBrowser: options.browser === false
+    noBrowser: options.browser === false,
+    manualCallback: options.manualCallback
   });
   await emit(result, globals(command).json || false);
 });
@@ -509,6 +880,8 @@ async function requestPayload(options, scraperId) {
     capability,
     input,
     ...options.provider?.length ? { providers: options.provider } : {},
+    ...options.providerOptions ? { provider_options: parseJsonObject(options.providerOptions, "provider options JSON") } : {},
+    ...options.concurrency !== void 0 ? { concurrency: Number(options.concurrency) } : {},
     ...options.limit !== void 0 ? { limit: Number(options.limit) } : {}
   };
 }
@@ -524,8 +897,12 @@ async function emitExecution(response, options, command) {
   await emit(response, jsonMode);
 }
 var auth = program.command("auth").description("Manage Clerk OAuth credentials");
-auth.command("login").option("--no-browser", "print the authorization URL without opening it").action(async (options, command) => {
-  const result = await login({ noBrowser: options.browser === false, apiUrl: globals(command).apiUrl });
+auth.command("login").option("--no-browser", "print the authorization URL without opening it").option("--manual-callback", "paste the final callback URL in this terminal for remote/headless login").action(async (options, command) => {
+  const result = await login({
+    noBrowser: options.browser === false,
+    manualCallback: options.manualCallback,
+    apiUrl: globals(command).apiUrl
+  });
   await emit({ authenticated: true, storage: result.storage }, globals(command).json || false);
 });
 auth.command("status").action(async (_options, command) => {
@@ -536,17 +913,34 @@ auth.command("status").action(async (_options, command) => {
     process.exitCode = EXIT.auth;
     return;
   }
-  const me = await client(command).get("/me");
+  let me;
+  try {
+    me = await client(command).get("/me");
+  } catch (error) {
+    if (error instanceof CliError && error.exitCode === EXIT.auth) {
+      await emit(
+        {
+          authenticated: false,
+          reason: error.message,
+          next_action: usingApiKey ? "Verify or replace the existing SCRAPPYCOCO_API_KEY environment variable." : "Run `scrappycoco auth login` to replace the stored OAuth login."
+        },
+        globals(command).json || false
+      );
+      process.exitCode = EXIT.auth;
+      return;
+    }
+    throw error;
+  }
   await emit(
     { authenticated: true, method: usingApiKey ? "api_key" : "oauth", account: me },
     globals(command).json || false
   );
 });
 auth.command("logout").action(async (_options, command) => {
-  await clearRefreshToken();
+  await clearRefreshToken2();
   await emit({ authenticated: false }, globals(command).json || false);
 });
-var scrapers = program.command("scrapers").description("Inspect and execute scraper capabilities");
+var scrapers = program.command("scrapers", { hidden: true }).description("Legacy scraper commands");
 scrapers.command("list").option("--source <source>", "filter by web, x, reddit, or filings").option("--provider <provider>", "filter by provider implementation").option("--available", "only include available providers").action(async (options, command) => {
   const query = new URLSearchParams();
   if (options.source) query.set("source", options.source);
@@ -564,20 +958,83 @@ scrapers.command("inspect <scraper-id>").action(async (scraperId, _options, comm
     globals(command).json || false
   );
 });
+var catalog = program.command("catalog").description("Inspect capabilities, provider schemas, options, and pricing");
+catalog.command("list").option("--source <source>", "filter by web, x, reddit, or filings").option("--provider <provider>", "filter by provider implementation").option("--available", "only include available providers").action(async (options, command) => {
+  const query = new URLSearchParams();
+  if (options.source) query.set("source", options.source);
+  if (options.provider) query.set("provider", options.provider);
+  if (options.available) query.set("available_only", "true");
+  await emit(
+    await client(command).get(`/scrapers${query.size ? `?${query}` : ""}`),
+    globals(command).json || false
+  );
+});
+catalog.command("inspect <capability-id>").action(async (capabilityId, _options, command) => {
+  const { source, capability } = splitScraperId(capabilityId);
+  await emit(
+    await client(command).get(
+      `/scrapers/${encodeURIComponent(source)}/${encodeURIComponent(capability)}`
+    ),
+    globals(command).json || false
+  );
+});
 function executionCommand(name) {
-  return scrapers.command(`${name} <scraper-id>`).description(name === "run" ? "Run one capability through a provider waterfall" : "Compare providers for one capability").option("-f, --file <path>", "canonical request JSON file").option("--input <json>", "capability input JSON").option("--provider <id>", "provider ID; repeat to choose and order providers", collect, []).option("--limit <number>", "maximum records").option("--idempotency-key <key>", "stable retry key").addOption(new Option("--format <format>", "output record format").choices(["json", "jsonl", "csv"]).default("json")).option("-o, --output <path>", "write records to a file").action(async (scraperId, options, command) => {
+  return scrapers.command(`${name} <scraper-id>`).description(name === "run" ? "Run one capability through a provider waterfall" : "Compare providers for one capability").option("-f, --file <path>", "canonical request JSON file").option("--input <json>", "capability input JSON").option("--provider <id>", "provider ID; repeat to choose and order providers", collect, []).option("--provider-options <json>", "provider-native options keyed by provider ID").option("--concurrency <number>", "batch concurrency (default 3, maximum 10)").option("--limit <number>", "maximum records").option("--idempotency-key <key>", "stable retry key").addOption(new Option("--format <format>", "output record format").choices(["json", "jsonl", "csv"]).default("json")).option("-o, --output <path>", "write records to a file").action(async (scraperId, options, command) => {
     const payload = await requestPayload(options, scraperId);
     if (payload.limit === void 0) payload.limit = 10;
     const response = await client(command).postJob(
       name === "run" ? "/scrapers/jobs" : "/scrapers/compare/jobs",
       payload,
-      options.idempotencyKey || randomUUID2()
+      options.idempotencyKey || randomUUID3()
     );
     await emitExecution(response, options, command);
   });
 }
 executionCommand("run");
 executionCommand("compare");
+program.command("run [capability-id]").description("Run a capability directly; discovery is optional").option("--config <discovery-id>", "run a finalized multi-step configuration").option("-f, --file <path>", "canonical request JSON file").option("--input <json>", "capability input JSON; use url or urls for web.extract_content").option("--provider <id>", "provider ID; repeat for an ordered fallback waterfall", collect, []).option("--provider-options <json>", "provider-native options keyed by provider ID").option("--concurrency <number>", "batch concurrency (default 3, maximum 10)").option("--limit <number>", "maximum records").option("--idempotency-key <key>", "stable retry key").option("--retry-failed <run-id>", "retry only failed URLs from a partial batch run").addOption(new Option("--format <format>", "output record format").choices(["json", "jsonl", "csv"]).default("json")).option("-o, --output <path>", "write records to a file").action(async (capabilityId, options, command) => {
+  if (options.retryFailed) {
+    if (capabilityId || options.config) {
+      throw new CliError("Do not combine --retry-failed with a capability ID or --config.", EXIT.usage);
+    }
+    const response2 = await client(command).postJob(
+      `/runs/${encodeURIComponent(options.retryFailed)}/retry-failed`,
+      {},
+      options.idempotencyKey || randomUUID3()
+    );
+    await emitExecution(response2, options, command);
+    return;
+  }
+  if (options.config) {
+    if (capabilityId) {
+      throw new CliError("Choose either a capability ID or --config.", EXIT.usage);
+    }
+    const fromFile = options.file ? await readJsonFile(options.file) : {};
+    const input = options.input ? parseJsonObject(options.input, "runtime input JSON") : fromFile.input || {};
+    const response2 = await client(command).postJob(
+      `/discoveries/${encodeURIComponent(options.config)}/jobs`,
+      {
+        ...fromFile,
+        input,
+        limit: Number(options.limit ?? fromFile.limit ?? 25)
+      },
+      options.idempotencyKey || randomUUID3()
+    );
+    await emitExecution(response2, options, command);
+    return;
+  }
+  if (!capabilityId) {
+    throw new CliError("Provide a capability ID or --retry-failed <run-id>.", EXIT.usage);
+  }
+  const payload = await requestPayload(options, capabilityId);
+  if (payload.limit === void 0) payload.limit = 10;
+  const response = await client(command).postJob(
+    "/scrapers/jobs",
+    payload,
+    options.idempotencyKey || randomUUID3()
+  );
+  await emitExecution(response, options, command);
+});
 var jobs = program.command("jobs").description("Inspect durable queued jobs");
 jobs.command("get <job-id>").action(async (jobId, _options, command) => {
   await emit(
@@ -585,17 +1042,17 @@ jobs.command("get <job-id>").action(async (jobId, _options, command) => {
     globals(command).json || false
   );
 });
-var providers = program.command("providers").description("Inspect integrated scraper providers");
+var providers = program.command("providers", { hidden: true }).description("Legacy provider commands");
 providers.command("list").option("--available", "only include available provider-capability routes").action(async (options, command) => {
   await emit(
     await client(command).get(`/providers${options.available ? "?available_only=true" : ""}`),
     globals(command).json || false
   );
 });
-var discoveries = program.command("discoveries").description("Design and run saved multi-capability scraper configurations");
-discoveries.command("create").requiredOption("--goal <text>", "natural-language data goal").addOption(new Option("--priority <priority>", "comparison priority").choices(["balanced", "quality", "coverage", "cost", "speed"]).default("balanced")).action(async (options, command) => {
+var discoveries = program.command("discoveries", { hidden: true }).description("Legacy discovery commands");
+discoveries.command("create").requiredOption("-f, --file <path>", "agent-authored discovery JSON with goal and configuration").action(async (options, command) => {
   await emit(
-    await client(command).post("/discoveries", { goal: options.goal, priority: options.priority }),
+    await client(command).post("/discoveries", await readJsonFile(options.file)),
     globals(command).json || false
   );
 });
@@ -629,7 +1086,7 @@ discoveries.command("run <discovery-id>").option("-f, --file <path>", "request J
   const response = await client(command).postJob(
     `/discoveries/${encodeURIComponent(discoveryId)}/jobs`,
     payload,
-    options.idempotencyKey || randomUUID2()
+    options.idempotencyKey || randomUUID3()
   );
   await emitExecution(response, options, command);
 });
@@ -637,6 +1094,50 @@ discoveries.command("delete <discovery-id>").requiredOption("--yes", "confirm pe
   await client(command).delete(`/discoveries/${encodeURIComponent(discoveryId)}`);
   await emit({ deleted: true, discovery_id: discoveryId }, globals(command).json || false);
 });
+program.command("discover").description("Save, sample-test, or finalize an agent-authored configuration").option("-f, --file <path>", "create from agent-authored discovery JSON").option("--id <discovery-id>", "existing discovery ID").option("--test", "approve and run a paid sample test").option("--input <json>", "sample runtime input JSON").option("--update <path>", "replace fields or configuration from agent-authored JSON").option("--finalize", "mark the current explicit configuration finalized").option("--idempotency-key <key>", "stable sample retry key").action(async (options, command) => {
+  const selected = Number(Boolean(options.file)) + Number(Boolean(options.test)) + Number(Boolean(options.update)) + Number(Boolean(options.finalize));
+  if (selected !== 1) {
+    throw new CliError(
+      "Choose exactly one action: --file, --id --update, --id --test, or --id --finalize.",
+      EXIT.usage
+    );
+  }
+  if (options.file) {
+    await emit(
+      await client(command).post("/discoveries", await readJsonFile(options.file)),
+      globals(command).json || false
+    );
+    return;
+  }
+  if (!options.id) throw new CliError("--id is required for --update, --test, and --finalize.", EXIT.usage);
+  if (options.update) {
+    await emit(
+      await client(command).patch(
+        `/discoveries/${encodeURIComponent(options.id)}`,
+        await readJsonFile(options.update)
+      ),
+      globals(command).json || false
+    );
+    return;
+  }
+  if (options.finalize) {
+    await emit(
+      await client(command).post(
+        `/discoveries/${encodeURIComponent(options.id)}/finalize`,
+        {}
+      ),
+      globals(command).json || false
+    );
+    return;
+  }
+  const input = options.input ? parseJsonObject(options.input, "sample input JSON") : {};
+  const response = await client(command).postJob(
+    `/discoveries/${encodeURIComponent(options.id)}/jobs`,
+    { input, limit: 25 },
+    options.idempotencyKey || randomUUID3()
+  );
+  await emit(response, globals(command).json || false);
+});
 program.configureOutput({ writeErr: (text) => process.stderr.write(text) });
 program.parseAsync(process.argv).catch(async (error) => {
   if (error instanceof CommanderError) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@scrappycoco/cli",
-  "version": "0.3.0",
-  "description": "CLI for Scrappycoco scraper discovery, execution, and provider comparison",
+  "version": "0.4.1",
+  "description": "CLI for Scrappycoco scraper discovery and execution",
   "type": "module",
   "bin": {
     "scrappycoco": "dist/index.js"
@@ -22,6 +22,7 @@
     "prepack": "npm run build"
   },
   "dependencies": {
+    "@vercel/detect-agent": "^1.2.3",
     "commander": "^14.0.0",
     "cross-keychain": "^1.1.0",
     "open": "^10.2.0"