@scrappycoco/cli 0.4.0 → 0.4.1

package/README.md

@@ -12,7 +12,23 @@ npx --yes @scrappycoco/cli setup
 
 `setup` opens OAuth in the browser. New users can create an account in that
 flow. After authentication, it installs the public Scrappycoco skill, verifies
-the live catalog, and tells you when to reload or restart your agent.
+the live catalog, and tells you when to reload or restart your agent. Browser
+authorization is not complete until the terminal confirms that the credential
+was saved and the catalog check succeeded.
+
+When the CLI runs on a remote or headless host whose `127.0.0.1` is not the
+browser's localhost, keep the command running and use the manual callback
+flow:
+
+```sh
+npx --yes @scrappycoco/cli setup --no-browser --manual-callback
+```
+
+Open the printed authorization URL, then paste the browser's final callback
+URL into that same terminal. The callback contains a short-lived
+authorization code: never paste it into chat, logs, or a saved file. A timed
+out or exited login cannot be resumed; start a new command and use its new
+URL.
 
 Individual commands remain available:
 
@@ -22,6 +38,7 @@ npx --yes @scrappycoco/cli catalog list --available --json
 npx --yes @scrappycoco/cli catalog inspect web.extract_content --json
 npx --yes @scrappycoco/cli run web.extract_content --file request.json --json
 npx --yes @scrappycoco/cli discover --file discovery.json --json
+npx --yes @scrappycoco/cli discover --id DISCOVERY_ID --test --input '{"url":"https://example.com"}' --json
 npx --yes @scrappycoco/cli discover --id DISCOVERY_ID --finalize --json
 npx --yes @scrappycoco/cli run --config DISCOVERY_ID --input '{}' --json
 ```
@@ -35,6 +52,10 @@ assistant, agent, workflow, or schedule commands.
 
 Node.js 20 or newer is required. Interactive use authenticates with Clerk OAuth Authorization Code + PKCE. CI can set `SCRAPPYCOCO_API_KEY`.
 
+OAuth and API requests have bounded timeouts. Set
+`SCRAPPYCOCO_AUTH_TIMEOUT_MS` or `SCRAPPYCOCO_API_TIMEOUT_MS` only when a slow
+network requires a longer window.
+
 Use `--json` for machine-readable responses. Execution commands support
 `--format json|jsonl|csv` with `--output`, provider-native
 `--provider-options`, batch `--concurrency`, and an explicit
@@ -43,6 +64,12 @@ poll for completion; set `SCRAPPYCOCO_JOB_TIMEOUT_MS` to change the 20-minute
 local wait. If a command times out while its job continues, inspect it with
 `scrappycoco jobs get <job-id>`.
 
+When provider or configuration choice is uncertain, define 2 to 5 named
+`candidates` in a Discover route. Each candidate has its own `id`, `provider`,
+and `options`, so one discovery can test both `zyte-http` with
+`{"browser_html":false}` and `zyte-browser` with
+`{"browser_html":true}` against the same input.
+
 Use `scrappycoco --help` for the complete command reference. See the
 [Scrappycoco API documentation](https://scrappycoco.ai/docs) for the public
 contract.

package/dist/index.js

@@ -1,13 +1,14 @@
 #!/usr/bin/env node
 
 // src/index.ts
-import { randomUUID as randomUUID2 } from "crypto";
+import { randomUUID as randomUUID3 } from "crypto";
 import { readFileSync } from "fs";
 import { Command, CommanderError, Option } from "commander";
 
 // src/auth.ts
 import { createHash, randomBytes, timingSafeEqual } from "crypto";
 import { createServer } from "http";
+import { createInterface } from "readline/promises";
 import open2 from "open";
 
 // src/errors.ts
@@ -32,19 +33,25 @@ var CliError = class extends Error {
 };
 
 // src/storage.ts
-import { chmod, mkdir, open, readFile, rm, stat, writeFile } from "fs/promises";
+import { randomUUID } from "crypto";
+import { chmod, mkdir, open, readFile, rename, rm, stat, writeFile } from "fs/promises";
 import { homedir } from "os";
 import { dirname, join } from "path";
 import { deletePassword, getPassword, setPassword } from "cross-keychain";
 var SERVICE = "scrappycoco-cli";
 var ACCOUNT = "refresh-token";
 function fallbackCredentialPath() {
+  const configured = process.env.SCRAPPYCOCO_CONFIG_HOME;
+  if (configured) return join(configured, "credentials.json");
   const base = process.platform === "win32" ? process.env.APPDATA || join(homedir(), "AppData", "Roaming") : process.env.XDG_CONFIG_HOME || join(homedir(), ".config");
   return join(base, "scrappycoco", "credentials.json");
 }
 function credentialRefreshLockPath() {
   return `${fallbackCredentialPath()}.refresh.lock`;
 }
+function credentialStatePath() {
+  return `${fallbackCredentialPath()}.state`;
+}
 var LOCK_RETRY_MS = 100;
 var LOCK_TIMEOUT_MS = 3e4;
 var LOCK_STALE_MS = 6e4;
@@ -93,23 +100,89 @@ async function readFallback() {
     return null;
   }
 }
-async function loadRefreshToken() {
+async function readKeychain() {
   try {
     const value = await getPassword(SERVICE, ACCOUNT);
-    if (value) return value;
+    return value || null;
+  } catch {
+    return null;
+  }
+}
+async function readCredentialState() {
+  try {
+    const value = JSON.parse(await readFile(credentialStatePath(), "utf8"));
+    return value.storage === "keychain" || value.storage === "file" || value.storage === "none" ? value.storage : null;
+  } catch {
+    return null;
+  }
+}
+async function writePrivateJson(path, value) {
+  await mkdir(dirname(path), { recursive: true, mode: 448 });
+  const temporary = `${path}.${process.pid}.${randomUUID()}.tmp`;
+  try {
+    await writeFile(temporary, JSON.stringify(value), { mode: 384 });
+    if (process.platform !== "win32") await chmod(temporary, 384);
+    await rename(temporary, path);
+    if (process.platform !== "win32") await chmod(path, 384);
+  } finally {
+    await rm(temporary, { force: true });
+  }
+}
+async function writeCredentialState(storage) {
+  await writePrivateJson(credentialStatePath(), { storage });
+}
+async function loadRefreshTokenCandidates() {
+  const state = await readCredentialState();
+  if (state === "none") return [];
+  if (state === "file") {
+    const value = await readFallback();
+    return value ? [{ value, storage: "file" }] : [];
+  }
+  if (state === "keychain") {
+    const value = await readKeychain();
+    return value ? [{ value, storage: "keychain" }] : [];
+  }
+  const [keychain, file] = await Promise.all([readKeychain(), readFallback()]);
+  const candidates = [];
+  if (keychain) candidates.push({ value: keychain, storage: "keychain" });
+  if (file && file !== keychain) candidates.push({ value: file, storage: "file" });
+  return candidates;
+}
+async function prepareCredentialStorage() {
+  const state = await readCredentialState();
+  if (state === "file") {
+    await assertFallbackWritable();
+    return "file";
+  }
+  try {
+    await getPassword(SERVICE, ACCOUNT);
+    return "keychain";
   } catch {
+    await assertFallbackWritable();
+    return "file";
   }
-  return readFallback();
+}
+async function assertFallbackWritable() {
+  const path = fallbackCredentialPath();
+  await mkdir(dirname(path), { recursive: true, mode: 448 });
+  const probe = `${path}.write-test-${process.pid}-${randomUUID()}`;
+  const handle = await open(probe, "w", 384);
+  await handle.close();
+  await rm(probe, { force: true });
 }
 async function saveRefreshToken(refreshToken) {
   try {
     await setPassword(SERVICE, ACCOUNT, refreshToken);
+    await rm(fallbackCredentialPath(), { force: true });
+    await writeCredentialState("keychain");
     return "keychain";
   } catch {
-    const path = fallbackCredentialPath();
-    await mkdir(dirname(path), { recursive: true, mode: 448 });
-    await writeFile(path, JSON.stringify({ refresh_token: refreshToken }), { mode: 384 });
-    if (process.platform !== "win32") await chmod(path, 384);
+    await writePrivateJson(fallbackCredentialPath(), { refresh_token: refreshToken });
+    await writeCredentialState("file");
+    try {
+      await deletePassword(SERVICE, ACCOUNT);
+    } catch {
+    }
     return "file";
   }
 }
@@ -119,11 +192,42 @@ async function clearRefreshToken() {
   } catch {
   }
   await rm(fallbackCredentialPath(), { force: true });
+  await writeCredentialState("none");
 }
 
 // src/auth.ts
 var cachedAccessToken;
 var refreshesInFlight = /* @__PURE__ */ new Map();
+var DEFAULT_AUTH_HTTP_TIMEOUT_MS = 15e3;
+function positiveInteger(value, fallback) {
+  if (!value) return fallback;
+  const parsed = Number(value);
+  return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback;
+}
+function authHttpTimeoutMs() {
+  return positiveInteger(process.env.SCRAPPYCOCO_AUTH_TIMEOUT_MS, DEFAULT_AUTH_HTTP_TIMEOUT_MS);
+}
+function networkMessage(error) {
+  if (error instanceof DOMException && (error.name === "AbortError" || error.name === "TimeoutError")) {
+    return `timed out after ${Math.ceil(authHttpTimeoutMs() / 1e3)} seconds`;
+  }
+  return "failed to reach the authentication service";
+}
+async function oauthFetch(url, init, phase, exitCode) {
+  try {
+    return await fetch(url, {
+      ...init,
+      signal: AbortSignal.timeout(authHttpTimeoutMs())
+    });
+  } catch (error) {
+    const timeout = error instanceof DOMException && (error.name === "AbortError" || error.name === "TimeoutError");
+    throw new CliError(
+      `OAuth ${phase} ${networkMessage(error)}. ${phase === "authorization-code exchange" ? "Authorization was received but was not saved; run `scrappycoco auth login` again." : "Check the connection and try again."}`,
+      timeout ? EXIT.timeout : exitCode,
+      { phase, network_error: true }
+    );
+  }
+}
 async function oauthConfig(apiUrl) {
   const clientId = process.env.SCRAPPYCOCO_OAUTH_CLIENT_ID;
   if (clientId) {
@@ -133,7 +237,12 @@ async function oauthConfig(apiUrl) {
       scopes: ["openid", "profile", "email"]
     };
   }
-  const response = await fetch(`${apiUrl.replace(/\/$/, "")}/api/v1/oauth/cli-config`);
+  const response = await oauthFetch(
+    `${apiUrl.replace(/\/$/, "")}/api/v1/oauth/cli-config`,
+    {},
+    "configuration request",
+    EXIT.api
+  );
   const payload = await response.json().catch(() => ({}));
   if (!response.ok || !payload.issuer || !payload.client_id || !Array.isArray(payload.scopes)) {
     throw new CliError(payload.detail || "Scrappycoco CLI OAuth is not configured.", EXIT.auth);
@@ -156,49 +265,99 @@ function statesMatch(expected, received) {
   const right = Buffer.from(received);
   return left.length === right.length && timingSafeEqual(left, right);
 }
-async function tokenRequest(issuer, body) {
-  const response = await fetch(`${issuer.replace(/\/$/, "")}/oauth/token`, {
-    method: "POST",
-    headers: { "content-type": "application/x-www-form-urlencoded" },
-    body
-  });
+async function tokenRequest(issuer, body, phase) {
+  const response = await oauthFetch(
+    `${issuer.replace(/\/$/, "")}/oauth/token`,
+    {
+      method: "POST",
+      headers: { "content-type": "application/x-www-form-urlencoded" },
+      body
+    },
+    phase,
+    EXIT.auth
+  );
   const payload = await response.json().catch(() => ({}));
   if (!response.ok || typeof payload.access_token !== "string") {
-    throw new CliError(String(payload.error_description || payload.error || "OAuth token exchange failed."), EXIT.auth);
+    throw new CliError(
+      String(payload.error_description || payload.error || `OAuth ${phase} failed.`),
+      EXIT.auth,
+      {
+        phase,
+        status: response.status,
+        ...typeof payload.error === "string" ? { oauth_error: payload.error } : {}
+      }
+    );
   }
   return payload;
 }
+function callbackResult(rawUrl, expectedState, redirectUri) {
+  const current = new URL(rawUrl, redirectUri);
+  const expected = new URL(redirectUri);
+  if (current.protocol !== expected.protocol || current.hostname !== expected.hostname || current.port !== expected.port || current.pathname !== expected.pathname) {
+    throw new CliError("OAuth callback URL did not match this login attempt.", EXIT.auth);
+  }
+  const state = current.searchParams.get("state") || "";
+  const providerError = current.searchParams.get("error");
+  if (!statesMatch(expectedState, state)) {
+    throw new CliError("OAuth state validation failed.", EXIT.auth);
+  }
+  if (providerError) {
+    const description = current.searchParams.get("error_description") || providerError;
+    throw new CliError(`OAuth authorization was denied: ${description}`, EXIT.auth);
+  }
+  const code = current.searchParams.get("code") || "";
+  if (!code) throw new CliError("OAuth callback did not include an authorization code.", EXIT.auth);
+  return { code, redirectUri };
+}
 async function login(options) {
   const config = await oauthConfig(options.apiUrl);
   const clientId = config.client_id;
   const pkce = createPkce();
   let timeout;
+  let manualReader;
   const callback = new Promise((resolve, reject) => {
+    let settled = false;
+    const finish = (result, error) => {
+      if (settled) return;
+      settled = true;
+      if (timeout) clearTimeout(timeout);
+      manualReader?.close();
+      server.close();
+      if (error) reject(error);
+      else if (result) resolve(result);
+    };
     const server = createServer((request, response) => {
-      const current = new URL(request.url || "/", "http://127.0.0.1");
-      if (current.pathname !== "/callback") {
-        response.writeHead(404).end("Not found");
+      const address = server.address();
+      if (!address || typeof address === "string") {
+        response.writeHead(500, { "content-type": "text/plain" }).end("OAuth callback failed.");
+        finish(void 0, new CliError("OAuth callback failed.", EXIT.auth));
         return;
       }
-      const state = current.searchParams.get("state") || "";
-      const code2 = current.searchParams.get("code") || "";
-      if (!statesMatch(pkce.state, state) || !code2) {
-        response.writeHead(400, { "content-type": "text/plain" }).end("Invalid OAuth callback. You may close this window.");
-        clearTimeout(timeout);
-        server.close();
-        reject(new CliError("OAuth state validation failed.", EXIT.auth));
+      const redirectUri2 = `http://127.0.0.1:${address.port}/callback`;
+      const current = new URL(request.url || "/", redirectUri2);
+      if (current.pathname !== "/callback") {
+        response.writeHead(404, { "content-type": "text/plain" }).end("Not found");
         return;
       }
-      const address = server.address();
-      if (!address || typeof address === "string") return reject(new CliError("OAuth callback failed.", EXIT.auth));
-      response.writeHead(200, { "content-type": "text/plain" }).end("Scrappycoco login complete. You may close this window.");
-      clearTimeout(timeout);
-      server.close();
-      resolve({ code: code2, redirectUri: `http://127.0.0.1:${address.port}/callback` });
+      try {
+        const result = callbackResult(request.url || "/", pkce.state, redirectUri2);
+        response.writeHead(200, { "content-type": "text/plain" }).end(
+          "Scrappycoco authorization received. Keep the terminal open while it securely saves the login; the terminal will report when it is complete."
+        );
+        finish(result);
+      } catch (error) {
+        response.writeHead(400, { "content-type": "text/plain" }).end(
+          error instanceof Error ? error.message : "Invalid OAuth callback."
+        );
+        finish(void 0, error);
+      }
     });
     server.listen(0, "127.0.0.1", async () => {
       const address = server.address();
-      if (!address || typeof address === "string") return reject(new CliError("Could not start OAuth callback.", EXIT.auth));
+      if (!address || typeof address === "string") {
+        finish(void 0, new CliError("Could not start OAuth callback.", EXIT.auth));
+        return;
+      }
       const redirectUri2 = `http://127.0.0.1:${address.port}/callback`;
       const authorize = new URL(`${config.issuer.replace(/\/$/, "")}/oauth/authorize`);
       authorize.search = new URLSearchParams({
@@ -213,11 +372,32 @@ async function login(options) {
       process.stderr.write(`Open this URL to authenticate:
 ${authorize.toString()}
 `);
-      if (!options.noBrowser) await open2(authorize.toString());
+      if (options.manualCallback) {
+        process.stderr.write(
+          "After signing in, paste the final callback URL into this terminal. Never paste it into chat.\n"
+        );
+        manualReader = createInterface({ input: process.stdin, output: process.stderr });
+        void manualReader.question("Callback URL: ").then((rawUrl) => {
+          try {
+            finish(callbackResult(rawUrl.trim(), pkce.state, redirectUri2));
+          } catch (error) {
+            finish(void 0, error);
+          }
+        }).catch((error) => finish(void 0, error));
+      }
+      if (!options.noBrowser) {
+        try {
+          await open2(authorize.toString());
+        } catch {
+          process.stderr.write("Could not open a browser automatically; use the URL above.\n");
+        }
+      }
     });
     timeout = setTimeout(() => {
-      server.close();
-      reject(new CliError("OAuth login timed out.", EXIT.timeout));
+      finish(void 0, new CliError(
+        "OAuth login timed out. The printed URL and callback port are no longer valid; start a new login.",
+        EXIT.timeout
+      ));
     }, 5 * 6e4);
   });
   const { code, redirectUri } = await callback;
@@ -227,7 +407,7 @@ ${authorize.toString()}
     code,
     code_verifier: pkce.verifier,
     redirect_uri: redirectUri
-  }));
+  }), "authorization-code exchange");
   if (!tokens.refresh_token) throw new CliError("OAuth response did not include a refresh token.", EXIT.auth);
   invalidateAccessToken();
   return { storage: await saveRefreshToken(tokens.refresh_token) };
@@ -251,16 +431,29 @@ async function accessToken(apiUrl) {
       return cachedAccessToken.value;
     }
     const config = await oauthConfig(normalizedApiUrl);
-    const refreshToken = await loadRefreshToken();
-    if (!refreshToken) throw new CliError("Not logged in. Run `scrappycoco auth login`.", EXIT.auth);
-    const tokens = await tokenRequest(config.issuer, new URLSearchParams({
-      grant_type: "refresh_token",
-      client_id: config.client_id,
-      refresh_token: refreshToken
-    }));
-    if (tokens.refresh_token && tokens.refresh_token !== refreshToken) {
-      await saveRefreshToken(tokens.refresh_token);
+    const candidates = await loadRefreshTokenCandidates();
+    if (!candidates.length) throw new CliError("Not logged in. Run `scrappycoco auth login`.", EXIT.auth);
+    let tokens;
+    let acceptedRefreshToken;
+    let lastAuthError;
+    for (const candidate of candidates) {
+      try {
+        tokens = await tokenRequest(config.issuer, new URLSearchParams({
+          grant_type: "refresh_token",
+          client_id: config.client_id,
+          refresh_token: candidate.value
+        }), "token refresh");
+        acceptedRefreshToken = candidate.value;
+        break;
+      } catch (error) {
+        if (!(error instanceof CliError) || error.exitCode !== EXIT.auth || typeof error.details === "object" && error.details !== null && "network_error" in error.details) {
+          throw error;
+        }
+        lastAuthError = error;
+      }
     }
+    if (!tokens || !acceptedRefreshToken) throw lastAuthError || new CliError("Stored OAuth login is no longer valid.", EXIT.auth);
+    await saveRefreshToken(tokens.refresh_token || acceptedRefreshToken);
     cachedAccessToken = {
       apiUrl: normalizedApiUrl,
       value: tokens.access_token,
@@ -277,14 +470,18 @@ async function accessToken(apiUrl) {
     }
   }
 }
+async function loadRefreshToken() {
+  return (await loadRefreshTokenCandidates())[0]?.value || null;
+}
 
 // src/client.ts
-import { randomUUID } from "crypto";
+import { randomUUID as randomUUID2 } from "crypto";
 var DEFAULT_API_URL = process.env.SCRAPPYCOCO_API_URL || "https://api.scrappycoco.ai";
 var DEFAULT_JOB_TIMEOUT_MS = 20 * 60 * 1e3;
 var DEFAULT_JOB_POLL_INITIAL_MS = 250;
 var DEFAULT_JOB_POLL_MAX_MS = 5e3;
-function positiveInteger(value, fallback) {
+var DEFAULT_API_REQUEST_TIMEOUT_MS = 3e4;
+function positiveInteger2(value, fallback) {
   if (!value) return fallback;
   const parsed = Number(value);
   return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback;
@@ -315,12 +512,28 @@ var ApiClient = class {
   async request(method, path, body, headers, signal) {
     const url = `${this.baseUrl.replace(/\/$/, "")}/api/v1${path}`;
     const requestBody = body === void 0 ? void 0 : JSON.stringify(body);
-    const send = async () => fetch(url, {
-      method,
-      headers: await this.headers(headers),
-      body: requestBody,
-      signal
-    });
+    const send = async () => {
+      const timeoutSignal = AbortSignal.timeout(
+        positiveInteger2(process.env.SCRAPPYCOCO_API_TIMEOUT_MS, DEFAULT_API_REQUEST_TIMEOUT_MS)
+      );
+      const requestSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
+      const requestHeaders = await this.headers(headers);
+      try {
+        return await fetch(url, {
+          method,
+          headers: requestHeaders,
+          body: requestBody,
+          signal: requestSignal
+        });
+      } catch (error) {
+        const timeout = error instanceof DOMException && (error.name === "AbortError" || error.name === "TimeoutError");
+        throw new CliError(
+          timeout ? `Scrappycoco API request timed out: ${method} ${path}.` : `Could not reach the Scrappycoco API: ${method} ${path}.`,
+          timeout ? EXIT.timeout : EXIT.api,
+          { method, path }
+        );
+      }
+    };
     let response = await send();
     if (response.status === 401 && !process.env.SCRAPPYCOCO_API_KEY) {
       invalidateAccessToken();
@@ -338,17 +551,17 @@ var ApiClient = class {
   get(path, signal) {
     return this.request("GET", path, void 0, void 0, signal);
   }
-  post(path, body, key = randomUUID()) {
+  post(path, body, key = randomUUID2()) {
     return this.request("POST", path, body, { "Idempotency-Key": key });
   }
-  async postJob(path, body, key = randomUUID()) {
+  async postJob(path, body, key = randomUUID2()) {
     const submitted = await this.post(path, body, key);
-    const timeoutMs = positiveInteger(process.env.SCRAPPYCOCO_JOB_TIMEOUT_MS, DEFAULT_JOB_TIMEOUT_MS);
-    const initialDelayMs = positiveInteger(
+    const timeoutMs = positiveInteger2(process.env.SCRAPPYCOCO_JOB_TIMEOUT_MS, DEFAULT_JOB_TIMEOUT_MS);
+    const initialDelayMs = positiveInteger2(
       process.env.SCRAPPYCOCO_JOB_POLL_INITIAL_MS,
       DEFAULT_JOB_POLL_INITIAL_MS
     );
-    const maxDelayMs = positiveInteger(process.env.SCRAPPYCOCO_JOB_POLL_MAX_MS, DEFAULT_JOB_POLL_MAX_MS);
+    const maxDelayMs = positiveInteger2(process.env.SCRAPPYCOCO_JOB_POLL_MAX_MS, DEFAULT_JOB_POLL_MAX_MS);
     const deadline = Date.now() + timeoutMs;
     let delayMs = Math.min(initialDelayMs, maxDelayMs);
     let job = submitted;
@@ -490,7 +703,20 @@ async function detectSkillInstallerAgent(detector = determineAgent) {
   if (!detected.isAgent || !detected.agent) return "universal";
   return SKILL_INSTALLER_AGENTS[detected.agent.name] || "universal";
 }
-function skillInstallInvocation(agent = "universal") {
+function skillInstallInvocation(agent = "universal", npmExecPath = process.env.npm_execpath) {
+  if (npmExecPath && /(?:^|[/\\])pnpm(?:\.c?js)?$/i.test(npmExecPath)) {
+    return {
+      command: process.execPath,
+      args: [
+        npmExecPath,
+        "dlx",
+        "skills",
+        ...SKILL_INSTALL_ARGS.slice(2),
+        "--agent",
+        agent
+      ]
+    };
+  }
   return {
     command: npxCommand(),
     args: [...SKILL_INSTALL_ARGS, "--agent", agent]
@@ -558,6 +784,7 @@ function defaultDependencies(apiUrl) {
     clearRefreshToken: clearRefreshToken2,
     loadRefreshToken,
     login,
+    prepareCredentialStorage,
     installSkill,
     listAvailableScrapers: () => client2.get("/scrapers?available_only=true")
   };
@@ -567,10 +794,15 @@ async function performSetup(options, dependencies = defaultDependencies(options.
   let authentication = usingApiKey ? "api_key" : "oauth";
   let credentialStorage = usingApiKey ? "environment" : "existing";
   const hadStoredToken = !usingApiKey && Boolean(await dependencies.loadRefreshToken());
+  if (!usingApiKey && !hadStoredToken) {
+    await dependencies.prepareCredentialStorage();
+  }
+  await dependencies.installSkill();
   if (!usingApiKey && !hadStoredToken) {
     const authenticated = await dependencies.login({
       apiUrl: options.apiUrl,
-      noBrowser: options.noBrowser
+      noBrowser: options.noBrowser,
+      manualCallback: options.manualCallback
     });
     authentication = "oauth";
     credentialStorage = authenticated.storage;
@@ -585,12 +817,12 @@ async function performSetup(options, dependencies = defaultDependencies(options.
     await dependencies.clearRefreshToken();
     const authenticated = await dependencies.login({
       apiUrl: options.apiUrl,
-      noBrowser: options.noBrowser
+      noBrowser: options.noBrowser,
+      manualCallback: options.manualCallback
     });
     credentialStorage = authenticated.storage;
     catalog2 = await dependencies.listAvailableScrapers();
   }
-  await dependencies.installSkill();
   return {
     authenticated: true,
     authentication,
@@ -622,10 +854,11 @@ function client(command) {
 function collect(value, previous) {
   return [...previous, value];
 }
-program.command("setup").description("Authenticate, install the Scrappycoco skill, and verify the connection").option("--no-browser", "print the authorization URL without opening it").action(async (options, command) => {
+program.command("setup").description("Authenticate, install the Scrappycoco skill, and verify the connection").option("--no-browser", "print the authorization URL without opening it").option("--manual-callback", "paste the final callback URL in this terminal for remote/headless login").action(async (options, command) => {
   const result = await performSetup({
     apiUrl: globals(command).apiUrl,
-    noBrowser: options.browser === false
+    noBrowser: options.browser === false,
+    manualCallback: options.manualCallback
   });
   await emit(result, globals(command).json || false);
 });
@@ -664,8 +897,12 @@ async function emitExecution(response, options, command) {
   await emit(response, jsonMode);
 }
 var auth = program.command("auth").description("Manage Clerk OAuth credentials");
-auth.command("login").option("--no-browser", "print the authorization URL without opening it").action(async (options, command) => {
-  const result = await login({ noBrowser: options.browser === false, apiUrl: globals(command).apiUrl });
+auth.command("login").option("--no-browser", "print the authorization URL without opening it").option("--manual-callback", "paste the final callback URL in this terminal for remote/headless login").action(async (options, command) => {
+  const result = await login({
+    noBrowser: options.browser === false,
+    manualCallback: options.manualCallback,
+    apiUrl: globals(command).apiUrl
+  });
   await emit({ authenticated: true, storage: result.storage }, globals(command).json || false);
 });
 auth.command("status").action(async (_options, command) => {
@@ -676,7 +913,24 @@ auth.command("status").action(async (_options, command) => {
     process.exitCode = EXIT.auth;
     return;
   }
-  const me = await client(command).get("/me");
+  let me;
+  try {
+    me = await client(command).get("/me");
+  } catch (error) {
+    if (error instanceof CliError && error.exitCode === EXIT.auth) {
+      await emit(
+        {
+          authenticated: false,
+          reason: error.message,
+          next_action: usingApiKey ? "Verify or replace the existing SCRAPPYCOCO_API_KEY environment variable." : "Run `scrappycoco auth login` to replace the stored OAuth login."
+        },
+        globals(command).json || false
+      );
+      process.exitCode = EXIT.auth;
+      return;
+    }
+    throw error;
+  }
   await emit(
     { authenticated: true, method: usingApiKey ? "api_key" : "oauth", account: me },
     globals(command).json || false
@@ -731,14 +985,14 @@ function executionCommand(name) {
     const response = await client(command).postJob(
       name === "run" ? "/scrapers/jobs" : "/scrapers/compare/jobs",
       payload,
-      options.idempotencyKey || randomUUID2()
+      options.idempotencyKey || randomUUID3()
     );
     await emitExecution(response, options, command);
   });
 }
 executionCommand("run");
 executionCommand("compare");
-program.command("run [capability-id]").description("Run a capability directly; discovery is optional").option("--config <discovery-id>", "run a finalized multi-step configuration").option("-f, --file <path>", "canonical request JSON file").option("--input <json>", "capability input JSON; use url or urls for web.extract_content").option("--provider <id>", "one provider selected by the calling agent", collect, []).option("--provider-options <json>", "provider-native options keyed by provider ID").option("--concurrency <number>", "batch concurrency (default 3, maximum 10)").option("--limit <number>", "maximum records").option("--idempotency-key <key>", "stable retry key").option("--retry-failed <run-id>", "retry only failed URLs from a partial batch run").addOption(new Option("--format <format>", "output record format").choices(["json", "jsonl", "csv"]).default("json")).option("-o, --output <path>", "write records to a file").action(async (capabilityId, options, command) => {
+program.command("run [capability-id]").description("Run a capability directly; discovery is optional").option("--config <discovery-id>", "run a finalized multi-step configuration").option("-f, --file <path>", "canonical request JSON file").option("--input <json>", "capability input JSON; use url or urls for web.extract_content").option("--provider <id>", "provider ID; repeat for an ordered fallback waterfall", collect, []).option("--provider-options <json>", "provider-native options keyed by provider ID").option("--concurrency <number>", "batch concurrency (default 3, maximum 10)").option("--limit <number>", "maximum records").option("--idempotency-key <key>", "stable retry key").option("--retry-failed <run-id>", "retry only failed URLs from a partial batch run").addOption(new Option("--format <format>", "output record format").choices(["json", "jsonl", "csv"]).default("json")).option("-o, --output <path>", "write records to a file").action(async (capabilityId, options, command) => {
   if (options.retryFailed) {
     if (capabilityId || options.config) {
       throw new CliError("Do not combine --retry-failed with a capability ID or --config.", EXIT.usage);
@@ -746,7 +1000,7 @@ program.command("run [capability-id]").description("Run a capability directly; d
     const response2 = await client(command).postJob(
       `/runs/${encodeURIComponent(options.retryFailed)}/retry-failed`,
       {},
-      options.idempotencyKey || randomUUID2()
+      options.idempotencyKey || randomUUID3()
     );
     await emitExecution(response2, options, command);
     return;
@@ -764,7 +1018,7 @@ program.command("run [capability-id]").description("Run a capability directly; d
         input,
         limit: Number(options.limit ?? fromFile.limit ?? 25)
       },
-      options.idempotencyKey || randomUUID2()
+      options.idempotencyKey || randomUUID3()
     );
     await emitExecution(response2, options, command);
     return;
@@ -777,7 +1031,7 @@ program.command("run [capability-id]").description("Run a capability directly; d
   const response = await client(command).postJob(
     "/scrapers/jobs",
     payload,
-    options.idempotencyKey || randomUUID2()
+    options.idempotencyKey || randomUUID3()
   );
   await emitExecution(response, options, command);
 });
@@ -832,7 +1086,7 @@ discoveries.command("run <discovery-id>").option("-f, --file <path>", "request J
   const response = await client(command).postJob(
     `/discoveries/${encodeURIComponent(discoveryId)}/jobs`,
     payload,
-    options.idempotencyKey || randomUUID2()
+    options.idempotencyKey || randomUUID3()
   );
   await emitExecution(response, options, command);
 });
@@ -880,7 +1134,7 @@ program.command("discover").description("Save, sample-test, or finalize an agent
   const response = await client(command).postJob(
     `/discoveries/${encodeURIComponent(options.id)}/jobs`,
     { input, limit: 25 },
-    options.idempotencyKey || randomUUID2()
+    options.idempotencyKey || randomUUID3()
   );
   await emit(response, globals(command).json || false);
 });

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@scrappycoco/cli",
-  "version": "0.4.0",
-  "description": "CLI for Scrappycoco scraper discovery, execution, and provider comparison",
+  "version": "0.4.1",
+  "description": "CLI for Scrappycoco scraper discovery and execution",
   "type": "module",
   "bin": {
     "scrappycoco": "dist/index.js"