@scrappycoco/cli 0.8.3 → 0.8.5

package/README.md

@@ -66,6 +66,9 @@ agent, general workflow, or scheduler commands. For repeated checks, callers
 schedule ordinary Runs and pass each returned cursor into the next Run.
 
 Node.js 20 or newer is required. Interactive use authenticates with Clerk OAuth Authorization Code + PKCE. CI can set `SCRAPPYCOCO_API_KEY`.
+API keys have no refresh-token rotation step and are preferred for unattended
+concurrent application workloads. OAuth refresh is serialized across CLI
+processes and recovers if another process rotates the stored token first.
 
 OAuth and API requests have bounded timeouts. Set
 `SCRAPPYCOCO_AUTH_TIMEOUT_MS` or `SCRAPPYCOCO_API_TIMEOUT_MS` only when a slow

package/dist/index.js

@@ -60,17 +60,23 @@ function wait(milliseconds) {
 }
 async function withCredentialRefreshLock(operation) {
   const path = credentialRefreshLockPath();
+  const owner = randomUUID();
   await mkdir(dirname(path), { recursive: true, mode: 448 });
   const deadline = Date.now() + LOCK_TIMEOUT_MS;
   while (true) {
     try {
       const handle = await open(path, "wx", 384);
       try {
-        await handle.writeFile(JSON.stringify({ pid: process.pid, created_at: Date.now() }));
+        await handle.writeFile(JSON.stringify({ owner, pid: process.pid, created_at: Date.now() }));
         return await operation();
       } finally {
         await handle.close();
-        await rm(path, { force: true });
+        try {
+          const current = JSON.parse(await readFile(path, "utf8"));
+          if (current.owner === owner) await rm(path, { force: true });
+        } catch (error) {
+          if (error.code !== "ENOENT") throw error;
+        }
       }
     } catch (error) {
       const code = error.code;
@@ -199,6 +205,8 @@ async function clearRefreshToken() {
 var cachedAccessToken;
 var refreshesInFlight = /* @__PURE__ */ new Map();
 var DEFAULT_AUTH_HTTP_TIMEOUT_MS = 15e3;
+var ROTATED_TOKEN_RELOAD_ATTEMPTS = 10;
+var ROTATED_TOKEN_RELOAD_DELAY_MS = 100;
 function positiveInteger(value, fallback) {
   if (!value) return fallback;
   const parsed = Number(value);
@@ -213,6 +221,12 @@ function networkMessage(error) {
   }
   return "failed to reach the authentication service";
 }
+function isInvalidGrant(error) {
+  return error instanceof CliError && error.exitCode === EXIT.auth && typeof error.details === "object" && error.details !== null && "oauth_error" in error.details && error.details.oauth_error === "invalid_grant";
+}
+function wait2(milliseconds) {
+  return new Promise((resolve) => setTimeout(resolve, milliseconds));
+}
 async function oauthFetch(url, init, phase, exitCode) {
   try {
     return await fetch(url, {
@@ -436,7 +450,9 @@ async function accessToken(apiUrl) {
     let tokens;
     let acceptedRefreshToken;
     let lastAuthError;
+    const attemptedRefreshTokens = /* @__PURE__ */ new Set();
     for (const candidate of candidates) {
+      attemptedRefreshTokens.add(candidate.value);
       try {
         tokens = await tokenRequest(config.issuer, new URLSearchParams({
           grant_type: "refresh_token",
@@ -452,6 +468,28 @@ async function accessToken(apiUrl) {
         lastAuthError = error;
       }
     }
+    if (!tokens && isInvalidGrant(lastAuthError)) {
+      for (let attempt = 0; attempt < ROTATED_TOKEN_RELOAD_ATTEMPTS && !tokens; attempt += 1) {
+        await wait2(ROTATED_TOKEN_RELOAD_DELAY_MS);
+        const rotatedCandidates = await loadRefreshTokenCandidates();
+        const rotated = rotatedCandidates.find(
+          (candidate) => !attemptedRefreshTokens.has(candidate.value)
+        );
+        if (!rotated) continue;
+        attemptedRefreshTokens.add(rotated.value);
+        try {
+          tokens = await tokenRequest(config.issuer, new URLSearchParams({
+            grant_type: "refresh_token",
+            client_id: config.client_id,
+            refresh_token: rotated.value
+          }), "token refresh");
+          acceptedRefreshToken = rotated.value;
+        } catch (error) {
+          if (!isInvalidGrant(error)) throw error;
+          lastAuthError = error;
+        }
+      }
+    }
     if (!tokens || !acceptedRefreshToken) throw lastAuthError || new CliError("Stored OAuth login is no longer valid.", EXIT.auth);
     await saveRefreshToken(tokens.refresh_token || acceptedRefreshToken);
     cachedAccessToken = {
@@ -601,6 +639,9 @@ var ApiClient = class {
       }
       delayMs = Math.min(delayMs * 2, maxDelayMs);
     }
+    if (job.result_expired) {
+      throw new CliError(`Job ${job.job_id} results expired after 30 days. Use your saved copy or submit a new execution.`, EXIT.api, job);
+    }
     if (!job.result || typeof job.result !== "object" || Array.isArray(job.result)) {
       throw new CliError(`Job ${job.job_id} completed without a result.`, EXIT.api, job);
     }
@@ -1238,9 +1279,11 @@ async function emitExecution(response, options, command) {
     await emit(summary, jsonMode);
     process.stderr.write(`Saved ${records.length} records to ${options.output}
 `);
+    if (response.status === "failed") process.exitCode = EXIT.api;
     return;
   }
   await emit(response, jsonMode);
+  if (response.status === "failed") process.exitCode = EXIT.api;
 }
 function executionSummary(response) {
   const records = response.records;
@@ -1266,6 +1309,7 @@ function executionSummary(response) {
       "result_count",
       "latency_ms",
       "estimated_cost_usd",
+      "provider_http_status",
       "error"
     ]));
   }
@@ -1303,6 +1347,7 @@ function executionSummary(response) {
           "result_count",
           "latency_ms",
           "estimated_cost_usd",
+          "provider_http_status",
           "error"
         ]));
       }
@@ -1499,7 +1544,7 @@ catalog.command("inspect <capability-id>").action(async (capabilityId, _options,
     globals(command).json || false
   );
 });
-program.command("run [capability-id]").description("Run a capability directly; use Discover only when configuration is uncertain").option("--config <discovery-id>", "run a finalized multi-step configuration").option("-f, --file <path>", "canonical request JSON file").option("--input <json>", "capability input JSON; use url or urls for web.extract_content").option("--provider <id>", "provider ID; repeat for an ordered fallback waterfall", collect, []).option("--provider-options <json>", "provider-native options keyed by provider ID").option("--concurrency <number>", "batch concurrency (default 3, maximum 10)").option("--limit <number>", "maximum records").option("--idempotency-key <key>", "stable retry key").option("--retry-failed <run-id>", "retry only failed URLs from a partial batch run").option("--detach", "queue the job and return immediately").addOption(new Option("--format <format>", "output record format").choices(["json", "jsonl", "csv"]).default("json")).option("-o, --output <path>", "write records to a file").action(async (capabilityId, options, command) => {
+program.command("run [capability-id]").description("Run a capability directly; use Discover only when configuration is uncertain").option("--config <discovery-id>", "run a finalized multi-step configuration").option("-f, --file <path>", "canonical request JSON file").option("--input <json>", "capability input JSON; use items for batches, urls for extraction, queries for search").option("--provider <id>", "provider ID; repeat for an ordered fallback waterfall", collect, []).option("--provider-options <json>", "provider-native options keyed by provider ID").option("--concurrency <number>", "individual-request concurrency; native provider batches use upstream concurrency").option("--limit <number>", "maximum records").option("--idempotency-key <key>", "stable retry key").option("--retry-failed <run-id>", "retry only failed inputs from a partial batch run").option("--detach", "queue the job and return immediately").addOption(new Option("--format <format>", "output record format").choices(["json", "jsonl", "csv"]).default("json")).option("-o, --output <path>", "write records to a file").action(async (capabilityId, options, command) => {
   if (options.retryFailed) {
     if (capabilityId || options.config) {
       throw new CliError("Do not combine --retry-failed with a capability ID or --config.", EXIT.usage);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scrappycoco/cli",
-  "version": "0.8.3",
+  "version": "0.8.5",
   "description": "CLI for Scrappycoco scraper discovery and execution",
   "type": "module",
   "bin": {