@scrappycoco/cli 0.8.2 → 0.8.4

package/README.md

@@ -17,10 +17,15 @@ the live catalog, and tells you when to reload or restart your agent. Browser
 authorization is not complete until the terminal confirms that the credential
 was saved and the catalog check succeeded.
 
-Ordinary commands never mutate installed agent instructions. Setup installs
-the matching skill release. Run `scrappycoco skill status` to inspect the
-active target and `scrappycoco skill update` to explicitly install a newer
-hash-verified release, then reload or restart the agent.
+Setup installs the matching skill release. On the first ordinary CLI command
+after the 24-hour cooldown, Scrappycoco compares the installed release with the
+hash-verified public feed and installs a changed skill automatically. The data
+command continues when the check or installer is unavailable. Reload or restart
+the agent after an update notice so it reads the new instructions. Run
+`scrappycoco skill status` to inspect the active target and update state, or
+`scrappycoco skill update` to check immediately. Set
+`SCRAPPYCOCO_SKILL_AUTO_UPDATE=0` to keep the installed copy pinned and use only
+explicit updates.
 
 When the CLI runs on a remote or headless host whose `127.0.0.1` is not the
 browser's localhost, keep the command running and use the manual callback
@@ -61,6 +66,9 @@ agent, general workflow, or scheduler commands. For repeated checks, callers
 schedule ordinary Runs and pass each returned cursor into the next Run.
 
 Node.js 20 or newer is required. Interactive use authenticates with Clerk OAuth Authorization Code + PKCE. CI can set `SCRAPPYCOCO_API_KEY`.
+API keys have no refresh-token rotation step and are preferred for unattended
+concurrent application workloads. OAuth refresh is serialized across CLI
+processes and recovers if another process rotates the stored token first.
 
 OAuth and API requests have bounded timeouts. Set
 `SCRAPPYCOCO_AUTH_TIMEOUT_MS` or `SCRAPPYCOCO_API_TIMEOUT_MS` only when a slow

package/dist/index.js

@@ -60,17 +60,23 @@ function wait(milliseconds) {
 }
 async function withCredentialRefreshLock(operation) {
   const path = credentialRefreshLockPath();
+  const owner = randomUUID();
   await mkdir(dirname(path), { recursive: true, mode: 448 });
   const deadline = Date.now() + LOCK_TIMEOUT_MS;
   while (true) {
     try {
       const handle = await open(path, "wx", 384);
       try {
-        await handle.writeFile(JSON.stringify({ pid: process.pid, created_at: Date.now() }));
+        await handle.writeFile(JSON.stringify({ owner, pid: process.pid, created_at: Date.now() }));
         return await operation();
       } finally {
         await handle.close();
-        await rm(path, { force: true });
+        try {
+          const current = JSON.parse(await readFile(path, "utf8"));
+          if (current.owner === owner) await rm(path, { force: true });
+        } catch (error) {
+          if (error.code !== "ENOENT") throw error;
+        }
       }
     } catch (error) {
       const code = error.code;
@@ -199,6 +205,8 @@ async function clearRefreshToken() {
 var cachedAccessToken;
 var refreshesInFlight = /* @__PURE__ */ new Map();
 var DEFAULT_AUTH_HTTP_TIMEOUT_MS = 15e3;
+var ROTATED_TOKEN_RELOAD_ATTEMPTS = 10;
+var ROTATED_TOKEN_RELOAD_DELAY_MS = 100;
 function positiveInteger(value, fallback) {
   if (!value) return fallback;
   const parsed = Number(value);
@@ -213,6 +221,12 @@ function networkMessage(error) {
   }
   return "failed to reach the authentication service";
 }
+function isInvalidGrant(error) {
+  return error instanceof CliError && error.exitCode === EXIT.auth && typeof error.details === "object" && error.details !== null && "oauth_error" in error.details && error.details.oauth_error === "invalid_grant";
+}
+function wait2(milliseconds) {
+  return new Promise((resolve) => setTimeout(resolve, milliseconds));
+}
 async function oauthFetch(url, init, phase, exitCode) {
   try {
     return await fetch(url, {
@@ -436,7 +450,9 @@ async function accessToken(apiUrl) {
     let tokens;
     let acceptedRefreshToken;
     let lastAuthError;
+    const attemptedRefreshTokens = /* @__PURE__ */ new Set();
     for (const candidate of candidates) {
+      attemptedRefreshTokens.add(candidate.value);
       try {
         tokens = await tokenRequest(config.issuer, new URLSearchParams({
           grant_type: "refresh_token",
@@ -452,6 +468,28 @@ async function accessToken(apiUrl) {
         lastAuthError = error;
       }
     }
+    if (!tokens && isInvalidGrant(lastAuthError)) {
+      for (let attempt = 0; attempt < ROTATED_TOKEN_RELOAD_ATTEMPTS && !tokens; attempt += 1) {
+        await wait2(ROTATED_TOKEN_RELOAD_DELAY_MS);
+        const rotatedCandidates = await loadRefreshTokenCandidates();
+        const rotated = rotatedCandidates.find(
+          (candidate) => !attemptedRefreshTokens.has(candidate.value)
+        );
+        if (!rotated) continue;
+        attemptedRefreshTokens.add(rotated.value);
+        try {
+          tokens = await tokenRequest(config.issuer, new URLSearchParams({
+            grant_type: "refresh_token",
+            client_id: config.client_id,
+            refresh_token: rotated.value
+          }), "token refresh");
+          acceptedRefreshToken = rotated.value;
+        } catch (error) {
+          if (!isInvalidGrant(error)) throw error;
+          lastAuthError = error;
+        }
+      }
+    }
     if (!tokens || !acceptedRefreshToken) throw lastAuthError || new CliError("Stored OAuth login is no longer valid.", EXIT.auth);
     await saveRefreshToken(tokens.refresh_token || acceptedRefreshToken);
     cachedAccessToken = {
@@ -861,7 +899,10 @@ var SKILL_NAME = "scrappycoco";
 var SKILL_INDEX_URL = `${SKILL_SOURCE}/.well-known/agent-skills/index.json`;
 var SKILL_UPDATE_TIMEOUT_MS = 3e3;
 var SKILL_UPDATE_LOCK_STALE_MS = 5 * 6e4;
+var SKILL_AUTO_UPDATE_INTERVAL_MS = 24 * 60 * 6e4;
 var SHA256_DIGEST = /^sha256:[a-f0-9]{64}$/i;
+var DISABLED_AUTO_UPDATE_VALUES = /* @__PURE__ */ new Set(["0", "false", "no", "off"]);
+var ENABLED_AUTO_UPDATE_VALUES = /* @__PURE__ */ new Set(["1", "true", "yes", "on"]);
 function skillReleaseStatePath() {
   return join2(dirname2(fallbackCredentialPath()), "skill-release.json");
 }
@@ -919,9 +960,21 @@ async function loadInstalledDigest() {
   const entry = (await loadSkillReleaseState()).targets[target];
   return entry?.digest || null;
 }
+async function loadLastCheckedAt() {
+  const target = await activeSkillTarget();
+  const entry = (await loadSkillReleaseState()).targets[target];
+  return entry?.checked_at || entry?.updated_at || null;
+}
+function skillAutoUpdateEnabled(environment = process.env) {
+  const configured = environment.SCRAPPYCOCO_SKILL_AUTO_UPDATE?.trim().toLowerCase();
+  if (configured && ENABLED_AUTO_UPDATE_VALUES.has(configured)) return true;
+  if (configured && DISABLED_AUTO_UPDATE_VALUES.has(configured)) return false;
+  return environment.NODE_ENV !== "test" && !environment.VITEST && !ENABLED_AUTO_UPDATE_VALUES.has(environment.CI?.trim().toLowerCase() || "");
+}
 async function installedSkillDiagnostics() {
   const target = await activeSkillTarget();
   const state = await loadSkillReleaseState();
+  const activeRelease = state.targets[target];
   const installedTargets = (await Promise.all(
     installedSkillCandidates().map(async (path) => ({
       path,
@@ -930,8 +983,13 @@ async function installedSkillDiagnostics() {
   )).filter((item) => item.installed);
   return {
     installed: installedTargets.some((item) => item.path === target),
-    digest: state.targets[target]?.digest || null,
+    digest: activeRelease?.digest || null,
     target,
+    auto_update: {
+      enabled: skillAutoUpdateEnabled(),
+      interval_hours: SKILL_AUTO_UPDATE_INTERVAL_MS / (60 * 6e4),
+      last_checked_at: activeRelease?.checked_at || activeRelease?.updated_at || null
+    },
     targets: installedTargets.map(({ path }) => ({
       path,
       digest: state.targets[path]?.digest || null
@@ -964,11 +1022,38 @@ function recordTargetDigest(state, target, digest, updatedAt = (/* @__PURE__ */
       ...state.targets,
       [target]: {
         digest,
-        updated_at: updatedAt
+        updated_at: updatedAt,
+        checked_at: updatedAt
+      }
+    }
+  };
+}
+function recordTargetCheck(state, target, checkedAt = (/* @__PURE__ */ new Date()).toISOString()) {
+  const release = state.targets[target];
+  if (!release) return state;
+  return {
+    targets: {
+      ...state.targets,
+      [target]: {
+        ...release,
+        checked_at: checkedAt
       }
     }
   };
 }
+async function saveLastCheckedAt(checkedAt) {
+  const target = await activeSkillTarget();
+  const state = await loadSkillReleaseState();
+  await writePrivateJson2(
+    skillReleaseStatePath(),
+    recordTargetCheck(state, target, checkedAt)
+  );
+}
+function automaticSkillUpdateDue(lastCheckedAt, now = /* @__PURE__ */ new Date(), intervalMs = SKILL_AUTO_UPDATE_INTERVAL_MS) {
+  if (!lastCheckedAt) return true;
+  const elapsed = now.getTime() - Date.parse(lastCheckedAt);
+  return !Number.isFinite(elapsed) || elapsed < 0 || elapsed >= intervalMs;
+}
 async function fetchPublishedSkillDigest(fetcher = fetch) {
   const response = await fetcher(SKILL_INDEX_URL, {
     headers: { accept: "application/json" },
@@ -1071,8 +1156,46 @@ async function updateInstalledSkill() {
 `
     );
   }
+  if (result.status === "current" || result.status === "updated" && result.state_saved) {
+    try {
+      await saveLastCheckedAt((/* @__PURE__ */ new Date()).toISOString());
+    } catch {
+    }
+  }
   return result;
 }
+function defaultAutomaticDependencies() {
+  return {
+    enabled: skillAutoUpdateEnabled,
+    now: () => /* @__PURE__ */ new Date(),
+    loadLastCheckedAt,
+    update: updateInstalledSkill,
+    saveLastCheckedAt
+  };
+}
+async function maybeUpdateInstalledSkill(dependencies = defaultAutomaticDependencies(), intervalMs = SKILL_AUTO_UPDATE_INTERVAL_MS) {
+  if (!dependencies.enabled()) return { status: "disabled" };
+  try {
+    const now = dependencies.now();
+    if (!automaticSkillUpdateDue(
+      await dependencies.loadLastCheckedAt(),
+      now,
+      intervalMs
+    )) {
+      return { status: "throttled" };
+    }
+    const result = await dependencies.update();
+    if (result.status === "current" || result.status === "updated") {
+      try {
+        await dependencies.saveLastCheckedAt(now.toISOString());
+      } catch {
+      }
+    }
+    return result;
+  } catch {
+    return { status: "unavailable" };
+  }
+}
 async function rememberInstalledSkillRelease() {
   try {
     await saveInstalledDigest(await fetchPublishedSkillDigest());
@@ -1140,7 +1263,7 @@ async function requestPayload(options, scraperId) {
   };
 }
 async function emitExecution(response, options, command) {
-  const records = "records" in response && Array.isArray(response.records) ? response.records : response.items;
+  const records = response.records;
   const jsonMode = globals(command).json || false;
   if (options.output) {
     await emit(response, jsonMode, options.output, formatRecords(records, options.format));
@@ -1153,15 +1276,16 @@ async function emitExecution(response, options, command) {
     await emit(summary, jsonMode);
     process.stderr.write(`Saved ${records.length} records to ${options.output}
 `);
+    if (response.status === "failed") process.exitCode = EXIT.api;
     return;
   }
   await emit(response, jsonMode);
+  if (response.status === "failed") process.exitCode = EXIT.api;
 }
 function executionSummary(response) {
-  const records = "records" in response && Array.isArray(response.records) ? response.records : response.items;
+  const records = response.records;
   const summary = { ...response };
   delete summary.records;
-  delete summary.items;
   delete summary.normalized_schema;
   summary.truncated_count = records.filter((record) => {
     const metadata = record.metadata;
@@ -1182,6 +1306,7 @@ function executionSummary(response) {
       "result_count",
       "latency_ms",
       "estimated_cost_usd",
+      "provider_http_status",
       "error"
     ]));
   }
@@ -1196,7 +1321,7 @@ function executionSummary(response) {
     if (Object.keys(counts).length) summary.provider_result_counts = counts;
     delete summary.provider_results;
   }
-  if (Array.isArray(response.items)) summary.item_count = response.items.length;
+  summary.item_count = records.length;
   if (Array.isArray(summary.routes)) {
     summary.routes = summary.routes.map((value) => {
       if (!value || typeof value !== "object" || Array.isArray(value)) return value;
@@ -1219,6 +1344,7 @@ function executionSummary(response) {
           "result_count",
           "latency_ms",
           "estimated_cost_usd",
+          "provider_http_status",
           "error"
         ]));
       }
@@ -1340,7 +1466,7 @@ auth.command("logout").action(async (_options, command) => {
   await clearRefreshToken2();
   await emit({ authenticated: false }, globals(command).json || false);
 });
-var skill = program.command("skill").description("Inspect or explicitly update the installed skill");
+var skill = program.command("skill").description("Inspect or update the installed skill");
 skill.command("status").action(async (_options, command) => {
   await emit(await installedSkillDiagnostics(), globals(command).json || false);
 });
@@ -1396,24 +1522,6 @@ program.command("doctor").description("Check the CLI, authentication, installed
   );
   if (!ok) process.exitCode = authentication.configured ? EXIT.api : EXIT.auth;
 });
-var scrapers = program.command("scrapers", { hidden: true }).description("Legacy scraper commands");
-scrapers.command("list").option("--source <source>", "filter by web, x, reddit, instagram, tiktok, or filings").option("--provider <provider>", "filter by provider implementation").option("--available", "only include available providers").action(async (options, command) => {
-  const query = new URLSearchParams();
-  if (options.source) query.set("source", options.source);
-  if (options.provider) query.set("provider", options.provider);
-  if (options.available) query.set("available_only", "true");
-  const suffix = query.size ? `?${query}` : "";
-  await emit(await client(command).get(`/scrapers${suffix}`), globals(command).json || false);
-});
-scrapers.command("inspect <scraper-id>").action(async (scraperId, _options, command) => {
-  const { source, capability } = splitScraperId(scraperId);
-  await emit(
-    await client(command).get(
-      `/scrapers/${encodeURIComponent(source)}/${encodeURIComponent(capability)}`
-    ),
-    globals(command).json || false
-  );
-});
 var catalog = program.command("catalog").description("Inspect capabilities, provider schemas, options, and pricing");
 catalog.command("list").option("--source <source>", "filter by web, x, reddit, instagram, tiktok, or filings").option("--provider <provider>", "filter by provider implementation").option("--available", "only include configured providers; this is not a live health check").option("--full", "include input/output and provider option schemas in JSON output").action(async (options, command) => {
   const query = new URLSearchParams();
@@ -1433,20 +1541,6 @@ catalog.command("inspect <capability-id>").action(async (capabilityId, _options,
     globals(command).json || false
   );
 });
-function executionCommand(name) {
-  return scrapers.command(`${name} <scraper-id>`).description(name === "run" ? "Run one capability through a provider waterfall" : "Compare providers for one capability").option("-f, --file <path>", "canonical request JSON file").option("--input <json>", "capability input JSON").option("--provider <id>", "provider ID; repeat to choose and order providers", collect, []).option("--provider-options <json>", "provider-native options keyed by provider ID").option("--concurrency <number>", "batch concurrency (default 3, maximum 10)").option("--limit <number>", "maximum records").option("--idempotency-key <key>", "stable retry key").addOption(new Option("--format <format>", "output record format").choices(["json", "jsonl", "csv"]).default("json")).option("-o, --output <path>", "write records to a file").action(async (scraperId, options, command) => {
-    const payload = await requestPayload(options, scraperId);
-    if (payload.limit === void 0) payload.limit = 10;
-    const response = await client(command).postJob(
-      name === "run" ? "/scrapers/jobs" : "/scrapers/compare/jobs",
-      payload,
-      options.idempotencyKey || randomUUID4()
-    );
-    await emitExecution(response, options, command);
-  });
-}
-executionCommand("run");
-executionCommand("compare");
 program.command("run [capability-id]").description("Run a capability directly; use Discover only when configuration is uncertain").option("--config <discovery-id>", "run a finalized multi-step configuration").option("-f, --file <path>", "canonical request JSON file").option("--input <json>", "capability input JSON; use url or urls for web.extract_content").option("--provider <id>", "provider ID; repeat for an ordered fallback waterfall", collect, []).option("--provider-options <json>", "provider-native options keyed by provider ID").option("--concurrency <number>", "batch concurrency (default 3, maximum 10)").option("--limit <number>", "maximum records").option("--idempotency-key <key>", "stable retry key").option("--retry-failed <run-id>", "retry only failed URLs from a partial batch run").option("--detach", "queue the job and return immediately").addOption(new Option("--format <format>", "output record format").choices(["json", "jsonl", "csv"]).default("json")).option("-o, --output <path>", "write records to a file").action(async (capabilityId, options, command) => {
   if (options.retryFailed) {
     if (capabilityId || options.config) {
@@ -1513,58 +1607,6 @@ jobs.command("cancel <job-id>").description("Cancel a pending or running queued
     globals(command).json || false
   );
 });
-var providers = program.command("providers", { hidden: true }).description("Legacy provider commands");
-providers.command("list").option("--available", "only include available provider-capability routes").action(async (options, command) => {
-  await emit(
-    await client(command).get(`/providers${options.available ? "?available_only=true" : ""}`),
-    globals(command).json || false
-  );
-});
-var discoveries = program.command("discoveries", { hidden: true }).description("Legacy discovery commands");
-discoveries.command("create").requiredOption("-f, --file <path>", "agent-authored discovery JSON with goal and configuration").action(async (options, command) => {
-  await emit(
-    await client(command).post("/discoveries", await readJsonFile(options.file)),
-    globals(command).json || false
-  );
-});
-discoveries.command("list").action(async (_options, command) => {
-  await emit(await client(command).get("/discoveries"), globals(command).json || false);
-});
-discoveries.command("get <discovery-id>").action(async (discoveryId, _options, command) => {
-  await emit(
-    await client(command).get(`/discoveries/${encodeURIComponent(discoveryId)}`),
-    globals(command).json || false
-  );
-});
-discoveries.command("update <discovery-id>").option("-f, --file <path>", "update JSON containing name, priority, or configuration").option("--name <name>", "saved discovery name").option("--priority <priority>", "balanced, quality, coverage, cost, or speed").action(async (discoveryId, options, command) => {
-  const payload = options.file ? await readJsonFile(options.file) : {};
-  if (options.name) payload.name = options.name;
-  if (options.priority) payload.priority = options.priority;
-  if (!Object.keys(payload).length) throw new CliError("Provide --file, --name, or --priority.", EXIT.usage);
-  await emit(
-    await client(command).patch(`/discoveries/${encodeURIComponent(discoveryId)}`, payload),
-    globals(command).json || false
-  );
-});
-discoveries.command("run <discovery-id>").option("-f, --file <path>", "request JSON containing input and optional limit").option("--input <json>", "runtime parameter JSON").option("--limit <number>", "maximum records per route").option("--idempotency-key <key>", "stable retry key").addOption(new Option("--format <format>", "output record format").choices(["json", "jsonl", "csv"]).default("json")).option("-o, --output <path>", "write records to a file").action(async (discoveryId, options, command) => {
-  const fromFile = options.file ? await readJsonFile(options.file) : {};
-  const input = options.input ? parseJsonObject(options.input, "input JSON") : fromFile.input;
-  const payload = {
-    ...fromFile,
-    input: input || {},
-    limit: Number(options.limit ?? fromFile.limit ?? 25)
-  };
-  const response = await client(command).postJob(
-    `/discoveries/${encodeURIComponent(discoveryId)}/jobs`,
-    payload,
-    options.idempotencyKey || randomUUID4()
-  );
-  await emitExecution(response, options, command);
-});
-discoveries.command("delete <discovery-id>").requiredOption("--yes", "confirm permanent deletion").action(async (discoveryId, _options, command) => {
-  await client(command).delete(`/discoveries/${encodeURIComponent(discoveryId)}`);
-  await emit({ deleted: true, discovery_id: discoveryId }, globals(command).json || false);
-});
 program.command("discover").description("Save, sample-test, or finalize an agent-authored configuration").option("-f, --file <path>", "create from agent-authored discovery JSON").option("--id <discovery-id>", "existing discovery ID").option("--test", "run a representative provider sample test").option("--input <json>", "sample runtime input JSON").option("--update <path>", "replace fields or configuration from agent-authored JSON").option("--finalize", "mark the current explicit configuration finalized").option("--idempotency-key <key>", "stable sample retry key").option("-o, --output <path>", "write complete sample-test evidence to a JSON file").action(async (options, command) => {
   const selected = Number(Boolean(options.file)) + Number(Boolean(options.test)) + Number(Boolean(options.update)) + Number(Boolean(options.finalize));
   if (selected !== 1) {
@@ -1632,6 +1674,11 @@ program.command("discover").description("Save, sample-test, or finalize an agent
   await emit(response, globals(command).json || false);
 });
 program.configureOutput({ writeErr: (text) => process.stderr.write(text) });
+program.hook("preAction", async (_command, actionCommand) => {
+  const parentName = actionCommand.parent?.name();
+  if (actionCommand.name() === "setup" || parentName === "skill") return;
+  await maybeUpdateInstalledSkill();
+});
 program.parseAsync(process.argv).catch(async (error) => {
   if (error instanceof CommanderError) {
     process.exitCode = error.exitCode === 0 ? EXIT.ok : EXIT.usage;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scrappycoco/cli",
-  "version": "0.8.2",
+  "version": "0.8.4",
   "description": "CLI for Scrappycoco scraper discovery and execution",
   "type": "module",
   "bin": {