@scrappycoco/cli 0.1.0 → 0.2.0

package/README.md

@@ -25,8 +25,11 @@ Node.js 20 or newer is required. Interactive use authenticates with Clerk OAuth
 
 Use `--json` for machine-readable responses. Execution commands support
 `--format json|jsonl|csv` with `--output`, repeatable `--provider` flags, and an
-explicit `--idempotency-key` for safe identical retries.
+explicit `--idempotency-key` for safe identical retries. They submit durable
+jobs and poll for completion; set `SCRAPPYCOCO_JOB_TIMEOUT_MS` to change the
+20-minute local wait. If a command times out while its job continues, inspect
+it with `scrappycoco jobs get <job-id>`.
 
 Use `scrappycoco --help` for the complete command reference. See the
-[Scrappycoco API documentation](https://scrappycoco.readme.io) for the public
+[Scrappycoco API documentation](https://scrappycoco.ai/docs) for the public
 contract.

package/dist/index.js

@@ -202,6 +202,21 @@ async function accessToken(apiUrl) {
 // src/client.ts
 import { randomUUID } from "crypto";
 var DEFAULT_API_URL = process.env.SCRAPPYCOCO_API_URL || "https://api.scrappycoco.ai";
+var DEFAULT_JOB_TIMEOUT_MS = 20 * 60 * 1e3;
+var DEFAULT_JOB_POLL_INITIAL_MS = 250;
+var DEFAULT_JOB_POLL_MAX_MS = 5e3;
+function positiveInteger(value, fallback) {
+  if (!value) return fallback;
+  const parsed = Number(value);
+  return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback;
+}
+function jobTimeoutError(job, timeoutMs) {
+  return new CliError(
+    `Job ${job.job_id} did not finish within ${Math.ceil(timeoutMs / 1e3)} seconds.`,
+    EXIT.timeout,
+    job
+  );
+}
 var ApiClient = class {
   constructor(baseUrl = DEFAULT_API_URL) {
     this.baseUrl = baseUrl;
@@ -218,11 +233,12 @@ var ApiClient = class {
       ...extra
     };
   }
-  async request(method, path, body, headers) {
+  async request(method, path, body, headers, signal) {
     const response = await fetch(`${this.baseUrl.replace(/\/$/, "")}/api/v1${path}`, {
       method,
       headers: await this.headers(headers),
-      body: body === void 0 ? void 0 : JSON.stringify(body)
+      body: body === void 0 ? void 0 : JSON.stringify(body),
+      signal
     });
     if (response.status === 204) return void 0;
     const payload = await response.json().catch(() => ({}));
@@ -233,12 +249,54 @@ var ApiClient = class {
     }
     return payload;
   }
-  get(path) {
-    return this.request("GET", path);
+  get(path, signal) {
+    return this.request("GET", path, void 0, void 0, signal);
   }
   post(path, body, key = randomUUID()) {
     return this.request("POST", path, body, { "Idempotency-Key": key });
   }
+  async postJob(path, body, key = randomUUID()) {
+    const submitted = await this.post(path, body, key);
+    const timeoutMs = positiveInteger(process.env.SCRAPPYCOCO_JOB_TIMEOUT_MS, DEFAULT_JOB_TIMEOUT_MS);
+    const initialDelayMs = positiveInteger(
+      process.env.SCRAPPYCOCO_JOB_POLL_INITIAL_MS,
+      DEFAULT_JOB_POLL_INITIAL_MS
+    );
+    const maxDelayMs = positiveInteger(process.env.SCRAPPYCOCO_JOB_POLL_MAX_MS, DEFAULT_JOB_POLL_MAX_MS);
+    const deadline = Date.now() + timeoutMs;
+    let delayMs = Math.min(initialDelayMs, maxDelayMs);
+    let job = submitted;
+    while (job.status !== "completed") {
+      if (job.status === "failed") {
+        throw new CliError(
+          job.error || `Job ${job.job_id} failed.`,
+          EXIT.api,
+          job
+        );
+      }
+      const remainingBeforeDelay = deadline - Date.now();
+      if (remainingBeforeDelay <= 0) throw jobTimeoutError(job, timeoutMs);
+      await new Promise((resolve) => setTimeout(resolve, Math.min(delayMs, remainingBeforeDelay)));
+      const remainingBeforeRequest = deadline - Date.now();
+      if (remainingBeforeRequest <= 0) throw jobTimeoutError(job, timeoutMs);
+      try {
+        job = await this.get(
+          `/jobs/${encodeURIComponent(job.job_id)}`,
+          AbortSignal.timeout(remainingBeforeRequest)
+        );
+      } catch (error) {
+        if (Date.now() >= deadline || error instanceof DOMException && (error.name === "AbortError" || error.name === "TimeoutError")) {
+          throw jobTimeoutError(job, timeoutMs);
+        }
+        throw error;
+      }
+      delayMs = Math.min(delayMs * 2, maxDelayMs);
+    }
+    if (!job.result || typeof job.result !== "object" || Array.isArray(job.result)) {
+      throw new CliError(`Job ${job.job_id} completed without a result.`, EXIT.api, job);
+    }
+    return job.result;
+  }
   patch(path, body) {
     return this.request("PATCH", path, body);
   }
@@ -397,8 +455,8 @@ function executionCommand(name) {
   return scrapers.command(`${name} <scraper-id>`).description(name === "run" ? "Run one capability through a provider waterfall" : "Compare providers for one capability").option("-f, --file <path>", "canonical request JSON file").option("--input <json>", "capability input JSON").option("--provider <id>", "provider ID; repeat to choose and order providers", collect, []).option("--limit <number>", "maximum records").option("--idempotency-key <key>", "stable retry key").addOption(new Option("--format <format>", "output record format").choices(["json", "jsonl", "csv"]).default("json")).option("-o, --output <path>", "write records to a file").action(async (scraperId, options, command) => {
     const payload = await requestPayload(options, scraperId);
     if (payload.limit === void 0) payload.limit = 10;
-    const response = await client(command).post(
-      `/scrapers/${name === "run" ? "execute" : "compare"}`,
+    const response = await client(command).postJob(
+      name === "run" ? "/scrapers/jobs" : "/scrapers/compare/jobs",
       payload,
       options.idempotencyKey || randomUUID2()
     );
@@ -407,6 +465,13 @@ function executionCommand(name) {
 }
 executionCommand("run");
 executionCommand("compare");
+var jobs = program.command("jobs").description("Inspect durable queued jobs");
+jobs.command("get <job-id>").action(async (jobId, _options, command) => {
+  await emit(
+    await client(command).get(`/jobs/${encodeURIComponent(jobId)}`),
+    globals(command).json || false
+  );
+});
 var providers = program.command("providers").description("Inspect integrated scraper providers");
 providers.command("list").option("--available", "only include available provider-capability routes").action(async (options, command) => {
   await emit(
@@ -448,8 +513,8 @@ discoveries.command("run <discovery-id>").option("-f, --file <path>", "request J
     input: input || {},
     limit: Number(options.limit ?? fromFile.limit ?? 25)
   };
-  const response = await client(command).post(
-    `/discoveries/${encodeURIComponent(discoveryId)}/run`,
+  const response = await client(command).postJob(
+    `/discoveries/${encodeURIComponent(discoveryId)}/jobs`,
     payload,
     options.idempotencyKey || randomUUID2()
   );

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scrappycoco/cli",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "CLI for Scrappycoco scraper discovery, execution, and provider comparison",
   "type": "module",
   "bin": {
@@ -18,6 +18,7 @@
     "build": "tsup src/index.ts --format esm --platform node --target node20 --clean && node scripts/make-executable.mjs",
     "test": "npm run build && vitest run",
     "typecheck": "tsc --noEmit",
+    "release:check": "npm run build && node scripts/check-release-readiness.mjs",
     "prepack": "npm run build"
   },
   "dependencies": {