crw-sdk 0.30.0 → 0.32.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -14,7 +14,7 @@ npm install crw-sdk
14
14
 
15
15
  ## Quick start — Cloud (default)
16
16
 
17
- CRW is **cloud-first**. [Sign up for 500 free credits](https://fastcrw.com/dashboard)
17
+ CRW is **cloud-first**. [Sign up for 1000 free credits](https://fastcrw.com/dashboard)
18
18
  — no payment, no monthly reset (GitHub/Google, ~10s) — then set `CRW_API_KEY`:
19
19
 
20
20
  ```ts
@@ -10,7 +10,7 @@ exports.CLOUD_API_URL = "https://api.fastcrw.com";
10
10
  exports.DASHBOARD_URL = "https://fastcrw.com/dashboard";
11
11
  exports.DOCS_URL = "https://us.github.io/crw";
12
12
  const SIGNUP_NUDGE = `No CRW API key found. CRW uses the managed cloud (${exports.CLOUD_API_URL}) by default.\n` +
13
- ` -> Sign up at ${exports.DASHBOARD_URL} for 500 free credits — no payment, no monthly ` +
13
+ ` -> Sign up at ${exports.DASHBOARD_URL} for 1000 free credits — no payment, no monthly ` +
14
14
  `reset (GitHub/Google, ~10s) — then set CRW_API_KEY (or pass apiKey).\n` +
15
15
  ` -> Prefer to self-host? Set CRW_LOCAL=1 to run the local engine. Docs: ${exports.DOCS_URL}`;
16
16
  function envTruthy(value) {
@@ -21,13 +21,33 @@ function httpOnlyHint(name, reason) {
21
21
  `Use the cloud (set CRW_API_KEY) or pass apiUrl for a self-hosted server.`);
22
22
  }
23
23
  const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
24
+ /**
25
+ * Next page request, rebuilt from OUR base path + the cursor's skip/limit. The
26
+ * server's `next` is an absolute URL from its own public origin/prefix; reusing
27
+ * it verbatim double-prefixes when apiUrl carries a path (`.../api`) and trusts a
28
+ * host that may not match apiUrl. We only borrow the cursor position.
29
+ */
30
+ function pageCursorPath(basePath, nextUrl) {
31
+ let search = "";
32
+ try {
33
+ search = new URL(nextUrl, "http://placeholder.invalid").search;
34
+ }
35
+ catch {
36
+ return basePath;
37
+ }
38
+ const q = new URLSearchParams(search);
39
+ const params = ["skip", "limit"]
40
+ .filter((k) => q.get(k) !== null)
41
+ .map((k) => `${k}=${q.get(k)}`);
42
+ return params.length ? `${basePath}?${params.join("&")}` : basePath;
43
+ }
24
44
  class CrwClient {
25
45
  apiUrl;
26
46
  apiKey;
27
47
  local = null;
28
48
  /**
29
49
  * CRW is cloud-first. With no arguments the client targets the managed cloud
30
- * (api.fastcrw.com) and needs an API key — sign up for 500 free credits at
50
+ * (api.fastcrw.com) and needs an API key — sign up for 1000 free credits at
31
51
  * https://fastcrw.com/dashboard. To self-host locally, set `CRW_LOCAL=1`.
32
52
  */
33
53
  constructor(opts = {}) {
@@ -145,7 +165,7 @@ class CrwClient {
145
165
  }
146
166
  /**
147
167
  * Firecrawl-compatible Research API (cloud only). Mirrors the Firecrawl
148
- * research SDK surface over `/v2/search/research/*`. Each method GETs the
168
+ * research SDK surface over `/v1/search/research/*`. Each method GETs the
149
169
  * hosted endpoint and returns its `{ success, ... }` payload verbatim.
150
170
  */
151
171
  get research() {
@@ -161,10 +181,10 @@ class CrwClient {
161
181
  });
162
182
  };
163
183
  return {
164
- searchPapers: (query, opts = {}) => get("/v2/search/research/papers", { query, ...opts }),
165
- getPaper: (id, opts = {}) => get(`/v2/search/research/papers/${encodeURIComponent(id)}`, { ...opts }),
166
- similarPapers: (id, opts) => get(`/v2/search/research/papers/${encodeURIComponent(id)}/similar`, { ...opts }),
167
- searchGithub: (query, opts = {}) => get("/v2/search/research/github", { query, ...opts }),
184
+ searchPapers: (query, opts = {}) => get("/v1/search/research/papers", { query, ...opts }),
185
+ getPaper: (id, opts = {}) => get(`/v1/search/research/papers/${encodeURIComponent(id)}`, { ...opts }),
186
+ similarPapers: (id, opts) => get(`/v1/search/research/papers/${encodeURIComponent(id)}/similar`, { ...opts }),
187
+ searchGithub: (query, opts = {}) => get("/v1/search/research/github", { query, ...opts }),
168
188
  };
169
189
  }
170
190
  /**
@@ -307,16 +327,42 @@ class CrwClient {
307
327
  if (!jobId)
308
328
  throw new errors_js_1.CrwError(`Batch scrape did not return job ID: ${JSON.stringify(start)}`);
309
329
  const deadline = Date.now() + timeout * 1000;
330
+ const statusPath = `/v2/batch/scrape/${jobId}`;
331
+ // Poll the first page until the job reaches a terminal state.
332
+ let status;
310
333
  for (;;) {
311
334
  if (Date.now() > deadline)
312
335
  throw new errors_js_1.CrwTimeoutError(`Batch scrape ${jobId} timed out after ${timeout}s`);
313
- const status = await this.httpRequest("GET", `/v2/batch/scrape/${jobId}`, undefined, { raw: true, checkSuccess: false });
336
+ status = await this.httpRequest("GET", statusPath, undefined, { raw: true, checkSuccess: false });
314
337
  if (status.status === "completed")
315
- return status.data ?? [];
338
+ break;
316
339
  if (status.status === "failed")
317
340
  throw new errors_js_1.CrwError(`Batch scrape failed: ${status.error ?? "unknown"}`);
341
+ // A cancelled job never completes; stop instead of spinning until timeout.
342
+ if (status.status === "cancelled")
343
+ throw new errors_js_1.CrwError(`Batch scrape ${jobId} was cancelled`);
318
344
  await sleep(pollInterval * 1000);
319
345
  }
346
+ // Page through the full result set. The batch status returns at most ~100
347
+ // documents per page and sets `next` to the cursor for the next page (null on
348
+ // the last). Returning only page 1 silently truncated batches over one page.
349
+ const docs = [...(status.data ?? [])];
350
+ let nextUrl = status.next;
351
+ // Seed with page 1's path: a `next` that rebuilds to it (skip=0, or no cursor
352
+ // at all) is a repeat, not a new page — stop rather than re-append.
353
+ const seen = new Set([statusPath]);
354
+ while (nextUrl) {
355
+ const pagePath = pageCursorPath(statusPath, nextUrl);
356
+ if (seen.has(pagePath))
357
+ break; // repeating/empty cursor; nothing new
358
+ if (Date.now() > deadline)
359
+ throw new errors_js_1.CrwTimeoutError(`Batch scrape ${jobId} timed out after ${timeout}s`);
360
+ seen.add(pagePath);
361
+ const page = await this.httpRequest("GET", pagePath, undefined, { raw: true, checkSuccess: false });
362
+ docs.push(...(page.data ?? []));
363
+ nextUrl = page.next;
364
+ }
365
+ return docs;
320
366
  }
321
367
  /** Feature-detect the engine (HTTP mode only). */
322
368
  async capabilities() {
@@ -360,6 +406,8 @@ class CrwClient {
360
406
  return result.data ?? [];
361
407
  if (result.status === "failed")
362
408
  throw new errors_js_1.CrwError(`Crawl failed: ${result.error ?? "unknown"}`);
409
+ if (result.status === "cancelled")
410
+ throw new errors_js_1.CrwError(`Crawl ${jobId} was cancelled`);
363
411
  await sleep(pollInterval * 1000);
364
412
  }
365
413
  }
@@ -431,6 +479,8 @@ class CrwClient {
431
479
  return status.data ?? [];
432
480
  if (status.status === "failed")
433
481
  throw new errors_js_1.CrwError(`Crawl failed: ${status.error ?? "unknown"}`);
482
+ if (status.status === "cancelled")
483
+ throw new errors_js_1.CrwError(`Crawl ${jobId} was cancelled`);
434
484
  await sleep(pollInterval * 1000);
435
485
  }
436
486
  }
@@ -9,7 +9,7 @@ export declare class CrwClient {
9
9
  private local;
10
10
  /**
11
11
  * CRW is cloud-first. With no arguments the client targets the managed cloud
12
- * (api.fastcrw.com) and needs an API key — sign up for 500 free credits at
12
+ * (api.fastcrw.com) and needs an API key — sign up for 1000 free credits at
13
13
  * https://fastcrw.com/dashboard. To self-host locally, set `CRW_LOCAL=1`.
14
14
  */
15
15
  constructor(opts?: ClientOptions);
@@ -22,7 +22,7 @@ export declare class CrwClient {
22
22
  search(query: string, opts?: SearchOptions): Promise<SearchResult>;
23
23
  /**
24
24
  * Firecrawl-compatible Research API (cloud only). Mirrors the Firecrawl
25
- * research SDK surface over `/v2/search/research/*`. Each method GETs the
25
+ * research SDK surface over `/v1/search/research/*`. Each method GETs the
26
26
  * hosted endpoint and returns its `{ success, ... }` payload verbatim.
27
27
  */
28
28
  get research(): {
@@ -7,7 +7,7 @@ export const CLOUD_API_URL = "https://api.fastcrw.com";
7
7
  export const DASHBOARD_URL = "https://fastcrw.com/dashboard";
8
8
  export const DOCS_URL = "https://us.github.io/crw";
9
9
  const SIGNUP_NUDGE = `No CRW API key found. CRW uses the managed cloud (${CLOUD_API_URL}) by default.\n` +
10
- ` -> Sign up at ${DASHBOARD_URL} for 500 free credits — no payment, no monthly ` +
10
+ ` -> Sign up at ${DASHBOARD_URL} for 1000 free credits — no payment, no monthly ` +
11
11
  `reset (GitHub/Google, ~10s) — then set CRW_API_KEY (or pass apiKey).\n` +
12
12
  ` -> Prefer to self-host? Set CRW_LOCAL=1 to run the local engine. Docs: ${DOCS_URL}`;
13
13
  function envTruthy(value) {
@@ -18,13 +18,33 @@ function httpOnlyHint(name, reason) {
18
18
  `Use the cloud (set CRW_API_KEY) or pass apiUrl for a self-hosted server.`);
19
19
  }
20
20
  const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
21
+ /**
22
+ * Next page request, rebuilt from OUR base path + the cursor's skip/limit. The
23
+ * server's `next` is an absolute URL from its own public origin/prefix; reusing
24
+ * it verbatim double-prefixes when apiUrl carries a path (`.../api`) and trusts a
25
+ * host that may not match apiUrl. We only borrow the cursor position.
26
+ */
27
+ function pageCursorPath(basePath, nextUrl) {
28
+ let search = "";
29
+ try {
30
+ search = new URL(nextUrl, "http://placeholder.invalid").search;
31
+ }
32
+ catch {
33
+ return basePath;
34
+ }
35
+ const q = new URLSearchParams(search);
36
+ const params = ["skip", "limit"]
37
+ .filter((k) => q.get(k) !== null)
38
+ .map((k) => `${k}=${q.get(k)}`);
39
+ return params.length ? `${basePath}?${params.join("&")}` : basePath;
40
+ }
21
41
  export class CrwClient {
22
42
  apiUrl;
23
43
  apiKey;
24
44
  local = null;
25
45
  /**
26
46
  * CRW is cloud-first. With no arguments the client targets the managed cloud
27
- * (api.fastcrw.com) and needs an API key — sign up for 500 free credits at
47
+ * (api.fastcrw.com) and needs an API key — sign up for 1000 free credits at
28
48
  * https://fastcrw.com/dashboard. To self-host locally, set `CRW_LOCAL=1`.
29
49
  */
30
50
  constructor(opts = {}) {
@@ -142,7 +162,7 @@ export class CrwClient {
142
162
  }
143
163
  /**
144
164
  * Firecrawl-compatible Research API (cloud only). Mirrors the Firecrawl
145
- * research SDK surface over `/v2/search/research/*`. Each method GETs the
165
+ * research SDK surface over `/v1/search/research/*`. Each method GETs the
146
166
  * hosted endpoint and returns its `{ success, ... }` payload verbatim.
147
167
  */
148
168
  get research() {
@@ -158,10 +178,10 @@ export class CrwClient {
158
178
  });
159
179
  };
160
180
  return {
161
- searchPapers: (query, opts = {}) => get("/v2/search/research/papers", { query, ...opts }),
162
- getPaper: (id, opts = {}) => get(`/v2/search/research/papers/${encodeURIComponent(id)}`, { ...opts }),
163
- similarPapers: (id, opts) => get(`/v2/search/research/papers/${encodeURIComponent(id)}/similar`, { ...opts }),
164
- searchGithub: (query, opts = {}) => get("/v2/search/research/github", { query, ...opts }),
181
+ searchPapers: (query, opts = {}) => get("/v1/search/research/papers", { query, ...opts }),
182
+ getPaper: (id, opts = {}) => get(`/v1/search/research/papers/${encodeURIComponent(id)}`, { ...opts }),
183
+ similarPapers: (id, opts) => get(`/v1/search/research/papers/${encodeURIComponent(id)}/similar`, { ...opts }),
184
+ searchGithub: (query, opts = {}) => get("/v1/search/research/github", { query, ...opts }),
165
185
  };
166
186
  }
167
187
  /**
@@ -304,16 +324,42 @@ export class CrwClient {
304
324
  if (!jobId)
305
325
  throw new CrwError(`Batch scrape did not return job ID: ${JSON.stringify(start)}`);
306
326
  const deadline = Date.now() + timeout * 1000;
327
+ const statusPath = `/v2/batch/scrape/${jobId}`;
328
+ // Poll the first page until the job reaches a terminal state.
329
+ let status;
307
330
  for (;;) {
308
331
  if (Date.now() > deadline)
309
332
  throw new CrwTimeoutError(`Batch scrape ${jobId} timed out after ${timeout}s`);
310
- const status = await this.httpRequest("GET", `/v2/batch/scrape/${jobId}`, undefined, { raw: true, checkSuccess: false });
333
+ status = await this.httpRequest("GET", statusPath, undefined, { raw: true, checkSuccess: false });
311
334
  if (status.status === "completed")
312
- return status.data ?? [];
335
+ break;
313
336
  if (status.status === "failed")
314
337
  throw new CrwError(`Batch scrape failed: ${status.error ?? "unknown"}`);
338
+ // A cancelled job never completes; stop instead of spinning until timeout.
339
+ if (status.status === "cancelled")
340
+ throw new CrwError(`Batch scrape ${jobId} was cancelled`);
315
341
  await sleep(pollInterval * 1000);
316
342
  }
343
+ // Page through the full result set. The batch status returns at most ~100
344
+ // documents per page and sets `next` to the cursor for the next page (null on
345
+ // the last). Returning only page 1 silently truncated batches over one page.
346
+ const docs = [...(status.data ?? [])];
347
+ let nextUrl = status.next;
348
+ // Seed with page 1's path: a `next` that rebuilds to it (skip=0, or no cursor
349
+ // at all) is a repeat, not a new page — stop rather than re-append.
350
+ const seen = new Set([statusPath]);
351
+ while (nextUrl) {
352
+ const pagePath = pageCursorPath(statusPath, nextUrl);
353
+ if (seen.has(pagePath))
354
+ break; // repeating/empty cursor; nothing new
355
+ if (Date.now() > deadline)
356
+ throw new CrwTimeoutError(`Batch scrape ${jobId} timed out after ${timeout}s`);
357
+ seen.add(pagePath);
358
+ const page = await this.httpRequest("GET", pagePath, undefined, { raw: true, checkSuccess: false });
359
+ docs.push(...(page.data ?? []));
360
+ nextUrl = page.next;
361
+ }
362
+ return docs;
317
363
  }
318
364
  /** Feature-detect the engine (HTTP mode only). */
319
365
  async capabilities() {
@@ -357,6 +403,8 @@ export class CrwClient {
357
403
  return result.data ?? [];
358
404
  if (result.status === "failed")
359
405
  throw new CrwError(`Crawl failed: ${result.error ?? "unknown"}`);
406
+ if (result.status === "cancelled")
407
+ throw new CrwError(`Crawl ${jobId} was cancelled`);
360
408
  await sleep(pollInterval * 1000);
361
409
  }
362
410
  }
@@ -428,6 +476,8 @@ export class CrwClient {
428
476
  return status.data ?? [];
429
477
  if (status.status === "failed")
430
478
  throw new CrwError(`Crawl failed: ${status.error ?? "unknown"}`);
479
+ if (status.status === "cancelled")
480
+ throw new CrwError(`Crawl ${jobId} was cancelled`);
431
481
  await sleep(pollInterval * 1000);
432
482
  }
433
483
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "crw-sdk",
3
- "version": "0.30.0",
3
+ "version": "0.32.0",
4
4
  "description": "TypeScript/JavaScript SDK for CRW — scrape, crawl, map, search, parse, and extract any website",
5
5
  "license": "MIT",
6
6
  "author": "fastCRW <hello@fastcrw.com>",
@@ -44,6 +44,6 @@
44
44
  },
45
45
  "devDependencies": {
46
46
  "@types/node": "^26",
47
- "typescript": "^6"
47
+ "typescript": "^7"
48
48
  }
49
49
  }