crw-sdk 0.31.0 → 0.32.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -21,6 +21,26 @@ function httpOnlyHint(name, reason) {
21
21
  `Use the cloud (set CRW_API_KEY) or pass apiUrl for a self-hosted server.`);
22
22
  }
23
23
  const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
24
+ /**
25
+ * Next page request, rebuilt from OUR base path + the cursor's skip/limit. The
26
+ * server's `next` is an absolute URL from its own public origin/prefix; reusing
27
+ * it verbatim double-prefixes when apiUrl carries a path (`.../api`) and trusts a
28
+ * host that may not match apiUrl. We only borrow the cursor position.
29
+ */
30
+ function pageCursorPath(basePath, nextUrl) {
31
+ let search = "";
32
+ try {
33
+ search = new URL(nextUrl, "http://placeholder.invalid").search;
34
+ }
35
+ catch {
36
+ return basePath;
37
+ }
38
+ const q = new URLSearchParams(search);
39
+ const params = ["skip", "limit"]
40
+ .filter((k) => q.get(k) !== null)
41
+ .map((k) => `${k}=${q.get(k)}`);
42
+ return params.length ? `${basePath}?${params.join("&")}` : basePath;
43
+ }
24
44
  class CrwClient {
25
45
  apiUrl;
26
46
  apiKey;
@@ -307,16 +327,42 @@ class CrwClient {
307
327
  if (!jobId)
308
328
  throw new errors_js_1.CrwError(`Batch scrape did not return job ID: ${JSON.stringify(start)}`);
309
329
  const deadline = Date.now() + timeout * 1000;
330
+ const statusPath = `/v2/batch/scrape/${jobId}`;
331
+ // Poll the first page until the job reaches a terminal state.
332
+ let status;
310
333
  for (;;) {
311
334
  if (Date.now() > deadline)
312
335
  throw new errors_js_1.CrwTimeoutError(`Batch scrape ${jobId} timed out after ${timeout}s`);
313
- const status = await this.httpRequest("GET", `/v2/batch/scrape/${jobId}`, undefined, { raw: true, checkSuccess: false });
336
+ status = await this.httpRequest("GET", statusPath, undefined, { raw: true, checkSuccess: false });
314
337
  if (status.status === "completed")
315
- return status.data ?? [];
338
+ break;
316
339
  if (status.status === "failed")
317
340
  throw new errors_js_1.CrwError(`Batch scrape failed: ${status.error ?? "unknown"}`);
341
+ // A cancelled job never completes; stop instead of spinning until timeout.
342
+ if (status.status === "cancelled")
343
+ throw new errors_js_1.CrwError(`Batch scrape ${jobId} was cancelled`);
318
344
  await sleep(pollInterval * 1000);
319
345
  }
346
+ // Page through the full result set. The batch status returns at most ~100
347
+ // documents per page and sets `next` to the cursor for the next page (null on
348
+ // the last). Returning only page 1 silently truncated batches over one page.
349
+ const docs = [...(status.data ?? [])];
350
+ let nextUrl = status.next;
351
+ // Seed with page 1's path: a `next` that rebuilds to it (skip=0, or no cursor
352
+ // at all) is a repeat, not a new page — stop rather than re-append.
353
+ const seen = new Set([statusPath]);
354
+ while (nextUrl) {
355
+ const pagePath = pageCursorPath(statusPath, nextUrl);
356
+ if (seen.has(pagePath))
357
+ break; // repeating/empty cursor; nothing new
358
+ if (Date.now() > deadline)
359
+ throw new errors_js_1.CrwTimeoutError(`Batch scrape ${jobId} timed out after ${timeout}s`);
360
+ seen.add(pagePath);
361
+ const page = await this.httpRequest("GET", pagePath, undefined, { raw: true, checkSuccess: false });
362
+ docs.push(...(page.data ?? []));
363
+ nextUrl = page.next;
364
+ }
365
+ return docs;
320
366
  }
321
367
  /** Feature-detect the engine (HTTP mode only). */
322
368
  async capabilities() {
@@ -360,6 +406,8 @@ class CrwClient {
360
406
  return result.data ?? [];
361
407
  if (result.status === "failed")
362
408
  throw new errors_js_1.CrwError(`Crawl failed: ${result.error ?? "unknown"}`);
409
+ if (result.status === "cancelled")
410
+ throw new errors_js_1.CrwError(`Crawl ${jobId} was cancelled`);
363
411
  await sleep(pollInterval * 1000);
364
412
  }
365
413
  }
@@ -431,6 +479,8 @@ class CrwClient {
431
479
  return status.data ?? [];
432
480
  if (status.status === "failed")
433
481
  throw new errors_js_1.CrwError(`Crawl failed: ${status.error ?? "unknown"}`);
482
+ if (status.status === "cancelled")
483
+ throw new errors_js_1.CrwError(`Crawl ${jobId} was cancelled`);
434
484
  await sleep(pollInterval * 1000);
435
485
  }
436
486
  }
@@ -18,6 +18,26 @@ function httpOnlyHint(name, reason) {
18
18
  `Use the cloud (set CRW_API_KEY) or pass apiUrl for a self-hosted server.`);
19
19
  }
20
20
  const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
21
+ /**
22
+ * Next page request, rebuilt from OUR base path + the cursor's skip/limit. The
23
+ * server's `next` is an absolute URL from its own public origin/prefix; reusing
24
+ * it verbatim double-prefixes when apiUrl carries a path (`.../api`) and trusts a
25
+ * host that may not match apiUrl. We only borrow the cursor position.
26
+ */
27
+ function pageCursorPath(basePath, nextUrl) {
28
+ let search = "";
29
+ try {
30
+ search = new URL(nextUrl, "http://placeholder.invalid").search;
31
+ }
32
+ catch {
33
+ return basePath;
34
+ }
35
+ const q = new URLSearchParams(search);
36
+ const params = ["skip", "limit"]
37
+ .filter((k) => q.get(k) !== null)
38
+ .map((k) => `${k}=${q.get(k)}`);
39
+ return params.length ? `${basePath}?${params.join("&")}` : basePath;
40
+ }
21
41
  export class CrwClient {
22
42
  apiUrl;
23
43
  apiKey;
@@ -304,16 +324,42 @@ export class CrwClient {
304
324
  if (!jobId)
305
325
  throw new CrwError(`Batch scrape did not return job ID: ${JSON.stringify(start)}`);
306
326
  const deadline = Date.now() + timeout * 1000;
327
+ const statusPath = `/v2/batch/scrape/${jobId}`;
328
+ // Poll the first page until the job reaches a terminal state.
329
+ let status;
307
330
  for (;;) {
308
331
  if (Date.now() > deadline)
309
332
  throw new CrwTimeoutError(`Batch scrape ${jobId} timed out after ${timeout}s`);
310
- const status = await this.httpRequest("GET", `/v2/batch/scrape/${jobId}`, undefined, { raw: true, checkSuccess: false });
333
+ status = await this.httpRequest("GET", statusPath, undefined, { raw: true, checkSuccess: false });
311
334
  if (status.status === "completed")
312
- return status.data ?? [];
335
+ break;
313
336
  if (status.status === "failed")
314
337
  throw new CrwError(`Batch scrape failed: ${status.error ?? "unknown"}`);
338
+ // A cancelled job never completes; stop instead of spinning until timeout.
339
+ if (status.status === "cancelled")
340
+ throw new CrwError(`Batch scrape ${jobId} was cancelled`);
315
341
  await sleep(pollInterval * 1000);
316
342
  }
343
+ // Page through the full result set. The batch status returns at most ~100
344
+ // documents per page and sets `next` to the cursor for the next page (null on
345
+ // the last). Returning only page 1 silently truncated batches over one page.
346
+ const docs = [...(status.data ?? [])];
347
+ let nextUrl = status.next;
348
+ // Seed with page 1's path: a `next` that rebuilds to it (skip=0, or no cursor
349
+ // at all) is a repeat, not a new page — stop rather than re-append.
350
+ const seen = new Set([statusPath]);
351
+ while (nextUrl) {
352
+ const pagePath = pageCursorPath(statusPath, nextUrl);
353
+ if (seen.has(pagePath))
354
+ break; // repeating/empty cursor; nothing new
355
+ if (Date.now() > deadline)
356
+ throw new CrwTimeoutError(`Batch scrape ${jobId} timed out after ${timeout}s`);
357
+ seen.add(pagePath);
358
+ const page = await this.httpRequest("GET", pagePath, undefined, { raw: true, checkSuccess: false });
359
+ docs.push(...(page.data ?? []));
360
+ nextUrl = page.next;
361
+ }
362
+ return docs;
317
363
  }
318
364
  /** Feature-detect the engine (HTTP mode only). */
319
365
  async capabilities() {
@@ -357,6 +403,8 @@ export class CrwClient {
357
403
  return result.data ?? [];
358
404
  if (result.status === "failed")
359
405
  throw new CrwError(`Crawl failed: ${result.error ?? "unknown"}`);
406
+ if (result.status === "cancelled")
407
+ throw new CrwError(`Crawl ${jobId} was cancelled`);
360
408
  await sleep(pollInterval * 1000);
361
409
  }
362
410
  }
@@ -428,6 +476,8 @@ export class CrwClient {
428
476
  return status.data ?? [];
429
477
  if (status.status === "failed")
430
478
  throw new CrwError(`Crawl failed: ${status.error ?? "unknown"}`);
479
+ if (status.status === "cancelled")
480
+ throw new CrwError(`Crawl ${jobId} was cancelled`);
431
481
  await sleep(pollInterval * 1000);
432
482
  }
433
483
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "crw-sdk",
3
- "version": "0.31.0",
3
+ "version": "0.32.0",
4
4
  "description": "TypeScript/JavaScript SDK for CRW — scrape, crawl, map, search, parse, and extract any website",
5
5
  "license": "MIT",
6
6
  "author": "fastCRW <hello@fastcrw.com>",