crw-sdk 0.31.0 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/client.js +52 -2
- package/dist/esm/client.js +52 -2
- package/package.json +1 -1
package/dist/cjs/client.js
CHANGED
|
@@ -21,6 +21,26 @@ function httpOnlyHint(name, reason) {
|
|
|
21
21
|
`Use the cloud (set CRW_API_KEY) or pass apiUrl for a self-hosted server.`);
|
|
22
22
|
}
|
|
23
23
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
24
|
+
/**
|
|
25
|
+
* Next page request, rebuilt from OUR base path + the cursor's skip/limit. The
|
|
26
|
+
* server's `next` is an absolute URL from its own public origin/prefix; reusing
|
|
27
|
+
* it verbatim double-prefixes when apiUrl carries a path (`.../api`) and trusts a
|
|
28
|
+
* host that may not match apiUrl. We only borrow the cursor position.
|
|
29
|
+
*/
|
|
30
|
+
function pageCursorPath(basePath, nextUrl) {
|
|
31
|
+
let search = "";
|
|
32
|
+
try {
|
|
33
|
+
search = new URL(nextUrl, "http://placeholder.invalid").search;
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
return basePath;
|
|
37
|
+
}
|
|
38
|
+
const q = new URLSearchParams(search);
|
|
39
|
+
const params = ["skip", "limit"]
|
|
40
|
+
.filter((k) => q.get(k) !== null)
|
|
41
|
+
.map((k) => `${k}=${q.get(k)}`);
|
|
42
|
+
return params.length ? `${basePath}?${params.join("&")}` : basePath;
|
|
43
|
+
}
|
|
24
44
|
class CrwClient {
|
|
25
45
|
apiUrl;
|
|
26
46
|
apiKey;
|
|
@@ -307,16 +327,42 @@ class CrwClient {
|
|
|
307
327
|
if (!jobId)
|
|
308
328
|
throw new errors_js_1.CrwError(`Batch scrape did not return job ID: ${JSON.stringify(start)}`);
|
|
309
329
|
const deadline = Date.now() + timeout * 1000;
|
|
330
|
+
const statusPath = `/v2/batch/scrape/${jobId}`;
|
|
331
|
+
// Poll the first page until the job reaches a terminal state.
|
|
332
|
+
let status;
|
|
310
333
|
for (;;) {
|
|
311
334
|
if (Date.now() > deadline)
|
|
312
335
|
throw new errors_js_1.CrwTimeoutError(`Batch scrape ${jobId} timed out after ${timeout}s`);
|
|
313
|
-
|
|
336
|
+
status = await this.httpRequest("GET", statusPath, undefined, { raw: true, checkSuccess: false });
|
|
314
337
|
if (status.status === "completed")
|
|
315
|
-
|
|
338
|
+
break;
|
|
316
339
|
if (status.status === "failed")
|
|
317
340
|
throw new errors_js_1.CrwError(`Batch scrape failed: ${status.error ?? "unknown"}`);
|
|
341
|
+
// A cancelled job never completes; stop instead of spinning until timeout.
|
|
342
|
+
if (status.status === "cancelled")
|
|
343
|
+
throw new errors_js_1.CrwError(`Batch scrape ${jobId} was cancelled`);
|
|
318
344
|
await sleep(pollInterval * 1000);
|
|
319
345
|
}
|
|
346
|
+
// Page through the full result set. The batch status returns at most ~100
|
|
347
|
+
// documents per page and sets `next` to the cursor for the next page (null on
|
|
348
|
+
// the last). Returning only page 1 silently truncated batches over one page.
|
|
349
|
+
const docs = [...(status.data ?? [])];
|
|
350
|
+
let nextUrl = status.next;
|
|
351
|
+
// Seed with page 1's path: a `next` that rebuilds to it (skip=0, or no cursor
|
|
352
|
+
// at all) is a repeat, not a new page — stop rather than re-append.
|
|
353
|
+
const seen = new Set([statusPath]);
|
|
354
|
+
while (nextUrl) {
|
|
355
|
+
const pagePath = pageCursorPath(statusPath, nextUrl);
|
|
356
|
+
if (seen.has(pagePath))
|
|
357
|
+
break; // repeating/empty cursor; nothing new
|
|
358
|
+
if (Date.now() > deadline)
|
|
359
|
+
throw new errors_js_1.CrwTimeoutError(`Batch scrape ${jobId} timed out after ${timeout}s`);
|
|
360
|
+
seen.add(pagePath);
|
|
361
|
+
const page = await this.httpRequest("GET", pagePath, undefined, { raw: true, checkSuccess: false });
|
|
362
|
+
docs.push(...(page.data ?? []));
|
|
363
|
+
nextUrl = page.next;
|
|
364
|
+
}
|
|
365
|
+
return docs;
|
|
320
366
|
}
|
|
321
367
|
/** Feature-detect the engine (HTTP mode only). */
|
|
322
368
|
async capabilities() {
|
|
@@ -360,6 +406,8 @@ class CrwClient {
|
|
|
360
406
|
return result.data ?? [];
|
|
361
407
|
if (result.status === "failed")
|
|
362
408
|
throw new errors_js_1.CrwError(`Crawl failed: ${result.error ?? "unknown"}`);
|
|
409
|
+
if (result.status === "cancelled")
|
|
410
|
+
throw new errors_js_1.CrwError(`Crawl ${jobId} was cancelled`);
|
|
363
411
|
await sleep(pollInterval * 1000);
|
|
364
412
|
}
|
|
365
413
|
}
|
|
@@ -431,6 +479,8 @@ class CrwClient {
|
|
|
431
479
|
return status.data ?? [];
|
|
432
480
|
if (status.status === "failed")
|
|
433
481
|
throw new errors_js_1.CrwError(`Crawl failed: ${status.error ?? "unknown"}`);
|
|
482
|
+
if (status.status === "cancelled")
|
|
483
|
+
throw new errors_js_1.CrwError(`Crawl ${jobId} was cancelled`);
|
|
434
484
|
await sleep(pollInterval * 1000);
|
|
435
485
|
}
|
|
436
486
|
}
|
package/dist/esm/client.js
CHANGED
|
@@ -18,6 +18,26 @@ function httpOnlyHint(name, reason) {
|
|
|
18
18
|
`Use the cloud (set CRW_API_KEY) or pass apiUrl for a self-hosted server.`);
|
|
19
19
|
}
|
|
20
20
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
21
|
+
/**
|
|
22
|
+
* Next page request, rebuilt from OUR base path + the cursor's skip/limit. The
|
|
23
|
+
* server's `next` is an absolute URL from its own public origin/prefix; reusing
|
|
24
|
+
* it verbatim double-prefixes when apiUrl carries a path (`.../api`) and trusts a
|
|
25
|
+
* host that may not match apiUrl. We only borrow the cursor position.
|
|
26
|
+
*/
|
|
27
|
+
function pageCursorPath(basePath, nextUrl) {
|
|
28
|
+
let search = "";
|
|
29
|
+
try {
|
|
30
|
+
search = new URL(nextUrl, "http://placeholder.invalid").search;
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
return basePath;
|
|
34
|
+
}
|
|
35
|
+
const q = new URLSearchParams(search);
|
|
36
|
+
const params = ["skip", "limit"]
|
|
37
|
+
.filter((k) => q.get(k) !== null)
|
|
38
|
+
.map((k) => `${k}=${q.get(k)}`);
|
|
39
|
+
return params.length ? `${basePath}?${params.join("&")}` : basePath;
|
|
40
|
+
}
|
|
21
41
|
export class CrwClient {
|
|
22
42
|
apiUrl;
|
|
23
43
|
apiKey;
|
|
@@ -304,16 +324,42 @@ export class CrwClient {
|
|
|
304
324
|
if (!jobId)
|
|
305
325
|
throw new CrwError(`Batch scrape did not return job ID: ${JSON.stringify(start)}`);
|
|
306
326
|
const deadline = Date.now() + timeout * 1000;
|
|
327
|
+
const statusPath = `/v2/batch/scrape/${jobId}`;
|
|
328
|
+
// Poll the first page until the job reaches a terminal state.
|
|
329
|
+
let status;
|
|
307
330
|
for (;;) {
|
|
308
331
|
if (Date.now() > deadline)
|
|
309
332
|
throw new CrwTimeoutError(`Batch scrape ${jobId} timed out after ${timeout}s`);
|
|
310
|
-
|
|
333
|
+
status = await this.httpRequest("GET", statusPath, undefined, { raw: true, checkSuccess: false });
|
|
311
334
|
if (status.status === "completed")
|
|
312
|
-
|
|
335
|
+
break;
|
|
313
336
|
if (status.status === "failed")
|
|
314
337
|
throw new CrwError(`Batch scrape failed: ${status.error ?? "unknown"}`);
|
|
338
|
+
// A cancelled job never completes; stop instead of spinning until timeout.
|
|
339
|
+
if (status.status === "cancelled")
|
|
340
|
+
throw new CrwError(`Batch scrape ${jobId} was cancelled`);
|
|
315
341
|
await sleep(pollInterval * 1000);
|
|
316
342
|
}
|
|
343
|
+
// Page through the full result set. The batch status returns at most ~100
|
|
344
|
+
// documents per page and sets `next` to the cursor for the next page (null on
|
|
345
|
+
// the last). Returning only page 1 silently truncated batches over one page.
|
|
346
|
+
const docs = [...(status.data ?? [])];
|
|
347
|
+
let nextUrl = status.next;
|
|
348
|
+
// Seed with page 1's path: a `next` that rebuilds to it (skip=0, or no cursor
|
|
349
|
+
// at all) is a repeat, not a new page — stop rather than re-append.
|
|
350
|
+
const seen = new Set([statusPath]);
|
|
351
|
+
while (nextUrl) {
|
|
352
|
+
const pagePath = pageCursorPath(statusPath, nextUrl);
|
|
353
|
+
if (seen.has(pagePath))
|
|
354
|
+
break; // repeating/empty cursor; nothing new
|
|
355
|
+
if (Date.now() > deadline)
|
|
356
|
+
throw new CrwTimeoutError(`Batch scrape ${jobId} timed out after ${timeout}s`);
|
|
357
|
+
seen.add(pagePath);
|
|
358
|
+
const page = await this.httpRequest("GET", pagePath, undefined, { raw: true, checkSuccess: false });
|
|
359
|
+
docs.push(...(page.data ?? []));
|
|
360
|
+
nextUrl = page.next;
|
|
361
|
+
}
|
|
362
|
+
return docs;
|
|
317
363
|
}
|
|
318
364
|
/** Feature-detect the engine (HTTP mode only). */
|
|
319
365
|
async capabilities() {
|
|
@@ -357,6 +403,8 @@ export class CrwClient {
|
|
|
357
403
|
return result.data ?? [];
|
|
358
404
|
if (result.status === "failed")
|
|
359
405
|
throw new CrwError(`Crawl failed: ${result.error ?? "unknown"}`);
|
|
406
|
+
if (result.status === "cancelled")
|
|
407
|
+
throw new CrwError(`Crawl ${jobId} was cancelled`);
|
|
360
408
|
await sleep(pollInterval * 1000);
|
|
361
409
|
}
|
|
362
410
|
}
|
|
@@ -428,6 +476,8 @@ export class CrwClient {
|
|
|
428
476
|
return status.data ?? [];
|
|
429
477
|
if (status.status === "failed")
|
|
430
478
|
throw new CrwError(`Crawl failed: ${status.error ?? "unknown"}`);
|
|
479
|
+
if (status.status === "cancelled")
|
|
480
|
+
throw new CrwError(`Crawl ${jobId} was cancelled`);
|
|
431
481
|
await sleep(pollInterval * 1000);
|
|
432
482
|
}
|
|
433
483
|
}
|
package/package.json
CHANGED