crw-sdk 0.30.0 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cjs/client.js +59 -9
- package/dist/esm/client.d.ts +2 -2
- package/dist/esm/client.js +59 -9
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -14,7 +14,7 @@ npm install crw-sdk
|
|
|
14
14
|
|
|
15
15
|
## Quick start — Cloud (default)
|
|
16
16
|
|
|
17
|
-
CRW is **cloud-first**. [Sign up for
|
|
17
|
+
CRW is **cloud-first**. [Sign up for 1000 free credits](https://fastcrw.com/dashboard)
|
|
18
18
|
— no payment, no monthly reset (GitHub/Google, ~10s) — then set `CRW_API_KEY`:
|
|
19
19
|
|
|
20
20
|
```ts
|
package/dist/cjs/client.js
CHANGED
|
@@ -10,7 +10,7 @@ exports.CLOUD_API_URL = "https://api.fastcrw.com";
|
|
|
10
10
|
exports.DASHBOARD_URL = "https://fastcrw.com/dashboard";
|
|
11
11
|
exports.DOCS_URL = "https://us.github.io/crw";
|
|
12
12
|
const SIGNUP_NUDGE = `No CRW API key found. CRW uses the managed cloud (${exports.CLOUD_API_URL}) by default.\n` +
|
|
13
|
-
` -> Sign up at ${exports.DASHBOARD_URL} for
|
|
13
|
+
` -> Sign up at ${exports.DASHBOARD_URL} for 1000 free credits — no payment, no monthly ` +
|
|
14
14
|
`reset (GitHub/Google, ~10s) — then set CRW_API_KEY (or pass apiKey).\n` +
|
|
15
15
|
` -> Prefer to self-host? Set CRW_LOCAL=1 to run the local engine. Docs: ${exports.DOCS_URL}`;
|
|
16
16
|
function envTruthy(value) {
|
|
@@ -21,13 +21,33 @@ function httpOnlyHint(name, reason) {
|
|
|
21
21
|
`Use the cloud (set CRW_API_KEY) or pass apiUrl for a self-hosted server.`);
|
|
22
22
|
}
|
|
23
23
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
24
|
+
/**
|
|
25
|
+
* Next page request, rebuilt from OUR base path + the cursor's skip/limit. The
|
|
26
|
+
* server's `next` is an absolute URL from its own public origin/prefix; reusing
|
|
27
|
+
* it verbatim double-prefixes when apiUrl carries a path (`.../api`) and trusts a
|
|
28
|
+
* host that may not match apiUrl. We only borrow the cursor position.
|
|
29
|
+
*/
|
|
30
|
+
function pageCursorPath(basePath, nextUrl) {
|
|
31
|
+
let search = "";
|
|
32
|
+
try {
|
|
33
|
+
search = new URL(nextUrl, "http://placeholder.invalid").search;
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
return basePath;
|
|
37
|
+
}
|
|
38
|
+
const q = new URLSearchParams(search);
|
|
39
|
+
const params = ["skip", "limit"]
|
|
40
|
+
.filter((k) => q.get(k) !== null)
|
|
41
|
+
.map((k) => `${k}=${q.get(k)}`);
|
|
42
|
+
return params.length ? `${basePath}?${params.join("&")}` : basePath;
|
|
43
|
+
}
|
|
24
44
|
class CrwClient {
|
|
25
45
|
apiUrl;
|
|
26
46
|
apiKey;
|
|
27
47
|
local = null;
|
|
28
48
|
/**
|
|
29
49
|
* CRW is cloud-first. With no arguments the client targets the managed cloud
|
|
30
|
-
* (api.fastcrw.com) and needs an API key — sign up for
|
|
50
|
+
* (api.fastcrw.com) and needs an API key — sign up for 1000 free credits at
|
|
31
51
|
* https://fastcrw.com/dashboard. To self-host locally, set `CRW_LOCAL=1`.
|
|
32
52
|
*/
|
|
33
53
|
constructor(opts = {}) {
|
|
@@ -145,7 +165,7 @@ class CrwClient {
|
|
|
145
165
|
}
|
|
146
166
|
/**
|
|
147
167
|
* Firecrawl-compatible Research API (cloud only). Mirrors the Firecrawl
|
|
148
|
-
* research SDK surface over `/
|
|
168
|
+
* research SDK surface over `/v1/search/research/*`. Each method GETs the
|
|
149
169
|
* hosted endpoint and returns its `{ success, ... }` payload verbatim.
|
|
150
170
|
*/
|
|
151
171
|
get research() {
|
|
@@ -161,10 +181,10 @@ class CrwClient {
|
|
|
161
181
|
});
|
|
162
182
|
};
|
|
163
183
|
return {
|
|
164
|
-
searchPapers: (query, opts = {}) => get("/
|
|
165
|
-
getPaper: (id, opts = {}) => get(`/
|
|
166
|
-
similarPapers: (id, opts) => get(`/
|
|
167
|
-
searchGithub: (query, opts = {}) => get("/
|
|
184
|
+
searchPapers: (query, opts = {}) => get("/v1/search/research/papers", { query, ...opts }),
|
|
185
|
+
getPaper: (id, opts = {}) => get(`/v1/search/research/papers/${encodeURIComponent(id)}`, { ...opts }),
|
|
186
|
+
similarPapers: (id, opts) => get(`/v1/search/research/papers/${encodeURIComponent(id)}/similar`, { ...opts }),
|
|
187
|
+
searchGithub: (query, opts = {}) => get("/v1/search/research/github", { query, ...opts }),
|
|
168
188
|
};
|
|
169
189
|
}
|
|
170
190
|
/**
|
|
@@ -307,16 +327,42 @@ class CrwClient {
|
|
|
307
327
|
if (!jobId)
|
|
308
328
|
throw new errors_js_1.CrwError(`Batch scrape did not return job ID: ${JSON.stringify(start)}`);
|
|
309
329
|
const deadline = Date.now() + timeout * 1000;
|
|
330
|
+
const statusPath = `/v2/batch/scrape/${jobId}`;
|
|
331
|
+
// Poll the first page until the job reaches a terminal state.
|
|
332
|
+
let status;
|
|
310
333
|
for (;;) {
|
|
311
334
|
if (Date.now() > deadline)
|
|
312
335
|
throw new errors_js_1.CrwTimeoutError(`Batch scrape ${jobId} timed out after ${timeout}s`);
|
|
313
|
-
|
|
336
|
+
status = await this.httpRequest("GET", statusPath, undefined, { raw: true, checkSuccess: false });
|
|
314
337
|
if (status.status === "completed")
|
|
315
|
-
|
|
338
|
+
break;
|
|
316
339
|
if (status.status === "failed")
|
|
317
340
|
throw new errors_js_1.CrwError(`Batch scrape failed: ${status.error ?? "unknown"}`);
|
|
341
|
+
// A cancelled job never completes; stop instead of spinning until timeout.
|
|
342
|
+
if (status.status === "cancelled")
|
|
343
|
+
throw new errors_js_1.CrwError(`Batch scrape ${jobId} was cancelled`);
|
|
318
344
|
await sleep(pollInterval * 1000);
|
|
319
345
|
}
|
|
346
|
+
// Page through the full result set. The batch status returns at most ~100
|
|
347
|
+
// documents per page and sets `next` to the cursor for the next page (null on
|
|
348
|
+
// the last). Returning only page 1 silently truncated batches over one page.
|
|
349
|
+
const docs = [...(status.data ?? [])];
|
|
350
|
+
let nextUrl = status.next;
|
|
351
|
+
// Seed with page 1's path: a `next` that rebuilds to it (skip=0, or no cursor
|
|
352
|
+
// at all) is a repeat, not a new page — stop rather than re-append.
|
|
353
|
+
const seen = new Set([statusPath]);
|
|
354
|
+
while (nextUrl) {
|
|
355
|
+
const pagePath = pageCursorPath(statusPath, nextUrl);
|
|
356
|
+
if (seen.has(pagePath))
|
|
357
|
+
break; // repeating/empty cursor; nothing new
|
|
358
|
+
if (Date.now() > deadline)
|
|
359
|
+
throw new errors_js_1.CrwTimeoutError(`Batch scrape ${jobId} timed out after ${timeout}s`);
|
|
360
|
+
seen.add(pagePath);
|
|
361
|
+
const page = await this.httpRequest("GET", pagePath, undefined, { raw: true, checkSuccess: false });
|
|
362
|
+
docs.push(...(page.data ?? []));
|
|
363
|
+
nextUrl = page.next;
|
|
364
|
+
}
|
|
365
|
+
return docs;
|
|
320
366
|
}
|
|
321
367
|
/** Feature-detect the engine (HTTP mode only). */
|
|
322
368
|
async capabilities() {
|
|
@@ -360,6 +406,8 @@ class CrwClient {
|
|
|
360
406
|
return result.data ?? [];
|
|
361
407
|
if (result.status === "failed")
|
|
362
408
|
throw new errors_js_1.CrwError(`Crawl failed: ${result.error ?? "unknown"}`);
|
|
409
|
+
if (result.status === "cancelled")
|
|
410
|
+
throw new errors_js_1.CrwError(`Crawl ${jobId} was cancelled`);
|
|
363
411
|
await sleep(pollInterval * 1000);
|
|
364
412
|
}
|
|
365
413
|
}
|
|
@@ -431,6 +479,8 @@ class CrwClient {
|
|
|
431
479
|
return status.data ?? [];
|
|
432
480
|
if (status.status === "failed")
|
|
433
481
|
throw new errors_js_1.CrwError(`Crawl failed: ${status.error ?? "unknown"}`);
|
|
482
|
+
if (status.status === "cancelled")
|
|
483
|
+
throw new errors_js_1.CrwError(`Crawl ${jobId} was cancelled`);
|
|
434
484
|
await sleep(pollInterval * 1000);
|
|
435
485
|
}
|
|
436
486
|
}
|
package/dist/esm/client.d.ts
CHANGED
|
@@ -9,7 +9,7 @@ export declare class CrwClient {
|
|
|
9
9
|
private local;
|
|
10
10
|
/**
|
|
11
11
|
* CRW is cloud-first. With no arguments the client targets the managed cloud
|
|
12
|
-
* (api.fastcrw.com) and needs an API key — sign up for
|
|
12
|
+
* (api.fastcrw.com) and needs an API key — sign up for 1000 free credits at
|
|
13
13
|
* https://fastcrw.com/dashboard. To self-host locally, set `CRW_LOCAL=1`.
|
|
14
14
|
*/
|
|
15
15
|
constructor(opts?: ClientOptions);
|
|
@@ -22,7 +22,7 @@ export declare class CrwClient {
|
|
|
22
22
|
search(query: string, opts?: SearchOptions): Promise<SearchResult>;
|
|
23
23
|
/**
|
|
24
24
|
* Firecrawl-compatible Research API (cloud only). Mirrors the Firecrawl
|
|
25
|
-
* research SDK surface over `/
|
|
25
|
+
* research SDK surface over `/v1/search/research/*`. Each method GETs the
|
|
26
26
|
* hosted endpoint and returns its `{ success, ... }` payload verbatim.
|
|
27
27
|
*/
|
|
28
28
|
get research(): {
|
package/dist/esm/client.js
CHANGED
|
@@ -7,7 +7,7 @@ export const CLOUD_API_URL = "https://api.fastcrw.com";
|
|
|
7
7
|
export const DASHBOARD_URL = "https://fastcrw.com/dashboard";
|
|
8
8
|
export const DOCS_URL = "https://us.github.io/crw";
|
|
9
9
|
const SIGNUP_NUDGE = `No CRW API key found. CRW uses the managed cloud (${CLOUD_API_URL}) by default.\n` +
|
|
10
|
-
` -> Sign up at ${DASHBOARD_URL} for
|
|
10
|
+
` -> Sign up at ${DASHBOARD_URL} for 1000 free credits — no payment, no monthly ` +
|
|
11
11
|
`reset (GitHub/Google, ~10s) — then set CRW_API_KEY (or pass apiKey).\n` +
|
|
12
12
|
` -> Prefer to self-host? Set CRW_LOCAL=1 to run the local engine. Docs: ${DOCS_URL}`;
|
|
13
13
|
function envTruthy(value) {
|
|
@@ -18,13 +18,33 @@ function httpOnlyHint(name, reason) {
|
|
|
18
18
|
`Use the cloud (set CRW_API_KEY) or pass apiUrl for a self-hosted server.`);
|
|
19
19
|
}
|
|
20
20
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
21
|
+
/**
|
|
22
|
+
* Next page request, rebuilt from OUR base path + the cursor's skip/limit. The
|
|
23
|
+
* server's `next` is an absolute URL from its own public origin/prefix; reusing
|
|
24
|
+
* it verbatim double-prefixes when apiUrl carries a path (`.../api`) and trusts a
|
|
25
|
+
* host that may not match apiUrl. We only borrow the cursor position.
|
|
26
|
+
*/
|
|
27
|
+
function pageCursorPath(basePath, nextUrl) {
|
|
28
|
+
let search = "";
|
|
29
|
+
try {
|
|
30
|
+
search = new URL(nextUrl, "http://placeholder.invalid").search;
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
return basePath;
|
|
34
|
+
}
|
|
35
|
+
const q = new URLSearchParams(search);
|
|
36
|
+
const params = ["skip", "limit"]
|
|
37
|
+
.filter((k) => q.get(k) !== null)
|
|
38
|
+
.map((k) => `${k}=${q.get(k)}`);
|
|
39
|
+
return params.length ? `${basePath}?${params.join("&")}` : basePath;
|
|
40
|
+
}
|
|
21
41
|
export class CrwClient {
|
|
22
42
|
apiUrl;
|
|
23
43
|
apiKey;
|
|
24
44
|
local = null;
|
|
25
45
|
/**
|
|
26
46
|
* CRW is cloud-first. With no arguments the client targets the managed cloud
|
|
27
|
-
* (api.fastcrw.com) and needs an API key — sign up for
|
|
47
|
+
* (api.fastcrw.com) and needs an API key — sign up for 1000 free credits at
|
|
28
48
|
* https://fastcrw.com/dashboard. To self-host locally, set `CRW_LOCAL=1`.
|
|
29
49
|
*/
|
|
30
50
|
constructor(opts = {}) {
|
|
@@ -142,7 +162,7 @@ export class CrwClient {
|
|
|
142
162
|
}
|
|
143
163
|
/**
|
|
144
164
|
* Firecrawl-compatible Research API (cloud only). Mirrors the Firecrawl
|
|
145
|
-
* research SDK surface over `/
|
|
165
|
+
* research SDK surface over `/v1/search/research/*`. Each method GETs the
|
|
146
166
|
* hosted endpoint and returns its `{ success, ... }` payload verbatim.
|
|
147
167
|
*/
|
|
148
168
|
get research() {
|
|
@@ -158,10 +178,10 @@ export class CrwClient {
|
|
|
158
178
|
});
|
|
159
179
|
};
|
|
160
180
|
return {
|
|
161
|
-
searchPapers: (query, opts = {}) => get("/
|
|
162
|
-
getPaper: (id, opts = {}) => get(`/
|
|
163
|
-
similarPapers: (id, opts) => get(`/
|
|
164
|
-
searchGithub: (query, opts = {}) => get("/
|
|
181
|
+
searchPapers: (query, opts = {}) => get("/v1/search/research/papers", { query, ...opts }),
|
|
182
|
+
getPaper: (id, opts = {}) => get(`/v1/search/research/papers/${encodeURIComponent(id)}`, { ...opts }),
|
|
183
|
+
similarPapers: (id, opts) => get(`/v1/search/research/papers/${encodeURIComponent(id)}/similar`, { ...opts }),
|
|
184
|
+
searchGithub: (query, opts = {}) => get("/v1/search/research/github", { query, ...opts }),
|
|
165
185
|
};
|
|
166
186
|
}
|
|
167
187
|
/**
|
|
@@ -304,16 +324,42 @@ export class CrwClient {
|
|
|
304
324
|
if (!jobId)
|
|
305
325
|
throw new CrwError(`Batch scrape did not return job ID: ${JSON.stringify(start)}`);
|
|
306
326
|
const deadline = Date.now() + timeout * 1000;
|
|
327
|
+
const statusPath = `/v2/batch/scrape/${jobId}`;
|
|
328
|
+
// Poll the first page until the job reaches a terminal state.
|
|
329
|
+
let status;
|
|
307
330
|
for (;;) {
|
|
308
331
|
if (Date.now() > deadline)
|
|
309
332
|
throw new CrwTimeoutError(`Batch scrape ${jobId} timed out after ${timeout}s`);
|
|
310
|
-
|
|
333
|
+
status = await this.httpRequest("GET", statusPath, undefined, { raw: true, checkSuccess: false });
|
|
311
334
|
if (status.status === "completed")
|
|
312
|
-
|
|
335
|
+
break;
|
|
313
336
|
if (status.status === "failed")
|
|
314
337
|
throw new CrwError(`Batch scrape failed: ${status.error ?? "unknown"}`);
|
|
338
|
+
// A cancelled job never completes; stop instead of spinning until timeout.
|
|
339
|
+
if (status.status === "cancelled")
|
|
340
|
+
throw new CrwError(`Batch scrape ${jobId} was cancelled`);
|
|
315
341
|
await sleep(pollInterval * 1000);
|
|
316
342
|
}
|
|
343
|
+
// Page through the full result set. The batch status returns at most ~100
|
|
344
|
+
// documents per page and sets `next` to the cursor for the next page (null on
|
|
345
|
+
// the last). Returning only page 1 silently truncated batches over one page.
|
|
346
|
+
const docs = [...(status.data ?? [])];
|
|
347
|
+
let nextUrl = status.next;
|
|
348
|
+
// Seed with page 1's path: a `next` that rebuilds to it (skip=0, or no cursor
|
|
349
|
+
// at all) is a repeat, not a new page — stop rather than re-append.
|
|
350
|
+
const seen = new Set([statusPath]);
|
|
351
|
+
while (nextUrl) {
|
|
352
|
+
const pagePath = pageCursorPath(statusPath, nextUrl);
|
|
353
|
+
if (seen.has(pagePath))
|
|
354
|
+
break; // repeating/empty cursor; nothing new
|
|
355
|
+
if (Date.now() > deadline)
|
|
356
|
+
throw new CrwTimeoutError(`Batch scrape ${jobId} timed out after ${timeout}s`);
|
|
357
|
+
seen.add(pagePath);
|
|
358
|
+
const page = await this.httpRequest("GET", pagePath, undefined, { raw: true, checkSuccess: false });
|
|
359
|
+
docs.push(...(page.data ?? []));
|
|
360
|
+
nextUrl = page.next;
|
|
361
|
+
}
|
|
362
|
+
return docs;
|
|
317
363
|
}
|
|
318
364
|
/** Feature-detect the engine (HTTP mode only). */
|
|
319
365
|
async capabilities() {
|
|
@@ -357,6 +403,8 @@ export class CrwClient {
|
|
|
357
403
|
return result.data ?? [];
|
|
358
404
|
if (result.status === "failed")
|
|
359
405
|
throw new CrwError(`Crawl failed: ${result.error ?? "unknown"}`);
|
|
406
|
+
if (result.status === "cancelled")
|
|
407
|
+
throw new CrwError(`Crawl ${jobId} was cancelled`);
|
|
360
408
|
await sleep(pollInterval * 1000);
|
|
361
409
|
}
|
|
362
410
|
}
|
|
@@ -428,6 +476,8 @@ export class CrwClient {
|
|
|
428
476
|
return status.data ?? [];
|
|
429
477
|
if (status.status === "failed")
|
|
430
478
|
throw new CrwError(`Crawl failed: ${status.error ?? "unknown"}`);
|
|
479
|
+
if (status.status === "cancelled")
|
|
480
|
+
throw new CrwError(`Crawl ${jobId} was cancelled`);
|
|
431
481
|
await sleep(pollInterval * 1000);
|
|
432
482
|
}
|
|
433
483
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "crw-sdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.32.0",
|
|
4
4
|
"description": "TypeScript/JavaScript SDK for CRW — scrape, crawl, map, search, parse, and extract any website",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "fastCRW <hello@fastcrw.com>",
|
|
@@ -44,6 +44,6 @@
|
|
|
44
44
|
},
|
|
45
45
|
"devDependencies": {
|
|
46
46
|
"@types/node": "^26",
|
|
47
|
-
"typescript": "^
|
|
47
|
+
"typescript": "^7"
|
|
48
48
|
}
|
|
49
49
|
}
|