crw-sdk 0.24.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dist/cjs/client.js +81 -26
- package/dist/cjs/errors.js +13 -1
- package/dist/cjs/index.js +2 -1
- package/dist/esm/client.d.ts +13 -4
- package/dist/esm/client.js +82 -27
- package/dist/esm/errors.d.ts +7 -0
- package/dist/esm/errors.js +11 -0
- package/dist/esm/index.d.ts +2 -2
- package/dist/esm/index.js +1 -1
- package/dist/esm/types.d.ts +31 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -53,6 +53,7 @@ CRW_LOCAL=1 node app.js
|
|
|
53
53
|
| `search(query, opts?)` | Web search (+ optional scrape) | both¹ |
|
|
54
54
|
| `parseFile(bytes, opts?)` | PDF → markdown / structured JSON | both |
|
|
55
55
|
| `extract({urls, schema?})` | Structured LLM extraction | HTTP |
|
|
56
|
+
| `startExtract(opts)` / `getExtract(id)` / `cancelExtract(id)` | Typed extract lifecycle | HTTP |
|
|
56
57
|
| `batchScrape(urls, opts?)` | Scrape many URLs (async) | HTTP |
|
|
57
58
|
| `capabilities()` | Feature-detect the engine | HTTP |
|
|
58
59
|
| `changeTrackingDiff(cur, prev?)` | Diff vs a prior snapshot | HTTP |
|
|
@@ -69,6 +70,14 @@ const results = await crw.extract({
|
|
|
69
70
|
// results: [{ url, status, data, error, llmUsage }]
|
|
70
71
|
for (const r of results) if (r.status === "completed") console.log(r.url, r.data);
|
|
71
72
|
|
|
73
|
+
// Explicit lifecycle. startExtract always sends Prefer: respond-async.
|
|
74
|
+
const accepted = await crw.startExtract({
|
|
75
|
+
urls: ["https://a.example", "https://b.example"],
|
|
76
|
+
schema: { type: "object", properties: { title: { type: "string" } } },
|
|
77
|
+
});
|
|
78
|
+
const status = await crw.getExtract(accepted.id);
|
|
79
|
+
await crw.cancelExtract(accepted.id); // idempotent
|
|
80
|
+
|
|
72
81
|
// Parse a PDF:
|
|
73
82
|
import { readFileSync } from "node:fs";
|
|
74
83
|
const doc = await crw.parseFile(readFileSync("invoice.pdf"), { formats: ["markdown"] });
|
package/dist/cjs/client.js
CHANGED
|
@@ -199,49 +199,101 @@ class CrwClient {
|
|
|
199
199
|
return this.localTransport().toolCall("crw_parse_file", args);
|
|
200
200
|
}
|
|
201
201
|
/**
|
|
202
|
-
*
|
|
203
|
-
*
|
|
204
|
-
|
|
202
|
+
* Start native multi-URL extraction without waiting. `Prefer: respond-async`
|
|
203
|
+
* is always sent so managed and self-hosted servers use the same lifecycle.
|
|
204
|
+
*/
|
|
205
|
+
async startExtract(opts) {
|
|
206
|
+
if (!this.apiUrl)
|
|
207
|
+
throw new errors_js_1.CrwError(httpOnlyHint("startExtract", "LLM extract job endpoint"));
|
|
208
|
+
const start = await this.startExtractRequest(opts);
|
|
209
|
+
if (typeof start.id === "string" && start.status === "processing") {
|
|
210
|
+
return start;
|
|
211
|
+
}
|
|
212
|
+
throw new errors_js_1.CrwError(`startExtract did not return an accepted job: ${JSON.stringify(start)}`);
|
|
213
|
+
}
|
|
214
|
+
/** Get the canonical lifecycle envelope for an extract job. */
|
|
215
|
+
async getExtract(id) {
|
|
216
|
+
if (!this.apiUrl)
|
|
217
|
+
throw new errors_js_1.CrwError(httpOnlyHint("getExtract", "LLM extract job endpoint"));
|
|
218
|
+
const result = await this.httpRequest("GET", `/v1/extract/${encodeURIComponent(id)}`, undefined, {
|
|
219
|
+
raw: true,
|
|
220
|
+
checkSuccess: false,
|
|
221
|
+
});
|
|
222
|
+
return result;
|
|
223
|
+
}
|
|
224
|
+
/** Idempotently request cancellation and return the canonical lifecycle envelope. */
|
|
225
|
+
async cancelExtract(id) {
|
|
226
|
+
if (!this.apiUrl)
|
|
227
|
+
throw new errors_js_1.CrwError(httpOnlyHint("cancelExtract", "LLM extract job endpoint"));
|
|
228
|
+
const result = await this.httpRequest("DELETE", `/v1/extract/${encodeURIComponent(id)}`, undefined, {
|
|
229
|
+
raw: true,
|
|
230
|
+
checkSuccess: false,
|
|
231
|
+
});
|
|
232
|
+
return result;
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Convenience waiter over start/get/cancel. `cancelling` is non-terminal;
|
|
236
|
+
* terminal cancellation raises a typed error carrying partial results.
|
|
205
237
|
*/
|
|
206
238
|
async extract(opts) {
|
|
207
239
|
if (!this.apiUrl)
|
|
208
240
|
throw new errors_js_1.CrwError(httpOnlyHint("extract", "LLM extract job endpoint"));
|
|
209
|
-
const {
|
|
210
|
-
const
|
|
211
|
-
if (prompt !== undefined)
|
|
212
|
-
body.prompt = prompt;
|
|
213
|
-
if (schema !== undefined)
|
|
214
|
-
body.schema = schema;
|
|
215
|
-
if (basis !== undefined)
|
|
216
|
-
body.basis = basis;
|
|
217
|
-
if (llmApiKey !== undefined)
|
|
218
|
-
body.llmApiKey = llmApiKey;
|
|
219
|
-
if (llmProvider !== undefined)
|
|
220
|
-
body.llmProvider = llmProvider;
|
|
221
|
-
if (llmModel !== undefined)
|
|
222
|
-
body.llmModel = llmModel;
|
|
223
|
-
const start = await this.httpRequest("POST", "/v1/extract", body, { raw: true });
|
|
241
|
+
const { pollInterval = 2, timeout = 120 } = opts;
|
|
242
|
+
const start = await this.startExtractRequest(opts);
|
|
224
243
|
const jobId = start.id;
|
|
225
|
-
//
|
|
226
|
-
//
|
|
227
|
-
// path hands back an `id`. Demanding one made every managed extract() throw.
|
|
244
|
+
// Preserve the managed synchronous response during rollout if an older
|
|
245
|
+
// endpoint ignores Prefer. New managed/self-hosted fixtures return an id.
|
|
228
246
|
if (!jobId) {
|
|
247
|
+
if (start.status === "cancelled") {
|
|
248
|
+
throw new errors_js_1.CrwExtractCancelledError(start);
|
|
249
|
+
}
|
|
229
250
|
if (Array.isArray(start.results))
|
|
230
251
|
return start.results;
|
|
231
252
|
throw new errors_js_1.CrwError(`extract returned neither a job id nor results: ${JSON.stringify(start)}`);
|
|
232
253
|
}
|
|
233
254
|
const deadline = Date.now() + timeout * 1000;
|
|
234
255
|
for (;;) {
|
|
235
|
-
if (Date.now() > deadline)
|
|
256
|
+
if (Date.now() > deadline) {
|
|
257
|
+
try {
|
|
258
|
+
await this.cancelExtract(jobId);
|
|
259
|
+
}
|
|
260
|
+
catch {
|
|
261
|
+
// Timeout remains the caller-visible failure; DELETE is best effort.
|
|
262
|
+
}
|
|
236
263
|
throw new errors_js_1.CrwTimeoutError(`Extract ${jobId} timed out after ${timeout}s`);
|
|
237
|
-
|
|
264
|
+
}
|
|
265
|
+
const status = await this.getExtract(jobId);
|
|
238
266
|
if (status.status === "completed")
|
|
239
|
-
return status.results
|
|
267
|
+
return status.results;
|
|
240
268
|
if (status.status === "failed")
|
|
241
269
|
throw new errors_js_1.CrwError(`Extract failed: ${status.error ?? "unknown"}`);
|
|
270
|
+
if (status.status === "cancelled")
|
|
271
|
+
throw new errors_js_1.CrwExtractCancelledError(status);
|
|
242
272
|
await sleep(pollInterval * 1000);
|
|
243
273
|
}
|
|
244
274
|
}
|
|
275
|
+
async startExtractRequest(opts) {
|
|
276
|
+
const { urls, prompt, schema, basis, llmApiKey, llmProvider, llmModel, pollInterval = 2, timeout = 120 } = opts;
|
|
277
|
+
void pollInterval;
|
|
278
|
+
void timeout;
|
|
279
|
+
const body = { urls: [...urls] };
|
|
280
|
+
if (prompt !== undefined)
|
|
281
|
+
body.prompt = prompt;
|
|
282
|
+
if (schema !== undefined)
|
|
283
|
+
body.schema = schema;
|
|
284
|
+
if (basis !== undefined)
|
|
285
|
+
body.basis = basis;
|
|
286
|
+
if (llmApiKey !== undefined)
|
|
287
|
+
body.llmApiKey = llmApiKey;
|
|
288
|
+
if (llmProvider !== undefined)
|
|
289
|
+
body.llmProvider = llmProvider;
|
|
290
|
+
if (llmModel !== undefined)
|
|
291
|
+
body.llmModel = llmModel;
|
|
292
|
+
return this.httpRequest("POST", "/v1/extract", body, {
|
|
293
|
+
raw: true,
|
|
294
|
+
headers: { Prefer: "respond-async" },
|
|
295
|
+
});
|
|
296
|
+
}
|
|
245
297
|
/** Scrape many URLs in one async batch job (HTTP mode only). */
|
|
246
298
|
async batchScrape(urls, opts = {}) {
|
|
247
299
|
if (!this.apiUrl)
|
|
@@ -312,11 +364,14 @@ class CrwClient {
|
|
|
312
364
|
}
|
|
313
365
|
}
|
|
314
366
|
// --- HTTP mode ---
|
|
315
|
-
async httpRequest(method, path, body, { raw = false, checkSuccess = true } = {}) {
|
|
367
|
+
async httpRequest(method, path, body, { raw = false, checkSuccess = true, headers: extraHeaders, } = {}) {
|
|
316
368
|
if (this.apiUrl === null)
|
|
317
369
|
throw new errors_js_1.CrwError("internal: httpRequest in local mode");
|
|
318
370
|
const url = `${this.apiUrl.replace(/\/$/, "")}${path}`;
|
|
319
|
-
const headers = {
|
|
371
|
+
const headers = {
|
|
372
|
+
"Content-Type": "application/json",
|
|
373
|
+
...extraHeaders,
|
|
374
|
+
};
|
|
320
375
|
if (this.apiKey)
|
|
321
376
|
headers.Authorization = `Bearer ${this.apiKey}`;
|
|
322
377
|
const resp = await fetch(url, { method, headers, body: body ? JSON.stringify(body) : undefined });
|
package/dist/cjs/errors.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/** CRW SDK error types. */
|
|
3
3
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
-
exports.CrwBinaryNotFoundError = exports.CrwTimeoutError = exports.CrwApiError = exports.CrwError = void 0;
|
|
4
|
+
exports.CrwBinaryNotFoundError = exports.CrwExtractCancelledError = exports.CrwTimeoutError = exports.CrwApiError = exports.CrwError = void 0;
|
|
5
5
|
class CrwError extends Error {
|
|
6
6
|
constructor(message) {
|
|
7
7
|
super(message);
|
|
@@ -25,6 +25,18 @@ class CrwTimeoutError extends CrwError {
|
|
|
25
25
|
}
|
|
26
26
|
}
|
|
27
27
|
exports.CrwTimeoutError = CrwTimeoutError;
|
|
28
|
+
/** A convenience extract waiter reached the immutable cancelled state. */
|
|
29
|
+
class CrwExtractCancelledError extends CrwError {
|
|
30
|
+
status;
|
|
31
|
+
results;
|
|
32
|
+
constructor(status) {
|
|
33
|
+
super(`Extract ${status.id} was cancelled`);
|
|
34
|
+
this.name = "CrwExtractCancelledError";
|
|
35
|
+
this.status = status;
|
|
36
|
+
this.results = status.results;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
exports.CrwExtractCancelledError = CrwExtractCancelledError;
|
|
28
40
|
class CrwBinaryNotFoundError extends CrwError {
|
|
29
41
|
constructor(message) {
|
|
30
42
|
super(message);
|
package/dist/cjs/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.CrwBinaryNotFoundError = exports.CrwTimeoutError = exports.CrwApiError = exports.CrwError = exports.DOCS_URL = exports.DASHBOARD_URL = exports.CLOUD_API_URL = exports.CrwClient = void 0;
|
|
3
|
+
exports.CrwBinaryNotFoundError = exports.CrwExtractCancelledError = exports.CrwTimeoutError = exports.CrwApiError = exports.CrwError = exports.DOCS_URL = exports.DASHBOARD_URL = exports.CLOUD_API_URL = exports.CrwClient = void 0;
|
|
4
4
|
var client_js_1 = require("./client.js");
|
|
5
5
|
Object.defineProperty(exports, "CrwClient", { enumerable: true, get: function () { return client_js_1.CrwClient; } });
|
|
6
6
|
Object.defineProperty(exports, "CLOUD_API_URL", { enumerable: true, get: function () { return client_js_1.CLOUD_API_URL; } });
|
|
@@ -10,4 +10,5 @@ var errors_js_1 = require("./errors.js");
|
|
|
10
10
|
Object.defineProperty(exports, "CrwError", { enumerable: true, get: function () { return errors_js_1.CrwError; } });
|
|
11
11
|
Object.defineProperty(exports, "CrwApiError", { enumerable: true, get: function () { return errors_js_1.CrwApiError; } });
|
|
12
12
|
Object.defineProperty(exports, "CrwTimeoutError", { enumerable: true, get: function () { return errors_js_1.CrwTimeoutError; } });
|
|
13
|
+
Object.defineProperty(exports, "CrwExtractCancelledError", { enumerable: true, get: function () { return errors_js_1.CrwExtractCancelledError; } });
|
|
13
14
|
Object.defineProperty(exports, "CrwBinaryNotFoundError", { enumerable: true, get: function () { return errors_js_1.CrwBinaryNotFoundError; } });
|
package/dist/esm/client.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/** CRW client — cloud (default), self-hosted HTTP, or local subprocess mode. */
|
|
2
|
-
import type { BatchResult, BatchScrapeOptions, Capabilities, ChangeTrackingOptions, ClientOptions, CrawlOptions, CrawlResult, DiffResult, ExtractOptions, ExtractResult, Json, MapOptions, ParseFileOptions, ParseResult, ScrapeOptions, ScrapeResult, SearchOptions, SearchResult, ResearchSearchOptions, ResearchReadOptions, ResearchSimilarOptions } from "./types.js";
|
|
2
|
+
import type { BatchResult, BatchScrapeOptions, Capabilities, ChangeTrackingOptions, ClientOptions, CrawlOptions, CrawlResult, DiffResult, ExtractOptions, ExtractAccepted, ExtractResult, ExtractStatus, Json, MapOptions, ParseFileOptions, ParseResult, ScrapeOptions, ScrapeResult, SearchOptions, SearchResult, ResearchSearchOptions, ResearchReadOptions, ResearchSimilarOptions } from "./types.js";
|
|
3
3
|
export declare const CLOUD_API_URL = "https://api.fastcrw.com";
|
|
4
4
|
export declare const DASHBOARD_URL = "https://fastcrw.com/dashboard";
|
|
5
5
|
export declare const DOCS_URL = "https://us.github.io/crw";
|
|
@@ -38,11 +38,20 @@ export declare class CrwClient {
|
|
|
38
38
|
*/
|
|
39
39
|
parseFile(content: Uint8Array, opts?: ParseFileOptions): Promise<ParseResult>;
|
|
40
40
|
/**
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
|
|
41
|
+
* Start native multi-URL extraction without waiting. `Prefer: respond-async`
|
|
42
|
+
* is always sent so managed and self-hosted servers use the same lifecycle.
|
|
43
|
+
*/
|
|
44
|
+
startExtract(opts: ExtractOptions): Promise<ExtractAccepted>;
|
|
45
|
+
/** Get the canonical lifecycle envelope for an extract job. */
|
|
46
|
+
getExtract(id: string): Promise<ExtractStatus>;
|
|
47
|
+
/** Idempotently request cancellation and return the canonical lifecycle envelope. */
|
|
48
|
+
cancelExtract(id: string): Promise<ExtractStatus>;
|
|
49
|
+
/**
|
|
50
|
+
* Convenience waiter over start/get/cancel. `cancelling` is non-terminal;
|
|
51
|
+
* terminal cancellation raises a typed error carrying partial results.
|
|
44
52
|
*/
|
|
45
53
|
extract(opts: ExtractOptions): Promise<ExtractResult>;
|
|
54
|
+
private startExtractRequest;
|
|
46
55
|
/** Scrape many URLs in one async batch job (HTTP mode only). */
|
|
47
56
|
batchScrape(urls: string[], opts?: BatchScrapeOptions): Promise<BatchResult>;
|
|
48
57
|
/** Feature-detect the engine (HTTP mode only). */
|
package/dist/esm/client.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/** CRW client — cloud (default), self-hosted HTTP, or local subprocess mode. */
|
|
2
|
-
import { CrwApiError, CrwError, CrwTimeoutError } from "./errors.js";
|
|
2
|
+
import { CrwApiError, CrwError, CrwExtractCancelledError, CrwTimeoutError } from "./errors.js";
|
|
3
3
|
import { LocalTransport } from "./local.js";
|
|
4
4
|
// CRW is cloud-first: with no explicit apiUrl and no CRW_LOCAL opt-in, the client
|
|
5
5
|
// talks to the managed cloud. Mirrors the Python SDK + CLI onboarding.
|
|
@@ -196,49 +196,101 @@ export class CrwClient {
|
|
|
196
196
|
return this.localTransport().toolCall("crw_parse_file", args);
|
|
197
197
|
}
|
|
198
198
|
/**
|
|
199
|
-
*
|
|
200
|
-
*
|
|
201
|
-
|
|
199
|
+
* Start native multi-URL extraction without waiting. `Prefer: respond-async`
|
|
200
|
+
* is always sent so managed and self-hosted servers use the same lifecycle.
|
|
201
|
+
*/
|
|
202
|
+
async startExtract(opts) {
|
|
203
|
+
if (!this.apiUrl)
|
|
204
|
+
throw new CrwError(httpOnlyHint("startExtract", "LLM extract job endpoint"));
|
|
205
|
+
const start = await this.startExtractRequest(opts);
|
|
206
|
+
if (typeof start.id === "string" && start.status === "processing") {
|
|
207
|
+
return start;
|
|
208
|
+
}
|
|
209
|
+
throw new CrwError(`startExtract did not return an accepted job: ${JSON.stringify(start)}`);
|
|
210
|
+
}
|
|
211
|
+
/** Get the canonical lifecycle envelope for an extract job. */
|
|
212
|
+
async getExtract(id) {
|
|
213
|
+
if (!this.apiUrl)
|
|
214
|
+
throw new CrwError(httpOnlyHint("getExtract", "LLM extract job endpoint"));
|
|
215
|
+
const result = await this.httpRequest("GET", `/v1/extract/${encodeURIComponent(id)}`, undefined, {
|
|
216
|
+
raw: true,
|
|
217
|
+
checkSuccess: false,
|
|
218
|
+
});
|
|
219
|
+
return result;
|
|
220
|
+
}
|
|
221
|
+
/** Idempotently request cancellation and return the canonical lifecycle envelope. */
|
|
222
|
+
async cancelExtract(id) {
|
|
223
|
+
if (!this.apiUrl)
|
|
224
|
+
throw new CrwError(httpOnlyHint("cancelExtract", "LLM extract job endpoint"));
|
|
225
|
+
const result = await this.httpRequest("DELETE", `/v1/extract/${encodeURIComponent(id)}`, undefined, {
|
|
226
|
+
raw: true,
|
|
227
|
+
checkSuccess: false,
|
|
228
|
+
});
|
|
229
|
+
return result;
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* Convenience waiter over start/get/cancel. `cancelling` is non-terminal;
|
|
233
|
+
* terminal cancellation raises a typed error carrying partial results.
|
|
202
234
|
*/
|
|
203
235
|
async extract(opts) {
|
|
204
236
|
if (!this.apiUrl)
|
|
205
237
|
throw new CrwError(httpOnlyHint("extract", "LLM extract job endpoint"));
|
|
206
|
-
const {
|
|
207
|
-
const
|
|
208
|
-
if (prompt !== undefined)
|
|
209
|
-
body.prompt = prompt;
|
|
210
|
-
if (schema !== undefined)
|
|
211
|
-
body.schema = schema;
|
|
212
|
-
if (basis !== undefined)
|
|
213
|
-
body.basis = basis;
|
|
214
|
-
if (llmApiKey !== undefined)
|
|
215
|
-
body.llmApiKey = llmApiKey;
|
|
216
|
-
if (llmProvider !== undefined)
|
|
217
|
-
body.llmProvider = llmProvider;
|
|
218
|
-
if (llmModel !== undefined)
|
|
219
|
-
body.llmModel = llmModel;
|
|
220
|
-
const start = await this.httpRequest("POST", "/v1/extract", body, { raw: true });
|
|
238
|
+
const { pollInterval = 2, timeout = 120 } = opts;
|
|
239
|
+
const start = await this.startExtractRequest(opts);
|
|
221
240
|
const jobId = start.id;
|
|
222
|
-
//
|
|
223
|
-
//
|
|
224
|
-
// path hands back an `id`. Demanding one made every managed extract() throw.
|
|
241
|
+
// Preserve the managed synchronous response during rollout if an older
|
|
242
|
+
// endpoint ignores Prefer. New managed/self-hosted fixtures return an id.
|
|
225
243
|
if (!jobId) {
|
|
244
|
+
if (start.status === "cancelled") {
|
|
245
|
+
throw new CrwExtractCancelledError(start);
|
|
246
|
+
}
|
|
226
247
|
if (Array.isArray(start.results))
|
|
227
248
|
return start.results;
|
|
228
249
|
throw new CrwError(`extract returned neither a job id nor results: ${JSON.stringify(start)}`);
|
|
229
250
|
}
|
|
230
251
|
const deadline = Date.now() + timeout * 1000;
|
|
231
252
|
for (;;) {
|
|
232
|
-
if (Date.now() > deadline)
|
|
253
|
+
if (Date.now() > deadline) {
|
|
254
|
+
try {
|
|
255
|
+
await this.cancelExtract(jobId);
|
|
256
|
+
}
|
|
257
|
+
catch {
|
|
258
|
+
// Timeout remains the caller-visible failure; DELETE is best effort.
|
|
259
|
+
}
|
|
233
260
|
throw new CrwTimeoutError(`Extract ${jobId} timed out after ${timeout}s`);
|
|
234
|
-
|
|
261
|
+
}
|
|
262
|
+
const status = await this.getExtract(jobId);
|
|
235
263
|
if (status.status === "completed")
|
|
236
|
-
return status.results
|
|
264
|
+
return status.results;
|
|
237
265
|
if (status.status === "failed")
|
|
238
266
|
throw new CrwError(`Extract failed: ${status.error ?? "unknown"}`);
|
|
267
|
+
if (status.status === "cancelled")
|
|
268
|
+
throw new CrwExtractCancelledError(status);
|
|
239
269
|
await sleep(pollInterval * 1000);
|
|
240
270
|
}
|
|
241
271
|
}
|
|
272
|
+
async startExtractRequest(opts) {
|
|
273
|
+
const { urls, prompt, schema, basis, llmApiKey, llmProvider, llmModel, pollInterval = 2, timeout = 120 } = opts;
|
|
274
|
+
void pollInterval;
|
|
275
|
+
void timeout;
|
|
276
|
+
const body = { urls: [...urls] };
|
|
277
|
+
if (prompt !== undefined)
|
|
278
|
+
body.prompt = prompt;
|
|
279
|
+
if (schema !== undefined)
|
|
280
|
+
body.schema = schema;
|
|
281
|
+
if (basis !== undefined)
|
|
282
|
+
body.basis = basis;
|
|
283
|
+
if (llmApiKey !== undefined)
|
|
284
|
+
body.llmApiKey = llmApiKey;
|
|
285
|
+
if (llmProvider !== undefined)
|
|
286
|
+
body.llmProvider = llmProvider;
|
|
287
|
+
if (llmModel !== undefined)
|
|
288
|
+
body.llmModel = llmModel;
|
|
289
|
+
return this.httpRequest("POST", "/v1/extract", body, {
|
|
290
|
+
raw: true,
|
|
291
|
+
headers: { Prefer: "respond-async" },
|
|
292
|
+
});
|
|
293
|
+
}
|
|
242
294
|
/** Scrape many URLs in one async batch job (HTTP mode only). */
|
|
243
295
|
async batchScrape(urls, opts = {}) {
|
|
244
296
|
if (!this.apiUrl)
|
|
@@ -309,11 +361,14 @@ export class CrwClient {
|
|
|
309
361
|
}
|
|
310
362
|
}
|
|
311
363
|
// --- HTTP mode ---
|
|
312
|
-
async httpRequest(method, path, body, { raw = false, checkSuccess = true } = {}) {
|
|
364
|
+
async httpRequest(method, path, body, { raw = false, checkSuccess = true, headers: extraHeaders, } = {}) {
|
|
313
365
|
if (this.apiUrl === null)
|
|
314
366
|
throw new CrwError("internal: httpRequest in local mode");
|
|
315
367
|
const url = `${this.apiUrl.replace(/\/$/, "")}${path}`;
|
|
316
|
-
const headers = {
|
|
368
|
+
const headers = {
|
|
369
|
+
"Content-Type": "application/json",
|
|
370
|
+
...extraHeaders,
|
|
371
|
+
};
|
|
317
372
|
if (this.apiKey)
|
|
318
373
|
headers.Authorization = `Bearer ${this.apiKey}`;
|
|
319
374
|
const resp = await fetch(url, { method, headers, body: body ? JSON.stringify(body) : undefined });
|
package/dist/esm/errors.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
/** CRW SDK error types. */
|
|
2
|
+
import type { ExtractStatus } from "./types.js";
|
|
2
3
|
export declare class CrwError extends Error {
|
|
3
4
|
constructor(message: string);
|
|
4
5
|
}
|
|
@@ -9,6 +10,12 @@ export declare class CrwApiError extends CrwError {
|
|
|
9
10
|
export declare class CrwTimeoutError extends CrwError {
|
|
10
11
|
constructor(message: string);
|
|
11
12
|
}
|
|
13
|
+
/** A convenience extract waiter reached the immutable cancelled state. */
|
|
14
|
+
export declare class CrwExtractCancelledError extends CrwError {
|
|
15
|
+
readonly status: ExtractStatus;
|
|
16
|
+
readonly results: ExtractStatus["results"];
|
|
17
|
+
constructor(status: ExtractStatus);
|
|
18
|
+
}
|
|
12
19
|
export declare class CrwBinaryNotFoundError extends CrwError {
|
|
13
20
|
constructor(message: string);
|
|
14
21
|
}
|
package/dist/esm/errors.js
CHANGED
|
@@ -19,6 +19,17 @@ export class CrwTimeoutError extends CrwError {
|
|
|
19
19
|
this.name = "CrwTimeoutError";
|
|
20
20
|
}
|
|
21
21
|
}
|
|
22
|
+
/** A convenience extract waiter reached the immutable cancelled state. */
|
|
23
|
+
export class CrwExtractCancelledError extends CrwError {
|
|
24
|
+
status;
|
|
25
|
+
results;
|
|
26
|
+
constructor(status) {
|
|
27
|
+
super(`Extract ${status.id} was cancelled`);
|
|
28
|
+
this.name = "CrwExtractCancelledError";
|
|
29
|
+
this.status = status;
|
|
30
|
+
this.results = status.results;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
22
33
|
export class CrwBinaryNotFoundError extends CrwError {
|
|
23
34
|
constructor(message) {
|
|
24
35
|
super(message);
|
package/dist/esm/index.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
export { CrwClient, CLOUD_API_URL, DASHBOARD_URL, DOCS_URL } from "./client.js";
|
|
2
|
-
export { CrwError, CrwApiError, CrwTimeoutError, CrwBinaryNotFoundError } from "./errors.js";
|
|
3
|
-
export type { ClientOptions, ScrapeOptions, CrawlOptions, MapOptions, SearchOptions, ParseFileOptions, ExtractOptions, BatchScrapeOptions, ChangeTrackingOptions, ScrapeResult, CrawlResult, SearchResult, ParseResult, ExtractResult, BatchResult, Capabilities, DiffResult, Json, } from "./types.js";
|
|
2
|
+
export { CrwError, CrwApiError, CrwTimeoutError, CrwExtractCancelledError, CrwBinaryNotFoundError, } from "./errors.js";
|
|
3
|
+
export type { ClientOptions, ScrapeOptions, CrawlOptions, MapOptions, SearchOptions, ParseFileOptions, ExtractOptions, ExtractJobState, ExtractUrlState, ExtractAccepted, ExtractStatus, ExtractUrlResult, Basis, BasisWarning, EvidenceCitation, FieldStatus, BatchScrapeOptions, ChangeTrackingOptions, ScrapeResult, CrawlResult, SearchResult, ParseResult, ExtractResult, BatchResult, Capabilities, DiffResult, Json, } from "./types.js";
|
package/dist/esm/index.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
export { CrwClient, CLOUD_API_URL, DASHBOARD_URL, DOCS_URL } from "./client.js";
|
|
2
|
-
export { CrwError, CrwApiError, CrwTimeoutError, CrwBinaryNotFoundError } from "./errors.js";
|
|
2
|
+
export { CrwError, CrwApiError, CrwTimeoutError, CrwExtractCancelledError, CrwBinaryNotFoundError, } from "./errors.js";
|
package/dist/esm/types.d.ts
CHANGED
|
@@ -116,6 +116,36 @@ export interface ExtractOptions {
|
|
|
116
116
|
pollInterval?: number;
|
|
117
117
|
timeout?: number;
|
|
118
118
|
}
|
|
119
|
+
export type ExtractJobState = "processing" | "cancelling" | "completed" | "failed" | "cancelled";
|
|
120
|
+
export type ExtractUrlState = "processing" | "completed" | "failed" | "cancelled";
|
|
121
|
+
/** Response from an async extract admission request. */
|
|
122
|
+
export interface ExtractAccepted {
|
|
123
|
+
id: string;
|
|
124
|
+
status: "processing";
|
|
125
|
+
/** Count of URLs enqueued for fetch. */
|
|
126
|
+
urls: number;
|
|
127
|
+
}
|
|
128
|
+
/** One fixed, ordered result slot for a requested URL. */
|
|
129
|
+
export interface ExtractUrlResult {
|
|
130
|
+
url: string;
|
|
131
|
+
status: ExtractUrlState;
|
|
132
|
+
data?: unknown;
|
|
133
|
+
error?: string;
|
|
134
|
+
llmUsage?: Json;
|
|
135
|
+
basis?: Basis[];
|
|
136
|
+
basisWarnings?: BasisWarning[];
|
|
137
|
+
llmInputHash?: string;
|
|
138
|
+
}
|
|
139
|
+
/** Canonical GET/DELETE extract lifecycle envelope. */
|
|
140
|
+
export interface ExtractStatus {
|
|
141
|
+
id: string;
|
|
142
|
+
status: ExtractJobState;
|
|
143
|
+
results: ExtractUrlResult[];
|
|
144
|
+
error?: string;
|
|
145
|
+
expiresAt: string;
|
|
146
|
+
creditsUsed: number;
|
|
147
|
+
tokensUsed: number;
|
|
148
|
+
}
|
|
119
149
|
export interface BatchScrapeOptions {
|
|
120
150
|
formats?: string[];
|
|
121
151
|
pollInterval?: number;
|
|
@@ -133,7 +163,7 @@ export type CrawlResult = Json[];
|
|
|
133
163
|
export type SearchResult = Json | Json[];
|
|
134
164
|
export type ParseResult = Json;
|
|
135
165
|
/** Native `/v1/extract` returns one result object per URL, in request order. */
|
|
136
|
-
export type ExtractResult =
|
|
166
|
+
export type ExtractResult = ExtractUrlResult[];
|
|
137
167
|
export type BatchResult = Json[];
|
|
138
168
|
export type Capabilities = Json;
|
|
139
169
|
export type DiffResult = Json;
|
package/package.json
CHANGED