crw-sdk 0.27.0 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/esm/index.d.ts +1 -1
- package/dist/esm/types.d.ts +335 -7
- package/package.json +6 -3
package/README.md
CHANGED
package/dist/esm/index.d.ts
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
export { CrwClient, CLOUD_API_URL, DASHBOARD_URL, DOCS_URL } from "./client.js";
|
|
2
2
|
export { CrwError, CrwApiError, CrwTimeoutError, CrwExtractCancelledError, CrwBinaryNotFoundError, } from "./errors.js";
|
|
3
|
-
export type { ClientOptions, ScrapeOptions, CrawlOptions, MapOptions, SearchOptions, ParseFileOptions, ExtractOptions, ExtractJobState, ExtractUrlState, ExtractAccepted, ExtractStatus, ExtractUrlResult, Basis, BasisWarning, EvidenceCitation, FieldStatus, BatchScrapeOptions, ChangeTrackingOptions, ScrapeResult, CrawlResult, SearchResult, ParseResult, ExtractResult, BatchResult, Capabilities, DiffResult, Json, } from "./types.js";
|
|
3
|
+
export type { ClientOptions, ScrapeOptions, CrawlOptions, MapOptions, SearchOptions, ParseFileOptions, ExtractOptions, ExtractJobState, ExtractUrlState, ExtractAccepted, ExtractStatus, ExtractUrlResult, Basis, BasisWarning, EvidenceCitation, FieldStatus, BatchScrapeOptions, ChangeTrackingOptions, ScrapeResult, ScrapeDocument, ScrapeMetadata, ScrapedImage, RenderDecision, Chunk, BlockOutcome, LlmUsage, CrawlResult, SearchResult, SearchResultItem, ImageResultItem, SearchCitation, GroupedSearchResults, SearchAnswer, SearchResponseData, V2Document, V2Metadata, ParseResult, ExtractResult, BatchResult, Capabilities, LlmCapabilities, FormatCapabilities, SearchCapabilities, ScreenshotCapabilities, RendererCapabilities, ExtractCapabilities, DocumentCapabilities, FileUploadCapabilities, Limits, DiffResult, ChangeStatus, ChangeTrackingResult, ChangeTrackingSnapshot, ChangeDiff, ChangeJudgment, MeaningfulChange, Json, } from "./types.js";
|
package/dist/esm/types.d.ts
CHANGED
|
@@ -158,15 +158,343 @@ export interface ChangeTrackingOptions {
|
|
|
158
158
|
prompt?: string;
|
|
159
159
|
[key: string]: unknown;
|
|
160
160
|
}
|
|
161
|
-
|
|
162
|
-
export
|
|
163
|
-
|
|
164
|
-
|
|
161
|
+
/** Token usage for an LLM-backed call (extract, summary, search answer). */
|
|
162
|
+
export interface LlmUsage {
|
|
163
|
+
inputTokens: number;
|
|
164
|
+
outputTokens: number;
|
|
165
|
+
totalTokens: number;
|
|
166
|
+
estimatedCostUsd?: number;
|
|
167
|
+
model: string;
|
|
168
|
+
provider: string;
|
|
169
|
+
cacheHitInputTokens?: number;
|
|
170
|
+
cacheMissInputTokens?: number;
|
|
171
|
+
truncated?: boolean;
|
|
172
|
+
calls?: number;
|
|
173
|
+
executedSummaries: number;
|
|
174
|
+
answerExecuted: boolean;
|
|
175
|
+
[key: string]: unknown;
|
|
176
|
+
}
|
|
177
|
+
/** A scraped image reference. */
|
|
178
|
+
export interface ScrapedImage {
|
|
179
|
+
url: string;
|
|
180
|
+
alt?: string;
|
|
181
|
+
}
|
|
182
|
+
/** One semantic chunk, present when the request set a `chunkStrategy`. */
|
|
183
|
+
export interface Chunk {
|
|
184
|
+
content: string;
|
|
185
|
+
index: number;
|
|
186
|
+
score?: number;
|
|
187
|
+
}
|
|
188
|
+
/** Anti-bot verdict, present only when the page was a block/challenge. */
|
|
189
|
+
export interface BlockOutcome {
|
|
190
|
+
vendor: string;
|
|
191
|
+
reason: string;
|
|
192
|
+
}
|
|
193
|
+
/** Which renderer tier produced the page, and why (`kind` is the discriminant). */
|
|
194
|
+
export interface RenderDecision {
|
|
195
|
+
kind: string;
|
|
196
|
+
[key: string]: unknown;
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Page metadata. `title`/`description` are always present as keys (may be
|
|
200
|
+
* `null`); every other field appears only when the engine could resolve it.
|
|
201
|
+
* Extra `<meta>` tags are flattened onto the object, so unknown keys can appear.
|
|
202
|
+
*/
|
|
203
|
+
export interface ScrapeMetadata {
|
|
204
|
+
/** Always present on a `/v1` scrape (may be `null`); absent on some parse results. */
|
|
205
|
+
title?: string | null;
|
|
206
|
+
description?: string | null;
|
|
207
|
+
ogTitle?: string;
|
|
208
|
+
ogDescription?: string;
|
|
209
|
+
ogImage?: string;
|
|
210
|
+
canonicalUrl?: string;
|
|
211
|
+
/** Canonical source URL (note the capitalized `URL`). */
|
|
212
|
+
sourceURL: string;
|
|
213
|
+
language?: string;
|
|
214
|
+
statusCode: number;
|
|
215
|
+
renderedWith?: string;
|
|
216
|
+
elapsedMs: number;
|
|
217
|
+
/** Page count for multi-page documents (e.g. PDFs). */
|
|
218
|
+
numPages?: number;
|
|
219
|
+
sourceFilename?: string;
|
|
220
|
+
[key: string]: unknown;
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* A scraped document. Which fields are present is gated by the request
|
|
224
|
+
* `formats`; `metadata` is always returned. Unknown keys are preserved so the
|
|
225
|
+
* type stays forward-compatible with newer engine fields.
|
|
226
|
+
*/
|
|
227
|
+
export interface ScrapeDocument {
|
|
228
|
+
markdown?: string;
|
|
229
|
+
html?: string;
|
|
230
|
+
rawHtml?: string;
|
|
231
|
+
/** Present when `formats` includes `plainText`. */
|
|
232
|
+
plainText?: string;
|
|
233
|
+
links?: string[];
|
|
234
|
+
/** Present when `formats` includes `images`. */
|
|
235
|
+
images?: ScrapedImage[];
|
|
236
|
+
/** Structured extraction, present when a `jsonSchema` was supplied. */
|
|
237
|
+
json?: Json;
|
|
238
|
+
/** Present when `formats` includes `summary`. */
|
|
239
|
+
summary?: string;
|
|
240
|
+
/** A `data:image/png;base64,…` URL; present when `formats` includes `screenshot`. */
|
|
241
|
+
screenshot?: string;
|
|
242
|
+
metadata: ScrapeMetadata;
|
|
243
|
+
contentType?: string;
|
|
244
|
+
/** `sha256:`-prefixed hash of the canonical markdown. */
|
|
245
|
+
sourceHash?: string;
|
|
246
|
+
/** Per-field extraction evidence, present when the request set `basis: true`. */
|
|
247
|
+
basis?: Basis[];
|
|
248
|
+
basisWarnings?: BasisWarning[];
|
|
249
|
+
llmInputHash?: string;
|
|
250
|
+
llmUsage?: LlmUsage;
|
|
251
|
+
chunks?: Chunk[];
|
|
252
|
+
warning?: string;
|
|
253
|
+
warnings?: string[];
|
|
254
|
+
renderDecision?: RenderDecision;
|
|
255
|
+
creditCost?: number;
|
|
256
|
+
/** Present when `formats` includes `changeTracking`. */
|
|
257
|
+
changeTracking?: ChangeTrackingResult;
|
|
258
|
+
/** Present only when the page was a block/challenge. */
|
|
259
|
+
block?: BlockOutcome;
|
|
260
|
+
truncated?: boolean;
|
|
261
|
+
[key: string]: unknown;
|
|
262
|
+
}
|
|
263
|
+
/** One search hit. Scrape fields appear only when `scrapeOptions` requested them. */
|
|
264
|
+
export interface SearchResultItem {
|
|
265
|
+
url: string;
|
|
266
|
+
title: string;
|
|
267
|
+
description: string;
|
|
268
|
+
/** Alias of `description` (Firecrawl parity). */
|
|
269
|
+
snippet: string;
|
|
270
|
+
position: number;
|
|
271
|
+
score?: number;
|
|
272
|
+
publishedDate?: string;
|
|
273
|
+
category?: string;
|
|
274
|
+
markdown?: string;
|
|
275
|
+
html?: string;
|
|
276
|
+
rawHtml?: string;
|
|
277
|
+
links?: string[];
|
|
278
|
+
metadata?: ScrapeMetadata;
|
|
279
|
+
summary?: string;
|
|
280
|
+
/** Enrichment-scrape error for this hit, when `scrapeOptions` failed. */
|
|
281
|
+
error?: string;
|
|
282
|
+
truncated?: boolean;
|
|
283
|
+
[key: string]: unknown;
|
|
284
|
+
}
|
|
285
|
+
/** One image-search hit. */
|
|
286
|
+
export interface ImageResultItem {
|
|
287
|
+
url: string;
|
|
288
|
+
title: string;
|
|
289
|
+
description: string;
|
|
290
|
+
imageUrl: string;
|
|
291
|
+
position: number;
|
|
292
|
+
thumbnailUrl?: string;
|
|
293
|
+
imageFormat?: string;
|
|
294
|
+
resolution?: string;
|
|
295
|
+
[key: string]: unknown;
|
|
296
|
+
}
|
|
297
|
+
/** A source citation backing a synthesized search `answer`. */
|
|
298
|
+
export interface SearchCitation {
|
|
299
|
+
url: string;
|
|
300
|
+
title: string;
|
|
301
|
+
position: number;
|
|
302
|
+
}
|
|
303
|
+
/** Results grouped by source, returned when the request set `sources`. */
|
|
304
|
+
export interface GroupedSearchResults {
|
|
305
|
+
web?: SearchResultItem[];
|
|
306
|
+
news?: SearchResultItem[];
|
|
307
|
+
images?: ImageResultItem[];
|
|
308
|
+
[key: string]: unknown;
|
|
309
|
+
}
|
|
310
|
+
/**
|
|
311
|
+
* Answer-mode siblings. On the managed API these ride alongside the results
|
|
312
|
+
* (attached to the returned array/object); `answer` is present only when the
|
|
313
|
+
* request enabled answer synthesis. Note: on the flat-array/grouped shapes the
|
|
314
|
+
* client attaches them non-enumerably, so `JSON.stringify(result)` omits them —
|
|
315
|
+
* read them off the result directly.
|
|
316
|
+
*/
|
|
317
|
+
export interface SearchAnswer {
|
|
318
|
+
answer?: string;
|
|
319
|
+
citations?: SearchCitation[];
|
|
320
|
+
llmUsage?: LlmUsage;
|
|
321
|
+
warnings?: string[];
|
|
322
|
+
}
|
|
323
|
+
/**
|
|
324
|
+
* Self-hosted search envelope: results plus answer-mode siblings in one object.
|
|
325
|
+
* A self-hosted server (custom `apiUrl`) returns this shape rather than the
|
|
326
|
+
* managed API's flat/grouped {@link SearchResult}; cast to it with `as` when you
|
|
327
|
+
* point the client at your own engine.
|
|
328
|
+
*/
|
|
329
|
+
export interface SearchResponseData extends SearchAnswer {
|
|
330
|
+
results: SearchResultItem[] | GroupedSearchResults;
|
|
331
|
+
[key: string]: unknown;
|
|
332
|
+
}
|
|
333
|
+
export type ChangeStatus = "same" | "changed";
|
|
334
|
+
export interface ChangeTrackingSnapshot {
|
|
335
|
+
markdown?: string;
|
|
336
|
+
json?: Json;
|
|
337
|
+
contentHash: string;
|
|
338
|
+
capturedAt?: string;
|
|
339
|
+
}
|
|
340
|
+
export interface ChangeDiff {
|
|
341
|
+
/** Unified diff text (gitDiff mode). */
|
|
342
|
+
text?: string;
|
|
343
|
+
/** Structured diff: a parse-diff AST, or a per-field `{ previous, current }` map. */
|
|
344
|
+
json?: Json;
|
|
345
|
+
}
|
|
346
|
+
export interface MeaningfulChange {
|
|
347
|
+
type: string;
|
|
348
|
+
before?: string;
|
|
349
|
+
after?: string;
|
|
350
|
+
reason: string;
|
|
351
|
+
}
|
|
352
|
+
export interface ChangeJudgment {
|
|
353
|
+
meaningful: boolean;
|
|
354
|
+
confidence: "low" | "medium" | "high";
|
|
355
|
+
reason: string;
|
|
356
|
+
meaningfulChanges: MeaningfulChange[];
|
|
357
|
+
}
|
|
358
|
+
export interface ChangeTrackingResult {
|
|
359
|
+
status: ChangeStatus;
|
|
360
|
+
firstObservation: boolean;
|
|
361
|
+
contentHash: string;
|
|
362
|
+
snapshot?: ChangeTrackingSnapshot;
|
|
363
|
+
diff?: ChangeDiff;
|
|
364
|
+
judgment?: ChangeJudgment;
|
|
365
|
+
tag?: string;
|
|
366
|
+
truncated?: boolean;
|
|
367
|
+
[key: string]: unknown;
|
|
368
|
+
}
|
|
369
|
+
export interface LlmCapabilities {
|
|
370
|
+
providers: string[];
|
|
371
|
+
supportsBaseUrl: boolean;
|
|
372
|
+
serverKeyConfigured: boolean;
|
|
373
|
+
maxConcurrency: number;
|
|
374
|
+
requireByokHeader?: string;
|
|
375
|
+
}
|
|
376
|
+
export interface FormatCapabilities {
|
|
377
|
+
supported: string[];
|
|
378
|
+
llmRequired: string[];
|
|
379
|
+
changeTrackingModes: string[];
|
|
380
|
+
changeTrackingModesLlmRequired: string[];
|
|
381
|
+
}
|
|
382
|
+
export interface SearchCapabilities {
|
|
383
|
+
supported: boolean;
|
|
384
|
+
answer: boolean;
|
|
385
|
+
summarizeResults: boolean;
|
|
386
|
+
}
|
|
387
|
+
export interface ScreenshotCapabilities {
|
|
388
|
+
supported: boolean;
|
|
389
|
+
fullPage: boolean;
|
|
390
|
+
}
|
|
391
|
+
export interface RendererCapabilities {
|
|
392
|
+
available: string[];
|
|
393
|
+
mode: string;
|
|
394
|
+
renderJsDefault?: boolean;
|
|
395
|
+
}
|
|
396
|
+
export interface ExtractCapabilities {
|
|
397
|
+
supported: boolean;
|
|
398
|
+
maxUrls: number;
|
|
399
|
+
perFieldAttribution: boolean;
|
|
400
|
+
maxOutputTokens: number;
|
|
401
|
+
}
|
|
402
|
+
export interface FileUploadCapabilities {
|
|
403
|
+
supported: boolean;
|
|
404
|
+
endpoint: string;
|
|
405
|
+
maxBytes: number;
|
|
406
|
+
types: string[];
|
|
407
|
+
ocr: boolean;
|
|
408
|
+
}
|
|
409
|
+
export interface DocumentCapabilities {
|
|
410
|
+
parsers: string[];
|
|
411
|
+
fileUpload: FileUploadCapabilities;
|
|
412
|
+
}
|
|
413
|
+
export interface Limits {
|
|
414
|
+
maxBatchUrls: number;
|
|
415
|
+
maxExtractUrls: number;
|
|
416
|
+
searchDefaultLimit: number;
|
|
417
|
+
searchMaxLimit: number;
|
|
418
|
+
maxUploadBytes: number;
|
|
419
|
+
}
|
|
420
|
+
/** Engine feature-detection payload (`GET /v1/capabilities`). */
|
|
421
|
+
export interface Capabilities {
|
|
422
|
+
version: string;
|
|
423
|
+
llm: LlmCapabilities;
|
|
424
|
+
formats: FormatCapabilities;
|
|
425
|
+
search: SearchCapabilities;
|
|
426
|
+
screenshot: ScreenshotCapabilities;
|
|
427
|
+
renderers: RendererCapabilities;
|
|
428
|
+
extract: ExtractCapabilities;
|
|
429
|
+
documents: DocumentCapabilities;
|
|
430
|
+
limits: Limits;
|
|
431
|
+
[key: string]: unknown;
|
|
432
|
+
}
|
|
433
|
+
/**
|
|
434
|
+
* Page metadata on the Firecrawl-compatible `/v2` surface (parse, batch scrape).
|
|
435
|
+
* Differs from {@link ScrapeMetadata}: no `elapsedMs`/`og*`, and it carries
|
|
436
|
+
* v2-synthesized fields (`url`, `proxyUsed`, `cacheState`, `creditsUsed`, …).
|
|
437
|
+
*/
|
|
438
|
+
export interface V2Metadata {
|
|
439
|
+
title?: string;
|
|
440
|
+
description?: string;
|
|
441
|
+
language?: string;
|
|
442
|
+
/** Canonical source URL (note the capitalized `URL`). */
|
|
443
|
+
sourceURL: string;
|
|
444
|
+
url: string;
|
|
445
|
+
statusCode: number;
|
|
446
|
+
contentType?: string;
|
|
447
|
+
proxyUsed: string;
|
|
448
|
+
cacheState: string;
|
|
449
|
+
concurrencyLimited: boolean;
|
|
450
|
+
creditsUsed: number;
|
|
451
|
+
scrapeId: string;
|
|
452
|
+
/** Page count for paginated documents (PDFs). */
|
|
453
|
+
numPages?: number;
|
|
454
|
+
sourceFilename?: string;
|
|
455
|
+
[key: string]: unknown;
|
|
456
|
+
}
|
|
457
|
+
/**
|
|
458
|
+
* A document from the Firecrawl-compatible `/v2` surface (parse, batch scrape).
|
|
459
|
+
* Unlike {@link ScrapeDocument}, `images` is a flat array of URL strings and the
|
|
460
|
+
* metadata is a {@link V2Metadata}.
|
|
461
|
+
*/
|
|
462
|
+
export interface V2Document {
|
|
463
|
+
markdown?: string;
|
|
464
|
+
html?: string;
|
|
465
|
+
rawHtml?: string;
|
|
466
|
+
links?: string[];
|
|
467
|
+
/** Flat array of image URLs (v2 flattens the native `{ url, alt }` objects). */
|
|
468
|
+
images?: string[];
|
|
469
|
+
json?: Json;
|
|
470
|
+
summary?: string;
|
|
471
|
+
changeTracking?: ChangeTrackingResult;
|
|
472
|
+
screenshot?: string;
|
|
473
|
+
warning?: string;
|
|
474
|
+
metadata: V2Metadata;
|
|
475
|
+
[key: string]: unknown;
|
|
476
|
+
}
|
|
477
|
+
/** A scraped document (`/v1/scrape`). */
|
|
478
|
+
export type ScrapeResult = ScrapeDocument;
|
|
479
|
+
/** Crawl (`/v1/crawl`) returns the scraped document for each page, in crawl order. */
|
|
480
|
+
export type CrawlResult = ScrapeDocument[];
|
|
481
|
+
/** Parse (`/v2/parse`) returns a v2 document (markdown/json/metadata). */
|
|
482
|
+
export type ParseResult = V2Document;
|
|
483
|
+
/** Batch scrape (`/v2/batch/scrape`) returns one v2 document per URL. */
|
|
484
|
+
export type BatchResult = V2Document[];
|
|
165
485
|
/** Native `/v1/extract` returns one result object per URL, in request order. */
|
|
166
486
|
export type ExtractResult = ExtractUrlResult[];
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
487
|
+
/**
|
|
488
|
+
* Search results on the managed API: a flat list of hits, or a
|
|
489
|
+
* `{ web, news, images }` grouping when the request set `sources`, with the
|
|
490
|
+
* answer-mode siblings ({@link SearchAnswer}) attached alongside. Use
|
|
491
|
+
* `Array.isArray(result)` to tell the flat and grouped shapes apart. A
|
|
492
|
+
* self-hosted server (custom `apiUrl`) instead returns a {@link SearchResponseData}
|
|
493
|
+
* envelope — cast to that type in self-host code.
|
|
494
|
+
*/
|
|
495
|
+
export type SearchResult = (SearchResultItem[] | GroupedSearchResults) & SearchAnswer;
|
|
496
|
+
/** Change-tracking diff (`/v1/change-tracking/diff`): the diff for the page. */
|
|
497
|
+
export type DiffResult = ChangeTrackingResult;
|
|
170
498
|
/** Firecrawl-compatible Research API options (cloud only). */
|
|
171
499
|
export interface ResearchSearchOptions {
|
|
172
500
|
k?: number;
|
package/package.json
CHANGED
|
@@ -1,19 +1,22 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "crw-sdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.28.0",
|
|
4
4
|
"description": "TypeScript/JavaScript SDK for CRW — scrape, crawl, map, search, parse, and extract any website",
|
|
5
5
|
"license": "MIT",
|
|
6
|
-
"
|
|
6
|
+
"author": "fastCRW <hello@fastcrw.com>",
|
|
7
|
+
"homepage": "https://fastcrw.com",
|
|
7
8
|
"repository": {
|
|
8
9
|
"type": "git",
|
|
9
10
|
"url": "https://github.com/us/crw.git"
|
|
10
11
|
},
|
|
12
|
+
"bugs": {
|
|
13
|
+
"url": "https://github.com/us/crw/issues"
|
|
14
|
+
},
|
|
11
15
|
"keywords": [
|
|
12
16
|
"web-scraping",
|
|
13
17
|
"crawler",
|
|
14
18
|
"scraper",
|
|
15
19
|
"ai-agent",
|
|
16
|
-
"firecrawl",
|
|
17
20
|
"mcp"
|
|
18
21
|
],
|
|
19
22
|
"type": "module",
|