@mendable/firecrawl-js 4.39.0 → 4.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -0
- package/dist/{chunk-4DPCCKWG.js → chunk-OJ4YJQ4J.js} +1 -1
- package/dist/index.cjs +166 -9
- package/dist/index.d.cts +138 -4
- package/dist/index.d.ts +138 -4
- package/dist/index.js +165 -9
- package/dist/{package-D2QZMRW7.js → package-F6O7J2XI.js} +1 -1
- package/package.json +1 -1
- package/pnpm-workspace.yaml +2 -2
- package/src/__tests__/unit/v2/threat-protection.unit.test.ts +18 -1
- package/src/__tests__/unit/v2/tools.unit.test.ts +240 -0
- package/src/v2/client.ts +39 -3
- package/src/v2/methods/scrape.ts +3 -1
- package/src/v2/methods/search.ts +7 -1
- package/src/v2/methods/tools.ts +137 -0
- package/src/v2/types.ts +156 -1
- package/src/v2/utils/errorHandler.ts +19 -3
package/dist/index.d.ts
CHANGED
|
@@ -200,6 +200,8 @@ interface ScrapeOptions {
|
|
|
200
200
|
};
|
|
201
201
|
integration?: string;
|
|
202
202
|
origin?: string;
|
|
203
|
+
/** Include domain-matched Alexandria contracts for this URL in `tools`. Default off. */
|
|
204
|
+
domainTools?: boolean;
|
|
203
205
|
}
|
|
204
206
|
type RedactPIIEntity = "PERSON" | "EMAIL" | "PHONE" | "LOCATION" | "FINANCIAL" | "SECRET";
|
|
205
207
|
interface RedactPIIOptions {
|
|
@@ -549,6 +551,8 @@ interface Document {
|
|
|
549
551
|
pages?: PdfPage[];
|
|
550
552
|
/** Typed PDF layout blocks, present only when `parsers[].blocks` is true. */
|
|
551
553
|
blocks?: PdfPageBlocks[];
|
|
554
|
+
/** Present when `domainTools` discovered contracts matching this URL. */
|
|
555
|
+
tools?: DiscoveredTool[];
|
|
552
556
|
}
|
|
553
557
|
interface PaginationConfig {
|
|
554
558
|
/** When true (default), automatically follow `next` links and aggregate all documents. */
|
|
@@ -650,9 +654,121 @@ interface SearchResultImages {
|
|
|
650
654
|
position?: number;
|
|
651
655
|
}
|
|
652
656
|
interface SearchData {
|
|
657
|
+
warning?: string;
|
|
653
658
|
web?: Array<SearchResultWeb | Document>;
|
|
654
659
|
news?: Array<SearchResultNews | Document>;
|
|
655
660
|
images?: Array<SearchResultImages | Document>;
|
|
661
|
+
tools?: DiscoveredTool[];
|
|
662
|
+
}
|
|
663
|
+
/** A complete tool contract returned by semantic or contextual discovery. */
|
|
664
|
+
interface DiscoveredTool {
|
|
665
|
+
id?: string;
|
|
666
|
+
provider: string;
|
|
667
|
+
capability: string;
|
|
668
|
+
name: string;
|
|
669
|
+
description: string;
|
|
670
|
+
creditsCost: number;
|
|
671
|
+
perRecord: boolean;
|
|
672
|
+
options: Array<{
|
|
673
|
+
name: string;
|
|
674
|
+
type: string;
|
|
675
|
+
required?: boolean;
|
|
676
|
+
[key: string]: unknown;
|
|
677
|
+
}>;
|
|
678
|
+
requiresOneOf?: string[][];
|
|
679
|
+
response?: {
|
|
680
|
+
about?: string;
|
|
681
|
+
key?: string;
|
|
682
|
+
fields?: Array<{
|
|
683
|
+
name: string;
|
|
684
|
+
type: string;
|
|
685
|
+
[key: string]: unknown;
|
|
686
|
+
}>;
|
|
687
|
+
[key: string]: unknown;
|
|
688
|
+
};
|
|
689
|
+
examples?: Partial<Record<"javascript" | "python" | "curl", string>> & Record<string, string>;
|
|
690
|
+
example?: {
|
|
691
|
+
recordedAt: string;
|
|
692
|
+
request: Record<string, unknown>;
|
|
693
|
+
response: unknown;
|
|
694
|
+
};
|
|
695
|
+
label?: string;
|
|
696
|
+
whenToUse?: string;
|
|
697
|
+
returns?: unknown;
|
|
698
|
+
discovery?: unknown;
|
|
699
|
+
attribution?: unknown;
|
|
700
|
+
matchedBy?: Array<"semantic" | "domain">;
|
|
701
|
+
matchedUrls?: string[];
|
|
702
|
+
concept?: string;
|
|
703
|
+
cohorts?: string[];
|
|
704
|
+
similarity?: number;
|
|
705
|
+
}
|
|
706
|
+
interface FindToolsOptions {
|
|
707
|
+
urls?: string[];
|
|
708
|
+
providers?: string[];
|
|
709
|
+
categories?: string[];
|
|
710
|
+
groups?: string[];
|
|
711
|
+
capabilities?: string[];
|
|
712
|
+
level?: "providers" | "groups" | "tools";
|
|
713
|
+
expand?: Array<"options" | "response" | "examples">;
|
|
714
|
+
limit?: number;
|
|
715
|
+
offset?: number;
|
|
716
|
+
}
|
|
717
|
+
interface FindToolsData {
|
|
718
|
+
level: "providers" | "groups" | "tools";
|
|
719
|
+
items: Array<{
|
|
720
|
+
id: string;
|
|
721
|
+
name: string;
|
|
722
|
+
next?: AlexandriaCall;
|
|
723
|
+
execute?: Pick<AlexandriaCall, "provider" | "capability">;
|
|
724
|
+
[key: string]: unknown;
|
|
725
|
+
}>;
|
|
726
|
+
total: number;
|
|
727
|
+
next: AlexandriaCall | null;
|
|
728
|
+
}
|
|
729
|
+
interface AlexandriaScrapeRequest extends AlexandriaOptions {
|
|
730
|
+
alexandria: AlexandriaCall | AlexandriaCall[];
|
|
731
|
+
}
|
|
732
|
+
interface AlexandriaCall {
|
|
733
|
+
provider: string;
|
|
734
|
+
capability: string;
|
|
735
|
+
options?: Record<string, unknown>;
|
|
736
|
+
}
|
|
737
|
+
interface AlexandriaScrapeError {
|
|
738
|
+
code: string;
|
|
739
|
+
message: string;
|
|
740
|
+
status?: number;
|
|
741
|
+
/** Present when credits were captured before the failure. */
|
|
742
|
+
chargeId?: string;
|
|
743
|
+
}
|
|
744
|
+
type AlexandriaScrapeResult = {
|
|
745
|
+
provider: string;
|
|
746
|
+
capability: string;
|
|
747
|
+
creditsCost: number;
|
|
748
|
+
data: unknown;
|
|
749
|
+
records?: number;
|
|
750
|
+
upstreamStatus?: number;
|
|
751
|
+
recordedAt?: string;
|
|
752
|
+
error?: undefined;
|
|
753
|
+
[key: string]: unknown;
|
|
754
|
+
} | {
|
|
755
|
+
provider?: string;
|
|
756
|
+
capability?: string;
|
|
757
|
+
error: AlexandriaScrapeError;
|
|
758
|
+
[key: string]: unknown;
|
|
759
|
+
};
|
|
760
|
+
interface AlexandriaScrapeData {
|
|
761
|
+
scrapeId: string;
|
|
762
|
+
requestId: string;
|
|
763
|
+
alexandria: AlexandriaScrapeResult[];
|
|
764
|
+
creditsCost: number;
|
|
765
|
+
}
|
|
766
|
+
interface AlexandriaOptions {
|
|
767
|
+
/** Reuse this ID with the identical payload when retrying an execution. */
|
|
768
|
+
requestId?: string;
|
|
769
|
+
timeout?: number;
|
|
770
|
+
integration?: string;
|
|
771
|
+
origin?: string;
|
|
656
772
|
}
|
|
657
773
|
/**
|
|
658
774
|
* Narrows ordinary **web search**. A category does not switch `search()` to a
|
|
@@ -681,8 +797,10 @@ interface CategoryOption {
|
|
|
681
797
|
}
|
|
682
798
|
interface SearchRequest {
|
|
683
799
|
query: string;
|
|
684
|
-
|
|
685
|
-
|
|
800
|
+
/** Include domain-matched contracts in tools alongside semantic matches. */
|
|
801
|
+
domainTools?: boolean;
|
|
802
|
+
sources?: Array<"web" | "news" | "images" | "alexandria" | {
|
|
803
|
+
type: "web" | "news" | "images" | "alexandria";
|
|
686
804
|
}>;
|
|
687
805
|
/**
|
|
688
806
|
* Narrow web search by category. See {@link CategoryOption}.
|
|
@@ -1441,12 +1559,25 @@ interface ErrorDetails {
|
|
|
1441
1559
|
details?: Record<string, unknown>;
|
|
1442
1560
|
status?: number;
|
|
1443
1561
|
}
|
|
1562
|
+
/** An out-of-band step the API requires before the request can succeed. */
|
|
1563
|
+
interface RequiresAction {
|
|
1564
|
+
type: "accept_terms" | (string & {});
|
|
1565
|
+
terms?: string;
|
|
1566
|
+
version?: string;
|
|
1567
|
+
url?: string;
|
|
1568
|
+
}
|
|
1569
|
+
declare function parseRequiresAction(value: unknown): RequiresAction | undefined;
|
|
1444
1570
|
declare class SdkError extends Error {
|
|
1571
|
+
requestId?: string;
|
|
1445
1572
|
status?: number;
|
|
1446
1573
|
code?: string;
|
|
1447
1574
|
details?: unknown;
|
|
1448
1575
|
jobId?: string;
|
|
1449
|
-
|
|
1576
|
+
/** Present on exchange-mediated scrape failures that already captured credits. */
|
|
1577
|
+
chargeId?: string;
|
|
1578
|
+
/** Present when the API needs an out-of-band step first, such as accepting provider terms. */
|
|
1579
|
+
requiresAction?: RequiresAction;
|
|
1580
|
+
constructor(message: string, status?: number, code?: string, details?: unknown, jobId?: string, chargeId?: string, requiresAction?: RequiresAction);
|
|
1450
1581
|
}
|
|
1451
1582
|
declare class JobTimeoutError extends SdkError {
|
|
1452
1583
|
timeoutSeconds: number;
|
|
@@ -1972,6 +2103,9 @@ declare class FirecrawlClient {
|
|
|
1972
2103
|
json?: InferredJsonFromOptions<Opts>;
|
|
1973
2104
|
}>;
|
|
1974
2105
|
scrape(url: string, options?: ScrapeCallOptions): Promise<Document>;
|
|
2106
|
+
scrape(request: AlexandriaScrapeRequest): Promise<AlexandriaScrapeData>;
|
|
2107
|
+
/** Explore the catalogue without executing the tools it returns. */
|
|
2108
|
+
findTools(options?: FindToolsOptions): Promise<FindToolsData>;
|
|
1975
2109
|
/**
|
|
1976
2110
|
* Interact with the browser session associated with a scrape job.
|
|
1977
2111
|
* @param jobId Scrape job id.
|
|
@@ -3255,4 +3389,4 @@ declare class Firecrawl extends FirecrawlClient {
|
|
|
3255
3389
|
get v1(): FirecrawlApp;
|
|
3256
3390
|
}
|
|
3257
3391
|
|
|
3258
|
-
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentEffort, type AgentExchangeOptions, type AgentExchangeSummary, type AgentListItem, type AgentListOptions, type AgentListResponse, type AgentMode, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentSnapshotResponse, type AgentStatusResponse, type AgentSuggestion, type AgentThread, type AgentThreadResponse, type AgentThreadRun, type AgentTraceActiveBrowserSession, type AgentTraceAgentFinishedEvent, type AgentTraceAgentIdentity, type AgentTraceAgentRole, type AgentTraceAgentStartedEvent, type AgentTraceArtifactChange, type AgentTraceArtifactUpdatedEvent, type AgentTraceBrowserSessionFinishedEvent, type AgentTraceBrowserSessionStartedEvent, type AgentTraceError, type AgentTraceErrorOccurredEvent, type AgentTraceEvent, type AgentTraceProgressReportedEvent, type AgentTraceReasoningSummaryEvent, type AgentTraceResponse, type AgentTraceRunCancelRequestedEvent, type AgentTraceRunFinishedEvent, type AgentTraceRunStartedEvent, type AgentTraceToolCallFinishedEvent, type AgentTraceToolCallStartedEvent, type AgentWebhookConfig, type AgentWebhookEvent, type AttributesFormat, type AuditMetadata$1 as AuditMetadata, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type DeveloperSearchLicenseDisclosure, type DeveloperSearchOptions, type DeveloperSearchPassage, type DeveloperSearchRepoStatus, type DeveloperSearchResponse, type DeveloperSearchResult, type DeveloperSearchSourceStatus, type DeveloperSearchType, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PDFParser, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PdfBlockConfidence, type PdfBlockItem, type PdfPage, type PdfPageBlocks, type PendingApproval, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type ThreatProtectionOptions, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default };
|
|
3392
|
+
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentEffort, type AgentExchangeOptions, type AgentExchangeSummary, type AgentListItem, type AgentListOptions, type AgentListResponse, type AgentMode, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentSnapshotResponse, type AgentStatusResponse, type AgentSuggestion, type AgentThread, type AgentThreadResponse, type AgentThreadRun, type AgentTraceActiveBrowserSession, type AgentTraceAgentFinishedEvent, type AgentTraceAgentIdentity, type AgentTraceAgentRole, type AgentTraceAgentStartedEvent, type AgentTraceArtifactChange, type AgentTraceArtifactUpdatedEvent, type AgentTraceBrowserSessionFinishedEvent, type AgentTraceBrowserSessionStartedEvent, type AgentTraceError, type AgentTraceErrorOccurredEvent, type AgentTraceEvent, type AgentTraceProgressReportedEvent, type AgentTraceReasoningSummaryEvent, type AgentTraceResponse, type AgentTraceRunCancelRequestedEvent, type AgentTraceRunFinishedEvent, type AgentTraceRunStartedEvent, type AgentTraceToolCallFinishedEvent, type AgentTraceToolCallStartedEvent, type AgentWebhookConfig, type AgentWebhookEvent, type AlexandriaCall, type AlexandriaOptions, type AlexandriaScrapeData, type AlexandriaScrapeError, type AlexandriaScrapeRequest, type AlexandriaScrapeResult, type AttributesFormat, type AuditMetadata$1 as AuditMetadata, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type DeveloperSearchLicenseDisclosure, type DeveloperSearchOptions, type DeveloperSearchPassage, type DeveloperSearchRepoStatus, type DeveloperSearchResponse, type DeveloperSearchResult, type DeveloperSearchSourceStatus, type DeveloperSearchType, type DiscoveredTool, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, type FindToolsData, type FindToolsOptions, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PDFParser, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PdfBlockConfidence, type PdfBlockItem, type PdfPage, type PdfPageBlocks, type PendingApproval, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, type RequiresAction, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type ThreatProtectionOptions, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default, parseRequiresAction };
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
require_package
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-OJ4YJQ4J.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -138,18 +138,35 @@ var HttpClient = class {
|
|
|
138
138
|
};
|
|
139
139
|
|
|
140
140
|
// src/v2/types.ts
|
|
141
|
+
function parseRequiresAction(value) {
|
|
142
|
+
if (!value || typeof value !== "object") return void 0;
|
|
143
|
+
const record = value;
|
|
144
|
+
if (typeof record.type !== "string") return void 0;
|
|
145
|
+
const action = { type: record.type };
|
|
146
|
+
if (typeof record.terms === "string") action.terms = record.terms;
|
|
147
|
+
if (typeof record.version === "string") action.version = record.version;
|
|
148
|
+
if (typeof record.url === "string") action.url = record.url;
|
|
149
|
+
return action;
|
|
150
|
+
}
|
|
141
151
|
var SdkError = class extends Error {
|
|
152
|
+
requestId;
|
|
142
153
|
status;
|
|
143
154
|
code;
|
|
144
155
|
details;
|
|
145
156
|
jobId;
|
|
146
|
-
|
|
157
|
+
/** Present on exchange-mediated scrape failures that already captured credits. */
|
|
158
|
+
chargeId;
|
|
159
|
+
/** Present when the API needs an out-of-band step first, such as accepting provider terms. */
|
|
160
|
+
requiresAction;
|
|
161
|
+
constructor(message, status, code, details, jobId, chargeId, requiresAction) {
|
|
147
162
|
super(message);
|
|
148
163
|
this.name = "FirecrawlSdkError";
|
|
149
164
|
this.status = status;
|
|
150
165
|
this.code = code;
|
|
151
166
|
this.details = details;
|
|
152
167
|
this.jobId = jobId;
|
|
168
|
+
this.chargeId = chargeId;
|
|
169
|
+
this.requiresAction = requiresAction;
|
|
153
170
|
}
|
|
154
171
|
};
|
|
155
172
|
var JobTimeoutError = class extends SdkError {
|
|
@@ -401,14 +418,30 @@ function throwForBadResponse(resp, action) {
|
|
|
401
418
|
const status = resp.status;
|
|
402
419
|
const body = resp.data || {};
|
|
403
420
|
const msg = body?.error || body?.message || `Request failed (${status}) while trying to ${action}`;
|
|
404
|
-
throw new SdkError(
|
|
421
|
+
throw new SdkError(
|
|
422
|
+
msg,
|
|
423
|
+
status,
|
|
424
|
+
body?.code,
|
|
425
|
+
body?.details,
|
|
426
|
+
void 0,
|
|
427
|
+
body?.chargeId,
|
|
428
|
+
parseRequiresAction(body?.requiresAction)
|
|
429
|
+
);
|
|
405
430
|
}
|
|
406
431
|
function normalizeAxiosError(err, action) {
|
|
407
432
|
const status = err.response?.status;
|
|
408
433
|
const body = err.response?.data;
|
|
409
434
|
const message = body?.error || err.message || `Request failed${status ? ` (${status})` : ""} while trying to ${action}`;
|
|
410
435
|
const code = body?.code || err.code;
|
|
411
|
-
throw new SdkError(
|
|
436
|
+
throw new SdkError(
|
|
437
|
+
message,
|
|
438
|
+
status,
|
|
439
|
+
code,
|
|
440
|
+
body?.details ?? body,
|
|
441
|
+
void 0,
|
|
442
|
+
body?.chargeId,
|
|
443
|
+
parseRequiresAction(body?.requiresAction)
|
|
444
|
+
);
|
|
412
445
|
}
|
|
413
446
|
function isRetryableError(err) {
|
|
414
447
|
if (err instanceof JobTimeoutError) {
|
|
@@ -454,9 +487,10 @@ async function scrape(http, url, options, deps = {}) {
|
|
|
454
487
|
throw new Error("URL cannot be empty");
|
|
455
488
|
}
|
|
456
489
|
if (options) ensureValidScrapeOptions(options);
|
|
457
|
-
const { autoResume, ...requestOptions } = options ?? {};
|
|
490
|
+
const { autoResume, domainTools, ...requestOptions } = options ?? {};
|
|
458
491
|
const payload = { url: url.trim() };
|
|
459
492
|
Object.assign(payload, requestOptions);
|
|
493
|
+
if (domainTools) payload.domainTools = domainTools;
|
|
460
494
|
const requestTimeout = autoResume === false ? typeof options?.timeout === "number" ? { timeoutMs: options.timeout + 5e3 } : {} : {
|
|
461
495
|
timeoutMs: Math.max(
|
|
462
496
|
typeof options?.timeout === "number" ? options.timeout : 3e5,
|
|
@@ -613,6 +647,7 @@ function prepareSearchPayload(req) {
|
|
|
613
647
|
const payload = {
|
|
614
648
|
query: req.query
|
|
615
649
|
};
|
|
650
|
+
if (req.domainTools != null) payload.domainTools = req.domainTools;
|
|
616
651
|
if (req.sources) payload.sources = req.sources;
|
|
617
652
|
if (req.categories) payload.categories = req.categories;
|
|
618
653
|
if (req.includeDomains) payload.includeDomains = req.includeDomains;
|
|
@@ -665,10 +700,12 @@ async function search(http, request) {
|
|
|
665
700
|
}
|
|
666
701
|
const data = res.data.data || {};
|
|
667
702
|
const out = {};
|
|
703
|
+
if (res.data.warning) out.warning = res.data.warning;
|
|
668
704
|
if (data.web) out.web = transformArray(data.web);
|
|
669
705
|
if (data.news) out.news = transformArray(data.news);
|
|
670
706
|
if (data.images)
|
|
671
707
|
out.images = transformArray(data.images);
|
|
708
|
+
if (data.tools) out.tools = data.tools;
|
|
672
709
|
Object.defineProperty(out, "data", {
|
|
673
710
|
get() {
|
|
674
711
|
const parts = [];
|
|
@@ -676,7 +713,9 @@ async function search(http, request) {
|
|
|
676
713
|
if (out.news?.length) parts.push(`.news (${out.news.length} results)`);
|
|
677
714
|
if (out.images?.length)
|
|
678
715
|
parts.push(`.images (${out.images.length} results)`);
|
|
679
|
-
|
|
716
|
+
if (out.tools?.length)
|
|
717
|
+
parts.push(`.tools (${out.tools.length} results)`);
|
|
718
|
+
const available = parts.length ? parts.join(", ") : ".web, .news, .images, or .tools";
|
|
680
719
|
throw new Error(
|
|
681
720
|
`SearchData has no '.data'. Results are grouped by source: ${available}`
|
|
682
721
|
);
|
|
@@ -691,6 +730,101 @@ async function search(http, request) {
|
|
|
691
730
|
}
|
|
692
731
|
}
|
|
693
732
|
|
|
733
|
+
// src/v2/methods/tools.ts
|
|
734
|
+
var ALEXANDRIA_MAX_CALLS = 10;
|
|
735
|
+
function prepareAlexandriaPayload(calls, opts) {
|
|
736
|
+
if (!Array.isArray(calls) || calls.length === 0) {
|
|
737
|
+
throw new Error("alexandria requires at least one call");
|
|
738
|
+
}
|
|
739
|
+
if (calls.length > ALEXANDRIA_MAX_CALLS) {
|
|
740
|
+
throw new Error(`alexandria accepts at most ${ALEXANDRIA_MAX_CALLS} calls`);
|
|
741
|
+
}
|
|
742
|
+
const alexandria = calls.map((call, index) => {
|
|
743
|
+
if (!call || typeof call.provider !== "string" || !call.provider.trim()) {
|
|
744
|
+
throw new Error(`alexandria[${index}].provider cannot be empty`);
|
|
745
|
+
}
|
|
746
|
+
if (typeof call.capability !== "string" || !call.capability.trim()) {
|
|
747
|
+
throw new Error(`alexandria[${index}].capability cannot be empty`);
|
|
748
|
+
}
|
|
749
|
+
if (Object.keys(call).some(
|
|
750
|
+
(key) => !["provider", "capability", "options"].includes(key)
|
|
751
|
+
))
|
|
752
|
+
throw new Error("Unknown alexandria call option");
|
|
753
|
+
const item = {
|
|
754
|
+
provider: call.provider.trim(),
|
|
755
|
+
capability: call.capability.trim()
|
|
756
|
+
};
|
|
757
|
+
if (call.options != null) {
|
|
758
|
+
if (typeof call.options !== "object" || Array.isArray(call.options)) {
|
|
759
|
+
throw new Error(`alexandria[${index}].options must be an object`);
|
|
760
|
+
}
|
|
761
|
+
item.options = call.options;
|
|
762
|
+
}
|
|
763
|
+
return item;
|
|
764
|
+
});
|
|
765
|
+
if (opts.timeout != null && (!Number.isInteger(opts.timeout) || opts.timeout <= 0)) {
|
|
766
|
+
throw new Error("timeout must be a positive integer");
|
|
767
|
+
}
|
|
768
|
+
const payload = { alexandria };
|
|
769
|
+
if (opts.timeout != null) payload.timeout = opts.timeout;
|
|
770
|
+
if (opts.integration && opts.integration.trim())
|
|
771
|
+
payload.integration = opts.integration.trim();
|
|
772
|
+
if (opts.origin) payload.origin = opts.origin;
|
|
773
|
+
return payload;
|
|
774
|
+
}
|
|
775
|
+
async function scrapeAlexandria(http, calls, opts = {}) {
|
|
776
|
+
const payload = prepareAlexandriaPayload(calls, opts);
|
|
777
|
+
const requestId = opts.requestId ?? crypto.randomUUID();
|
|
778
|
+
if (!/^[A-Za-z0-9._:-]{1,128}$/.test(requestId))
|
|
779
|
+
throw new Error("Invalid requestId");
|
|
780
|
+
try {
|
|
781
|
+
const res = await http.post("/v2/scrape", payload, {
|
|
782
|
+
headers: { "x-request-id": requestId },
|
|
783
|
+
// Allow response delivery after the API's capped execution deadline.
|
|
784
|
+
timeoutMs: Math.min(opts.timeout ?? 5e4, 5e4) + 3e4
|
|
785
|
+
});
|
|
786
|
+
if (res.status !== 200 || !res.data?.success) {
|
|
787
|
+
throwForBadResponse(res, "alexandria");
|
|
788
|
+
}
|
|
789
|
+
const data = res.data.data;
|
|
790
|
+
if (!data || !Array.isArray(data.alexandria) || typeof data.creditsCost !== "number" || !Number.isInteger(data.creditsCost) || data.creditsCost < 0) {
|
|
791
|
+
throw new SdkError("Invalid alexandria response");
|
|
792
|
+
}
|
|
793
|
+
return {
|
|
794
|
+
scrapeId: res.data.scrape_id ?? "",
|
|
795
|
+
requestId,
|
|
796
|
+
alexandria: data.alexandria,
|
|
797
|
+
creditsCost: data.creditsCost
|
|
798
|
+
};
|
|
799
|
+
} catch (err) {
|
|
800
|
+
try {
|
|
801
|
+
if (err?.isAxiosError) normalizeAxiosError(err, "alexandria");
|
|
802
|
+
throw err;
|
|
803
|
+
} catch (error) {
|
|
804
|
+
if (error && typeof error === "object")
|
|
805
|
+
Object.assign(error, { requestId });
|
|
806
|
+
throw error;
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
async function findTools(http, options = {}) {
|
|
811
|
+
const result = await scrapeAlexandria(http, [
|
|
812
|
+
{
|
|
813
|
+
provider: "firecrawl",
|
|
814
|
+
capability: "find-tools",
|
|
815
|
+
options: { ...options }
|
|
816
|
+
}
|
|
817
|
+
]);
|
|
818
|
+
const item = result.alexandria[0];
|
|
819
|
+
if (!item) throw new SdkError("Missing Find Tools result");
|
|
820
|
+
if (item.error)
|
|
821
|
+
throw Object.assign(
|
|
822
|
+
new SdkError(item.error.message, item.error.status, item.error.code),
|
|
823
|
+
{ requestId: result.requestId }
|
|
824
|
+
);
|
|
825
|
+
return item.data;
|
|
826
|
+
}
|
|
827
|
+
|
|
694
828
|
// src/v2/methods/developer.ts
|
|
695
829
|
var ENDPOINT = "/v2/search/developer";
|
|
696
830
|
async function developerSearch(http, query, options = {}) {
|
|
@@ -2018,7 +2152,28 @@ var FirecrawlClient = class {
|
|
|
2018
2152
|
});
|
|
2019
2153
|
}
|
|
2020
2154
|
async scrape(url, options) {
|
|
2021
|
-
return scrape(this.http, url, options);
|
|
2155
|
+
if (typeof url === "string") return scrape(this.http, url, options);
|
|
2156
|
+
if (!url || options !== void 0 || Object.keys(url).some(
|
|
2157
|
+
(key) => ![
|
|
2158
|
+
"alexandria",
|
|
2159
|
+
"requestId",
|
|
2160
|
+
"timeout",
|
|
2161
|
+
"integration",
|
|
2162
|
+
"origin"
|
|
2163
|
+
].includes(key)
|
|
2164
|
+
)) {
|
|
2165
|
+
throw new Error("Provide an alexandria request without URL scrape options");
|
|
2166
|
+
}
|
|
2167
|
+
const { alexandria, ...opts } = url;
|
|
2168
|
+
return scrapeAlexandria(
|
|
2169
|
+
this.http,
|
|
2170
|
+
Array.isArray(alexandria) ? alexandria : [alexandria],
|
|
2171
|
+
opts
|
|
2172
|
+
);
|
|
2173
|
+
}
|
|
2174
|
+
/** Explore the catalogue without executing the tools it returns. */
|
|
2175
|
+
async findTools(options) {
|
|
2176
|
+
return findTools(this.http, options);
|
|
2022
2177
|
}
|
|
2023
2178
|
/**
|
|
2024
2179
|
* Interact with the browser session associated with a scrape job.
|
|
@@ -2510,7 +2665,7 @@ var FirecrawlApp = class {
|
|
|
2510
2665
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2511
2666
|
return process.env.npm_package_version;
|
|
2512
2667
|
}
|
|
2513
|
-
const packageJson = await import("./package-
|
|
2668
|
+
const packageJson = await import("./package-F6O7J2XI.js");
|
|
2514
2669
|
return packageJson.default.version;
|
|
2515
2670
|
} catch (error) {
|
|
2516
2671
|
const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
|
|
@@ -3910,5 +4065,6 @@ export {
|
|
|
3910
4065
|
ResearchClient,
|
|
3911
4066
|
SdkError,
|
|
3912
4067
|
Watcher,
|
|
3913
|
-
index_default as default
|
|
4068
|
+
index_default as default,
|
|
4069
|
+
parseRequiresAction
|
|
3914
4070
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mendable/firecrawl-js",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.40.0",
|
|
4
4
|
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
package/pnpm-workspace.yaml
CHANGED
|
@@ -8,8 +8,8 @@ overrides:
|
|
|
8
8
|
"picomatch@<4.0.4": ">=4.0.4"
|
|
9
9
|
handlebars: ">=4.7.9"
|
|
10
10
|
"brace-expansion@>=5.0.0 <5.0.9": "5.0.9"
|
|
11
|
-
"js-yaml@>=3.0.0 <3.15.
|
|
12
|
-
"js-yaml@>=4.0.0 <4.3.
|
|
11
|
+
"js-yaml@>=3.0.0 <3.15.2": "3.15.2"
|
|
12
|
+
"js-yaml@>=4.0.0 <4.3.2": "4.3.2"
|
|
13
13
|
"axios@>=1.0.0 <1.16.0": "1.16.1"
|
|
14
14
|
"follow-redirects@<1.16.0": ">=1.16.0 <2.0.0"
|
|
15
15
|
"form-data@>=4.0.0 <4.0.6": "4.0.6"
|
|
@@ -32,7 +32,17 @@ describe("v2 threatProtection request serialization", () => {
|
|
|
32
32
|
expect(http.post).toHaveBeenCalledWith(
|
|
33
33
|
"/v2/scrape",
|
|
34
34
|
expect.objectContaining({ url: "https://example.com", threatProtection }),
|
|
35
|
-
{},
|
|
35
|
+
{ timeoutMs: 330000 },
|
|
36
|
+
);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test("scrape sends domainTools when true", async () => {
|
|
40
|
+
const http = makeHttp({ success: true, data: {} });
|
|
41
|
+
await scrape(http, "https://example.com", { domainTools: true });
|
|
42
|
+
expect(http.post).toHaveBeenCalledWith(
|
|
43
|
+
"/v2/scrape",
|
|
44
|
+
expect.objectContaining({ url: "https://example.com", domainTools: true }),
|
|
45
|
+
expect.anything(),
|
|
36
46
|
);
|
|
37
47
|
});
|
|
38
48
|
|
|
@@ -121,3 +131,10 @@ describe("v2 threatProtection request serialization", () => {
|
|
|
121
131
|
expect(body).not.toHaveProperty("threatProtection");
|
|
122
132
|
});
|
|
123
133
|
});
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
test.each([false, undefined])("scrape omits domainTools when %s", async (domainTools) => {
|
|
137
|
+
const http = makeHttp({ success: true, data: {} });
|
|
138
|
+
await scrape(http, "https://example.com", { domainTools });
|
|
139
|
+
expect(http.post.mock.calls[0][1]).not.toHaveProperty("domainTools");
|
|
140
|
+
});
|