@dealcrawl/sdk 2.9.0 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +318 -53
- package/dist/index.d.mts +1151 -25
- package/dist/index.d.ts +1151 -25
- package/dist/index.js +631 -9
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +628 -10
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -19,7 +19,7 @@ interface ResponseMeta {
|
|
|
19
19
|
timestamp: string;
|
|
20
20
|
duration?: number;
|
|
21
21
|
}
|
|
22
|
-
type JobStatus = "pending" | "active" | "completed" | "failed" | "delayed" | "paused";
|
|
22
|
+
type JobStatus$1 = "pending" | "active" | "completed" | "failed" | "delayed" | "paused";
|
|
23
23
|
interface PaginatedResponse<T> {
|
|
24
24
|
data: T[];
|
|
25
25
|
pagination: {
|
|
@@ -130,10 +130,19 @@ interface DealScoreSummary {
|
|
|
130
130
|
};
|
|
131
131
|
}
|
|
132
132
|
interface ScreenshotResult {
|
|
133
|
+
/** Screenshot URL - private signed URL (default) or public URL (Enterprise opt-in) */
|
|
133
134
|
url: string;
|
|
135
|
+
/** Whether this is a public URL (false = private signed URL with TTL) */
|
|
136
|
+
isPublic: boolean;
|
|
137
|
+
/** Expiration timestamp for signed URLs (ISO 8601, only for private URLs) */
|
|
138
|
+
expiresAt?: string;
|
|
139
|
+
/** Screenshot width in pixels */
|
|
134
140
|
width: number;
|
|
141
|
+
/** Screenshot height in pixels */
|
|
135
142
|
height: number;
|
|
143
|
+
/** Image format */
|
|
136
144
|
format: "png" | "jpeg" | "webp";
|
|
145
|
+
/** File size in bytes */
|
|
137
146
|
sizeBytes: number;
|
|
138
147
|
}
|
|
139
148
|
interface AIExtraction {
|
|
@@ -438,7 +447,7 @@ interface JobResponse {
|
|
|
438
447
|
/** Unique job identifier */
|
|
439
448
|
jobId: string;
|
|
440
449
|
/** Current job status */
|
|
441
|
-
status: JobStatus;
|
|
450
|
+
status: JobStatus$1;
|
|
442
451
|
/** URL to check job status */
|
|
443
452
|
statusUrl: string;
|
|
444
453
|
/** Estimated time to completion */
|
|
@@ -468,6 +477,15 @@ interface BatchScrapeResultItem {
|
|
|
468
477
|
/** Error message if job creation failed */
|
|
469
478
|
error?: string;
|
|
470
479
|
}
|
|
480
|
+
/** Invalid URL entry (returned when ignoreInvalidURLs=true) */
|
|
481
|
+
interface InvalidURLEntry {
|
|
482
|
+
/** The invalid URL */
|
|
483
|
+
url: string;
|
|
484
|
+
/** Reference ID if provided */
|
|
485
|
+
ref?: string;
|
|
486
|
+
/** Reason for rejection */
|
|
487
|
+
reason: string;
|
|
488
|
+
}
|
|
471
489
|
/** Batch scrape response */
|
|
472
490
|
interface BatchScrapeResponse {
|
|
473
491
|
/** Batch ID for tracking */
|
|
@@ -482,6 +500,11 @@ interface BatchScrapeResponse {
|
|
|
482
500
|
results: BatchScrapeResultItem[];
|
|
483
501
|
/** Timestamp when batch was created */
|
|
484
502
|
createdAt: string;
|
|
503
|
+
/**
|
|
504
|
+
* Invalid URLs that were skipped (only present when ignoreInvalidURLs=true)
|
|
505
|
+
* Firecrawl-compatible field
|
|
506
|
+
*/
|
|
507
|
+
invalidURLs?: InvalidURLEntry[];
|
|
485
508
|
}
|
|
486
509
|
/** Batch status response */
|
|
487
510
|
interface BatchStatusResponse {
|
|
@@ -502,7 +525,7 @@ interface BatchStatusResponse {
|
|
|
502
525
|
/** Individual job statuses */
|
|
503
526
|
jobs: Array<{
|
|
504
527
|
id: string;
|
|
505
|
-
status: JobStatus;
|
|
528
|
+
status: JobStatus$1;
|
|
506
529
|
result?: unknown;
|
|
507
530
|
error?: string;
|
|
508
531
|
}>;
|
|
@@ -612,7 +635,7 @@ interface CheckpointInfo {
|
|
|
612
635
|
/** Job status response */
|
|
613
636
|
interface JobStatusResponse {
|
|
614
637
|
jobId: string;
|
|
615
|
-
status: JobStatus;
|
|
638
|
+
status: JobStatus$1;
|
|
616
639
|
progress?: number;
|
|
617
640
|
createdAt: string;
|
|
618
641
|
completedAt?: string;
|
|
@@ -652,7 +675,7 @@ interface ResumeJobResponse {
|
|
|
652
675
|
/** Job metrics response */
|
|
653
676
|
interface JobMetricsResponse {
|
|
654
677
|
jobId: string;
|
|
655
|
-
status: JobStatus;
|
|
678
|
+
status: JobStatus$1;
|
|
656
679
|
metrics: {
|
|
657
680
|
totalUrls: number;
|
|
658
681
|
successfulUrls: number;
|
|
@@ -671,7 +694,7 @@ interface CancelJobResponse {
|
|
|
671
694
|
interface JobSummary {
|
|
672
695
|
id: string;
|
|
673
696
|
type: "scrape" | "crawl" | "dork" | "extract";
|
|
674
|
-
status: JobStatus;
|
|
697
|
+
status: JobStatus$1;
|
|
675
698
|
input: Record<string, unknown>;
|
|
676
699
|
dealsFound?: number;
|
|
677
700
|
avgDealScore?: number;
|
|
@@ -964,7 +987,7 @@ interface AgentStepResponse {
|
|
|
964
987
|
currentUrl?: string;
|
|
965
988
|
}
|
|
966
989
|
/** Agent completion reason */
|
|
967
|
-
type AgentCompletionReason = "goal_achieved" | "max_steps_reached" | "stuck" | "error"
|
|
990
|
+
type AgentCompletionReason = "goal_achieved" | "max_steps_reached" | "stuck" | "error";
|
|
968
991
|
/** Agent result (from completed job) */
|
|
969
992
|
interface AgentResultResponse {
|
|
970
993
|
/** Extracted/collected data (matches provided schema if given) */
|
|
@@ -1013,6 +1036,113 @@ interface SchemaGenerationResponse {
|
|
|
1013
1036
|
/** Confidence score (0-1) in the generated schema */
|
|
1014
1037
|
confidence: number;
|
|
1015
1038
|
}
|
|
1039
|
+
/** SSE token response */
|
|
1040
|
+
interface SSETokenResponse {
|
|
1041
|
+
/** Short-lived SSE authentication token */
|
|
1042
|
+
token: string;
|
|
1043
|
+
/** Expiration timestamp (ISO 8601) */
|
|
1044
|
+
expiresAt: string;
|
|
1045
|
+
/** TTL in seconds (default: 300 = 5 minutes) */
|
|
1046
|
+
expiresIn: number;
|
|
1047
|
+
/** Token type identifier */
|
|
1048
|
+
type: "sse_token";
|
|
1049
|
+
/** Usage instructions for the token */
|
|
1050
|
+
usage: string;
|
|
1051
|
+
}
|
|
1052
|
+
/** SSE connection limits response */
|
|
1053
|
+
interface SSEConnectionLimitsResponse {
|
|
1054
|
+
/** Client tier */
|
|
1055
|
+
tier: "free" | "pro" | "enterprise";
|
|
1056
|
+
/** SSE connection limits */
|
|
1057
|
+
sse: {
|
|
1058
|
+
/** Maximum concurrent connections allowed */
|
|
1059
|
+
maxConnections: number;
|
|
1060
|
+
/** Current active connections */
|
|
1061
|
+
currentConnections: number;
|
|
1062
|
+
/** Available connection slots */
|
|
1063
|
+
available: number;
|
|
1064
|
+
};
|
|
1065
|
+
}
|
|
1066
|
+
/** Tier-specific TTL limits */
|
|
1067
|
+
interface TtlLimits {
|
|
1068
|
+
/** Minimum TTL in seconds */
|
|
1069
|
+
min: number;
|
|
1070
|
+
/** Maximum TTL in seconds */
|
|
1071
|
+
max: number;
|
|
1072
|
+
/** Default TTL in seconds */
|
|
1073
|
+
default: number;
|
|
1074
|
+
}
|
|
1075
|
+
/** Formatted TTL limits (human-readable) */
|
|
1076
|
+
interface FormattedTtlLimits {
|
|
1077
|
+
/** Minimum TTL formatted (e.g., "1 hour") */
|
|
1078
|
+
min: string;
|
|
1079
|
+
/** Maximum TTL formatted (e.g., "30 days") */
|
|
1080
|
+
max: string;
|
|
1081
|
+
/** Default TTL formatted (e.g., "7 days") */
|
|
1082
|
+
default: string;
|
|
1083
|
+
}
|
|
1084
|
+
/** Screenshot signed URL refresh response */
|
|
1085
|
+
interface ScreenshotRefreshResponse {
|
|
1086
|
+
/** Success status */
|
|
1087
|
+
success: boolean;
|
|
1088
|
+
/** New signed URL */
|
|
1089
|
+
url: string;
|
|
1090
|
+
/** Expiration timestamp (ISO 8601) */
|
|
1091
|
+
expiresAt: string;
|
|
1092
|
+
/** TTL in seconds */
|
|
1093
|
+
ttl: number;
|
|
1094
|
+
/** Tier-specific limits for reference */
|
|
1095
|
+
tierLimits: TtlLimits;
|
|
1096
|
+
/** Additional metadata */
|
|
1097
|
+
metadata: {
|
|
1098
|
+
/** File path in storage */
|
|
1099
|
+
path: string;
|
|
1100
|
+
/** Bucket name */
|
|
1101
|
+
bucket: string;
|
|
1102
|
+
/** Timestamp when URL was refreshed */
|
|
1103
|
+
refreshedAt: string;
|
|
1104
|
+
};
|
|
1105
|
+
}
|
|
1106
|
+
/** Screenshot TTL limits response */
|
|
1107
|
+
interface ScreenshotLimitsResponse {
|
|
1108
|
+
/** Client tier */
|
|
1109
|
+
tier: "free" | "pro" | "enterprise";
|
|
1110
|
+
/** TTL limits in seconds */
|
|
1111
|
+
limits: TtlLimits;
|
|
1112
|
+
/** Human-readable formatted limits */
|
|
1113
|
+
formattedLimits: FormattedTtlLimits;
|
|
1114
|
+
}
|
|
1115
|
+
/**
|
|
1116
|
+
* Metadata extracted during HTML to Markdown conversion
|
|
1117
|
+
*/
|
|
1118
|
+
interface ConversionMetadata {
|
|
1119
|
+
/** Page title if found */
|
|
1120
|
+
title?: string;
|
|
1121
|
+
/** Word count in resulting markdown */
|
|
1122
|
+
wordCount: number;
|
|
1123
|
+
/** Number of links in markdown */
|
|
1124
|
+
linkCount: number;
|
|
1125
|
+
/** Number of images in markdown */
|
|
1126
|
+
imageCount: number;
|
|
1127
|
+
/** Conversion duration in milliseconds */
|
|
1128
|
+
conversionTimeMs: number;
|
|
1129
|
+
}
|
|
1130
|
+
/**
|
|
1131
|
+
* HTML to Markdown conversion response
|
|
1132
|
+
*/
|
|
1133
|
+
interface ConvertResponse {
|
|
1134
|
+
/** Success status */
|
|
1135
|
+
success: boolean;
|
|
1136
|
+
/** Conversion result data */
|
|
1137
|
+
data: {
|
|
1138
|
+
/** Generated markdown content */
|
|
1139
|
+
markdown: string;
|
|
1140
|
+
/** Extraction metadata */
|
|
1141
|
+
metadata: ConversionMetadata;
|
|
1142
|
+
/** Warnings (truncation, excluded elements, etc.) */
|
|
1143
|
+
warnings?: string[];
|
|
1144
|
+
};
|
|
1145
|
+
}
|
|
1016
1146
|
|
|
1017
1147
|
/**
|
|
1018
1148
|
* Polling Utilities
|
|
@@ -1089,6 +1219,22 @@ interface ScreenshotOptions {
|
|
|
1089
1219
|
format?: "png" | "jpeg" | "webp";
|
|
1090
1220
|
/** JPEG/WebP quality 0-100 (default: 80, only for jpeg/webp) */
|
|
1091
1221
|
quality?: number;
|
|
1222
|
+
/**
|
|
1223
|
+
* Request public URL (Enterprise only, default: false)
|
|
1224
|
+
* SECURITY WARNING: Public URLs can expose personal data, copyrighted content, and tokens
|
|
1225
|
+
* By default, screenshots use private signed URLs with tier-specific expiration
|
|
1226
|
+
* Requires Enterprise tier
|
|
1227
|
+
*/
|
|
1228
|
+
publicUrl?: boolean;
|
|
1229
|
+
/**
|
|
1230
|
+
* TTL for signed URLs in seconds (only for private screenshots)
|
|
1231
|
+
* Tier limits:
|
|
1232
|
+
* - Free: 3600 (1h) - 86400 (24h), default: 86400
|
|
1233
|
+
* - Pro: 3600 (1h) - 604800 (7d), default: 604800
|
|
1234
|
+
* - Enterprise: 3600 (1h) - 2592000 (30d), default: 604800
|
|
1235
|
+
* If not specified, uses tier-specific default
|
|
1236
|
+
*/
|
|
1237
|
+
signedUrlTtl?: number;
|
|
1092
1238
|
}
|
|
1093
1239
|
/** Options for scraping a single page */
|
|
1094
1240
|
interface ScrapeOptions {
|
|
@@ -1151,8 +1297,6 @@ interface BatchScrapeItem {
|
|
|
1151
1297
|
outputMarkdown?: boolean;
|
|
1152
1298
|
/** Override markdownBaseUrl for this URL */
|
|
1153
1299
|
markdownBaseUrl?: string;
|
|
1154
|
-
/** Override actions for this URL */
|
|
1155
|
-
actions?: ActionInput[];
|
|
1156
1300
|
}
|
|
1157
1301
|
/** Default options applied to all URLs in a batch */
|
|
1158
1302
|
interface BatchScrapeDefaults {
|
|
@@ -1188,8 +1332,6 @@ interface BatchScrapeDefaults {
|
|
|
1188
1332
|
outputMarkdown?: boolean;
|
|
1189
1333
|
/** Base URL for resolving relative URLs in markdown */
|
|
1190
1334
|
markdownBaseUrl?: string;
|
|
1191
|
-
/** Browser actions to execute before scraping */
|
|
1192
|
-
actions?: ActionInput[];
|
|
1193
1335
|
}
|
|
1194
1336
|
/** Options for batch scraping multiple URLs */
|
|
1195
1337
|
interface BatchScrapeOptions {
|
|
@@ -1201,8 +1343,13 @@ interface BatchScrapeOptions {
|
|
|
1201
1343
|
webhookUrl?: string;
|
|
1202
1344
|
/** Priority for batch jobs (1-10, higher = faster) */
|
|
1203
1345
|
priority?: number;
|
|
1204
|
-
/** Delay between job submissions in ms (0-5000) */
|
|
1205
|
-
|
|
1346
|
+
/** Delay between job submissions in ms (0-5000, default: 0) */
|
|
1347
|
+
delayMs?: number;
|
|
1348
|
+
/**
|
|
1349
|
+
* Continue processing even if some URLs are invalid (default: false)
|
|
1350
|
+
* Invalid URLs will be returned in the response with status: "failed"
|
|
1351
|
+
*/
|
|
1352
|
+
ignoreInvalidURLs?: boolean;
|
|
1206
1353
|
}
|
|
1207
1354
|
/** AI provider for search optimization */
|
|
1208
1355
|
type SearchAiProvider = "openai" | "anthropic";
|
|
@@ -1230,11 +1377,11 @@ interface SearchOptions {
|
|
|
1230
1377
|
/** Search query (required) */
|
|
1231
1378
|
query: string;
|
|
1232
1379
|
/** Maximum number of results (1-100, default: 10) */
|
|
1233
|
-
|
|
1234
|
-
/** Auto-scrape
|
|
1235
|
-
|
|
1236
|
-
/**
|
|
1237
|
-
|
|
1380
|
+
limit?: number;
|
|
1381
|
+
/** Auto-scrape search results (default: false) */
|
|
1382
|
+
scrapeResults?: boolean;
|
|
1383
|
+
/** Maximum results to auto-scrape when scrapeResults=true (1-10, default: 5) */
|
|
1384
|
+
maxScrapeResults?: number;
|
|
1238
1385
|
/** Use AI to optimize the search query */
|
|
1239
1386
|
useAiOptimization?: boolean;
|
|
1240
1387
|
/** AI provider for query optimization */
|
|
@@ -1345,7 +1492,7 @@ interface CrawlTemplate {
|
|
|
1345
1492
|
defaultOptions: Partial<CrawlOptions>;
|
|
1346
1493
|
}
|
|
1347
1494
|
/** LLM model options for extraction */
|
|
1348
|
-
type ExtractModel = "gpt-4o-mini" | "gpt-4o" | "claude-3-haiku" | "claude-3-sonnet";
|
|
1495
|
+
type ExtractModel = "gpt-4o-mini" | "gpt-4o" | "gpt-4o-2024-11-20" | "o1-mini" | "claude-3-5-haiku-20241022" | "claude-3-5-sonnet-20241022" | "claude-3-opus-20240229";
|
|
1349
1496
|
/** Options for structured data extraction */
|
|
1350
1497
|
interface ExtractOptions {
|
|
1351
1498
|
/** URL to extract data from (required) */
|
|
@@ -1506,7 +1653,7 @@ interface UpdateWebhookOptions {
|
|
|
1506
1653
|
* API key scope - Must match @dealcrawl/shared/src/types/api-key.types.ts
|
|
1507
1654
|
* These are the actual scopes enforced by the backend via requireScope() middleware
|
|
1508
1655
|
*/
|
|
1509
|
-
type ApiKeyScope = "scrape" | "
|
|
1656
|
+
type ApiKeyScope = "scrape" | "crawl" | "dork" | "extract" | "agent" | "status" | "data:read" | "data:export" | "keys:manage" | "webhooks:manage";
|
|
1510
1657
|
/**
|
|
1511
1658
|
* All available scopes (for reference and validation)
|
|
1512
1659
|
*/
|
|
@@ -1684,6 +1831,71 @@ interface SchemaGenerationOptions {
|
|
|
1684
1831
|
/** LLM provider for generation (default: openai) */
|
|
1685
1832
|
model?: AgentModel;
|
|
1686
1833
|
}
|
|
1834
|
+
/**
|
|
1835
|
+
* Options for subscribing to SSE events
|
|
1836
|
+
*/
|
|
1837
|
+
interface SSESubscribeOptions {
|
|
1838
|
+
/**
|
|
1839
|
+
* Last event ID for reconnection/replay
|
|
1840
|
+
* If provided, server will replay missed events since this ID
|
|
1841
|
+
*/
|
|
1842
|
+
lastEventId?: string;
|
|
1843
|
+
/**
|
|
1844
|
+
* Callback for received events
|
|
1845
|
+
* Called for each event received from the server
|
|
1846
|
+
*/
|
|
1847
|
+
onEvent?: (event: MessageEvent) => void;
|
|
1848
|
+
/**
|
|
1849
|
+
* Callback for errors
|
|
1850
|
+
* Called when connection error occurs
|
|
1851
|
+
*/
|
|
1852
|
+
onError?: (error: Error) => void;
|
|
1853
|
+
/**
|
|
1854
|
+
* Callback for connection open
|
|
1855
|
+
* Called when SSE connection is established
|
|
1856
|
+
*/
|
|
1857
|
+
onOpen?: (event: Event) => void;
|
|
1858
|
+
/**
|
|
1859
|
+
* Auto-reconnect on connection loss (default: true)
|
|
1860
|
+
* If true, EventSource will automatically reconnect
|
|
1861
|
+
*/
|
|
1862
|
+
autoReconnect?: boolean;
|
|
1863
|
+
/**
|
|
1864
|
+
* Reconnection delay in milliseconds (default: 3000)
|
|
1865
|
+
* How long to wait before attempting reconnection
|
|
1866
|
+
*/
|
|
1867
|
+
reconnectDelay?: number;
|
|
1868
|
+
}
|
|
1869
|
+
/**
|
|
1870
|
+
* Options for HTML to Markdown conversion
|
|
1871
|
+
*/
|
|
1872
|
+
interface ConvertOptions {
|
|
1873
|
+
/** HTML content to convert (required, max 10MB) */
|
|
1874
|
+
html: string;
|
|
1875
|
+
/** Base URL for resolving relative URLs */
|
|
1876
|
+
baseUrl?: string;
|
|
1877
|
+
/** Conversion options */
|
|
1878
|
+
options?: {
|
|
1879
|
+
/** Enable GFM tables (default: true) */
|
|
1880
|
+
gfmTables?: boolean;
|
|
1881
|
+
/** Enable GFM strikethrough (default: true) */
|
|
1882
|
+
gfmStrikethrough?: boolean;
|
|
1883
|
+
/** Enable GFM task lists (default: true) */
|
|
1884
|
+
gfmTaskLists?: boolean;
|
|
1885
|
+
/** Remove noise elements like nav, footer, ads (default: true) */
|
|
1886
|
+
removeNoise?: boolean;
|
|
1887
|
+
/** CSS selectors to exclude from conversion */
|
|
1888
|
+
excludeSelectors?: string[];
|
|
1889
|
+
/** Resolve relative URLs to absolute (default: true) */
|
|
1890
|
+
absoluteUrls?: boolean;
|
|
1891
|
+
/** Maximum output length in characters (0 = unlimited, max: 1,000,000) */
|
|
1892
|
+
maxLength?: number;
|
|
1893
|
+
/** Include images in output (default: true) */
|
|
1894
|
+
includeImages?: boolean;
|
|
1895
|
+
/** Include links in output (default: true) */
|
|
1896
|
+
includeLinks?: boolean;
|
|
1897
|
+
};
|
|
1898
|
+
}
|
|
1687
1899
|
|
|
1688
1900
|
/**
|
|
1689
1901
|
* Account Resource
|
|
@@ -2028,6 +2240,257 @@ declare class AgentResource {
|
|
|
2028
2240
|
generateSchema(options: SchemaGenerationOptions): Promise<SchemaGenerationResponse>;
|
|
2029
2241
|
}
|
|
2030
2242
|
|
|
2243
|
+
/**
|
|
2244
|
+
* Auth Resource
|
|
2245
|
+
* Handles SSE (Server-Sent Events) authentication
|
|
2246
|
+
*/
|
|
2247
|
+
|
|
2248
|
+
/**
|
|
2249
|
+
* Options for generating an SSE token
|
|
2250
|
+
*/
|
|
2251
|
+
interface GenerateSSETokenOptions {
|
|
2252
|
+
/**
|
|
2253
|
+
* Optional job ID to restrict token to a specific job
|
|
2254
|
+
* If provided, token can only be used for /v1/events/:jobId
|
|
2255
|
+
* If omitted, token can be used for /v1/events (all events)
|
|
2256
|
+
*/
|
|
2257
|
+
jobId?: string;
|
|
2258
|
+
}
|
|
2259
|
+
/**
|
|
2260
|
+
* Auth resource class
|
|
2261
|
+
* Provides methods for SSE authentication
|
|
2262
|
+
*
|
|
2263
|
+
* @example
|
|
2264
|
+
* ```ts
|
|
2265
|
+
* // Generate SSE token for browser EventSource
|
|
2266
|
+
* const { token, expiresAt } = await client.auth.generateSSEToken();
|
|
2267
|
+
*
|
|
2268
|
+
* // Use in browser
|
|
2269
|
+
* const eventSource = new EventSource(`/v1/events?token=${token}`);
|
|
2270
|
+
*
|
|
2271
|
+
* // Token for specific job
|
|
2272
|
+
* const jobToken = await client.auth.generateSSEToken({ jobId: "job_123" });
|
|
2273
|
+
* const jobEvents = new EventSource(`/v1/events/job_123?token=${jobToken.token}`);
|
|
2274
|
+
* ```
|
|
2275
|
+
*/
|
|
2276
|
+
declare class AuthResource {
|
|
2277
|
+
private ctx;
|
|
2278
|
+
constructor(ctx: RequestContext);
|
|
2279
|
+
/**
|
|
2280
|
+
* Generate SSE authentication token
|
|
2281
|
+
*
|
|
2282
|
+
* Required for browser-based SSE connections because EventSource API
|
|
2283
|
+
* doesn't support custom headers. Token is short-lived (5 minutes).
|
|
2284
|
+
*
|
|
2285
|
+
* Security:
|
|
2286
|
+
* - Requires valid API key (Bearer token)
|
|
2287
|
+
* - Token expires in 5 minutes
|
|
2288
|
+
* - Token can be restricted to specific job
|
|
2289
|
+
* - Token stored in Redis (revocable)
|
|
2290
|
+
*
|
|
2291
|
+
* @example
|
|
2292
|
+
* ```ts
|
|
2293
|
+
* // 1. Generate token
|
|
2294
|
+
* const { token, expiresAt } = await client.auth.generateSSEToken();
|
|
2295
|
+
* console.log(`Token expires at: ${expiresAt}`);
|
|
2296
|
+
*
|
|
2297
|
+
* // 2. Use in browser EventSource
|
|
2298
|
+
* const eventSource = new EventSource(`/v1/events?token=${token}`);
|
|
2299
|
+
*
|
|
2300
|
+
* eventSource.addEventListener('job.completed', (event) => {
|
|
2301
|
+
* const data = JSON.parse(event.data);
|
|
2302
|
+
* console.log('Job completed:', data);
|
|
2303
|
+
* });
|
|
2304
|
+
*
|
|
2305
|
+
* // 3. For specific job only
|
|
2306
|
+
* const jobToken = await client.auth.generateSSEToken({ jobId: "job_abc123" });
|
|
2307
|
+
* const jobEvents = new EventSource(`/v1/events/job_abc123?token=${jobToken.token}`);
|
|
2308
|
+
* ```
|
|
2309
|
+
*/
|
|
2310
|
+
generateSSEToken(options?: GenerateSSETokenOptions): Promise<SSETokenResponse>;
|
|
2311
|
+
/**
|
|
2312
|
+
* Get SSE connection limits for current tier
|
|
2313
|
+
*
|
|
2314
|
+
* Shows how many concurrent SSE connections are allowed
|
|
2315
|
+
* and how many are currently active.
|
|
2316
|
+
*
|
|
2317
|
+
* Tier limits:
|
|
2318
|
+
* - Free: 2 concurrent connections
|
|
2319
|
+
* - Pro: 10 concurrent connections
|
|
2320
|
+
* - Enterprise: 50 concurrent connections
|
|
2321
|
+
*
|
|
2322
|
+
* @example
|
|
2323
|
+
* ```ts
|
|
2324
|
+
* const limits = await client.auth.getLimits();
|
|
2325
|
+
*
|
|
2326
|
+
* console.log(`Tier: ${limits.tier}`);
|
|
2327
|
+
* console.log(`Max connections: ${limits.sse.maxConnections}`);
|
|
2328
|
+
* console.log(`Current connections: ${limits.sse.currentConnections}`);
|
|
2329
|
+
* console.log(`Available: ${limits.sse.available}`);
|
|
2330
|
+
*
|
|
2331
|
+
* // Check before opening new connection
|
|
2332
|
+
* if (limits.sse.available > 0) {
|
|
2333
|
+
* const token = await client.auth.generateSSEToken();
|
|
2334
|
+
* const eventSource = new EventSource(`/v1/events?token=${token.token}`);
|
|
2335
|
+
* } else {
|
|
2336
|
+
* console.error('No available SSE connection slots');
|
|
2337
|
+
* }
|
|
2338
|
+
* ```
|
|
2339
|
+
*/
|
|
2340
|
+
getLimits(): Promise<SSEConnectionLimitsResponse>;
|
|
2341
|
+
}
|
|
2342
|
+
|
|
2343
|
+
/**
|
|
2344
|
+
* Convert Resource
|
|
2345
|
+
* Handles HTML to Markdown conversion
|
|
2346
|
+
*/
|
|
2347
|
+
|
|
2348
|
+
/**
|
|
2349
|
+
* Convert resource class
|
|
2350
|
+
* Provides methods for HTML to Markdown conversion
|
|
2351
|
+
*
|
|
2352
|
+
* @example
|
|
2353
|
+
* ```ts
|
|
2354
|
+
* // Convert HTML to markdown
|
|
2355
|
+
* const result = await client.convert.htmlToMarkdown({
|
|
2356
|
+
* html: "<h1>Title</h1><p>Content</p>",
|
|
2357
|
+
* baseUrl: "https://example.com",
|
|
2358
|
+
* options: {
|
|
2359
|
+
* gfmTables: true,
|
|
2360
|
+
* removeNoise: true
|
|
2361
|
+
* }
|
|
2362
|
+
* });
|
|
2363
|
+
*
|
|
2364
|
+
* console.log(result.data.markdown);
|
|
2365
|
+
* console.log(result.data.metadata.wordCount);
|
|
2366
|
+
* ```
|
|
2367
|
+
*/
|
|
2368
|
+
declare class ConvertResource {
|
|
2369
|
+
private ctx;
|
|
2370
|
+
constructor(ctx: RequestContext);
|
|
2371
|
+
/**
|
|
2372
|
+
* Convert HTML to Markdown
|
|
2373
|
+
*
|
|
2374
|
+
* Transforms raw HTML content into clean, readable Markdown using GitHub Flavored Markdown (GFM).
|
|
2375
|
+
* Useful for:
|
|
2376
|
+
* - Converting scraped HTML to markdown for LLM processing
|
|
2377
|
+
* - Cleaning up messy HTML from web pages
|
|
2378
|
+
* - Extracting main content while removing noise (ads, nav, footer)
|
|
2379
|
+
* - Creating documentation from HTML sources
|
|
2380
|
+
*
|
|
2381
|
+
* Features:
|
|
2382
|
+
* - GFM table, strikethrough, and task list support
|
|
2383
|
+
* - Automatic noise removal (scripts, ads, navigation)
|
|
2384
|
+
* - Relative URL resolution
|
|
2385
|
+
* - Custom element exclusion via CSS selectors
|
|
2386
|
+
* - Output length limiting
|
|
2387
|
+
*
|
|
2388
|
+
* @param options - Conversion options
|
|
2389
|
+
* @returns Conversion result with markdown, metadata, and warnings
|
|
2390
|
+
*
|
|
2391
|
+
* @example Basic usage
|
|
2392
|
+
* ```ts
|
|
2393
|
+
* const result = await client.convert.htmlToMarkdown({
|
|
2394
|
+
* html: "<h1>Product</h1><p>Price: $99</p>"
|
|
2395
|
+
* });
|
|
2396
|
+
* console.log(result.data.markdown);
|
|
2397
|
+
* ```
|
|
2398
|
+
*
|
|
2399
|
+
* @example With all options
|
|
2400
|
+
* ```ts
|
|
2401
|
+
* const result = await client.convert.htmlToMarkdown({
|
|
2402
|
+
* html: htmlContent,
|
|
2403
|
+
* baseUrl: "https://shop.example.com",
|
|
2404
|
+
* options: {
|
|
2405
|
+
* gfmTables: true,
|
|
2406
|
+
* removeNoise: true,
|
|
2407
|
+
* excludeSelectors: [".advertisement", "#sidebar"],
|
|
2408
|
+
* absoluteUrls: true,
|
|
2409
|
+
* maxLength: 100000,
|
|
2410
|
+
* includeImages: true,
|
|
2411
|
+
* includeLinks: true
|
|
2412
|
+
* }
|
|
2413
|
+
* });
|
|
2414
|
+
*
|
|
2415
|
+
* // Check metadata
|
|
2416
|
+
* console.log(`Words: ${result.data.metadata.wordCount}`);
|
|
2417
|
+
* console.log(`Links: ${result.data.metadata.linkCount}`);
|
|
2418
|
+
* console.log(`Images: ${result.data.metadata.imageCount}`);
|
|
2419
|
+
* console.log(`Conversion time: ${result.data.metadata.conversionTimeMs}ms`);
|
|
2420
|
+
*
|
|
2421
|
+
* // Check for warnings
|
|
2422
|
+
* if (result.data.warnings?.length) {
|
|
2423
|
+
* console.warn("Conversion warnings:", result.data.warnings);
|
|
2424
|
+
* }
|
|
2425
|
+
* ```
|
|
2426
|
+
*
|
|
2427
|
+
* @example Converting scraped HTML
|
|
2428
|
+
* ```ts
|
|
2429
|
+
* // First scrape a page
|
|
2430
|
+
* const scrapeJob = await client.scrape.create({
|
|
2431
|
+
* url: "https://example.com/article"
|
|
2432
|
+
* });
|
|
2433
|
+
* const scrapeResult = await client.waitForResult(scrapeJob.jobId);
|
|
2434
|
+
*
|
|
2435
|
+
* // Then convert HTML to markdown
|
|
2436
|
+
* const markdown = await client.convert.htmlToMarkdown({
|
|
2437
|
+
* html: scrapeResult.data.html,
|
|
2438
|
+
* baseUrl: scrapeResult.data.url,
|
|
2439
|
+
* options: {
|
|
2440
|
+
* removeNoise: true,
|
|
2441
|
+
* onlyMainContent: true
|
|
2442
|
+
* }
|
|
2443
|
+
* });
|
|
2444
|
+
* ```
|
|
2445
|
+
*/
|
|
2446
|
+
htmlToMarkdown(options: ConvertOptions): Promise<ConvertResponse>;
|
|
2447
|
+
/**
|
|
2448
|
+
* Alias for htmlToMarkdown() for convenience
|
|
2449
|
+
*
|
|
2450
|
+
* @example
|
|
2451
|
+
* ```ts
|
|
2452
|
+
* const result = await client.convert.toMarkdown({
|
|
2453
|
+
* html: "<h1>Hello</h1>"
|
|
2454
|
+
* });
|
|
2455
|
+
* ```
|
|
2456
|
+
*/
|
|
2457
|
+
toMarkdown(options: ConvertOptions): Promise<ConvertResponse>;
|
|
2458
|
+
/**
|
|
2459
|
+
* Convert HTML with minimal options (just the HTML content)
|
|
2460
|
+
* Uses all default settings
|
|
2461
|
+
*
|
|
2462
|
+
* @param html - HTML content to convert
|
|
2463
|
+
* @param baseUrl - Optional base URL for resolving relative links
|
|
2464
|
+
* @returns Conversion result
|
|
2465
|
+
*
|
|
2466
|
+
* @example
|
|
2467
|
+
* ```ts
|
|
2468
|
+
* const result = await client.convert.quick(
|
|
2469
|
+
* "<h1>Title</h1><p>Content</p>",
|
|
2470
|
+
* "https://example.com"
|
|
2471
|
+
* );
|
|
2472
|
+
* console.log(result.data.markdown);
|
|
2473
|
+
* ```
|
|
2474
|
+
*/
|
|
2475
|
+
quick(html: string, baseUrl?: string): Promise<ConvertResponse>;
|
|
2476
|
+
/**
|
|
2477
|
+
* Convert HTML with noise removal enabled
|
|
2478
|
+
* Removes navigation, footer, ads, scripts, and other clutter
|
|
2479
|
+
*
|
|
2480
|
+
* @param html - HTML content to convert
|
|
2481
|
+
* @param baseUrl - Optional base URL
|
|
2482
|
+
* @returns Conversion result with clean markdown
|
|
2483
|
+
*
|
|
2484
|
+
* @example
|
|
2485
|
+
* ```ts
|
|
2486
|
+
* // Extract just the main content from a messy page
|
|
2487
|
+
* const result = await client.convert.clean(messyHtml, "https://example.com");
|
|
2488
|
+
* console.log(result.data.markdown); // Clean, readable markdown
|
|
2489
|
+
* ```
|
|
2490
|
+
*/
|
|
2491
|
+
clean(html: string, baseUrl?: string): Promise<ConvertResponse>;
|
|
2492
|
+
}
|
|
2493
|
+
|
|
2031
2494
|
/**
|
|
2032
2495
|
* Crawl Resource
|
|
2033
2496
|
* Handles website crawling operations
|
|
@@ -2378,6 +2841,177 @@ declare class DorkResource {
|
|
|
2378
2841
|
buildQuery(options: DorkOptions): string;
|
|
2379
2842
|
}
|
|
2380
2843
|
|
|
2844
|
+
/**
|
|
2845
|
+
* Events Resource
|
|
2846
|
+
* Handles real-time SSE (Server-Sent Events) streaming
|
|
2847
|
+
*/
|
|
2848
|
+
|
|
2849
|
+
/**
|
|
2850
|
+
* Events resource class
|
|
2851
|
+
* Provides methods for subscribing to real-time job events via SSE
|
|
2852
|
+
*
|
|
2853
|
+
* IMPORTANT: Browser Support Only
|
|
2854
|
+
* This resource uses the EventSource API which is only available in browsers.
|
|
2855
|
+
* For Node.js, use polling via client.status.get() instead.
|
|
2856
|
+
*
|
|
2857
|
+
* @example Browser Usage
|
|
2858
|
+
* ```ts
|
|
2859
|
+
* // 1. Generate SSE token (required for EventSource)
|
|
2860
|
+
* const { token } = await client.auth.generateSSEToken();
|
|
2861
|
+
*
|
|
2862
|
+
* // 2. Subscribe to all events
|
|
2863
|
+
* const eventSource = client.events.subscribe({
|
|
2864
|
+
* token,
|
|
2865
|
+
* onEvent: (event) => {
|
|
2866
|
+
* console.log('Event:', event.type, event.data);
|
|
2867
|
+
* }
|
|
2868
|
+
* });
|
|
2869
|
+
*
|
|
2870
|
+
* // 3. Or subscribe to specific job
|
|
2871
|
+
* const jobEvents = client.events.subscribeToJob('job_123', {
|
|
2872
|
+
* token,
|
|
2873
|
+
* onEvent: (event) => {
|
|
2874
|
+
* const data = JSON.parse(event.data);
|
|
2875
|
+
* console.log('Job event:', data);
|
|
2876
|
+
* }
|
|
2877
|
+
* });
|
|
2878
|
+
*
|
|
2879
|
+
* // 4. Clean up when done
|
|
2880
|
+
* eventSource.close();
|
|
2881
|
+
* ```
|
|
2882
|
+
*/
|
|
2883
|
+
declare class EventsResource {
|
|
2884
|
+
private ctx;
|
|
2885
|
+
constructor(ctx: RequestContext);
|
|
2886
|
+
/**
|
|
2887
|
+
* Subscribe to all events for authenticated client
|
|
2888
|
+
*
|
|
2889
|
+
* Opens an SSE connection to receive real-time events for all jobs.
|
|
2890
|
+
* Requires an SSE token obtained via client.auth.generateSSEToken().
|
|
2891
|
+
*
|
|
2892
|
+
* Event Types:
|
|
2893
|
+
* - Job lifecycle: job.created, job.queued, job.started, job.progress,
|
|
2894
|
+
* job.completed, job.failed, job.cancelled
|
|
2895
|
+
* - Job details: job.log, job.metric, job.alert, job.checkpoint
|
|
2896
|
+
* - Deals: deal.found, deal.validated
|
|
2897
|
+
* - System: ping, connection.open, connection.close, error
|
|
2898
|
+
*
|
|
2899
|
+
* Features:
|
|
2900
|
+
* - Automatic reconnection on disconnect
|
|
2901
|
+
* - Event replay via Last-Event-ID
|
|
2902
|
+
* - Keepalive pings every 15 seconds
|
|
2903
|
+
* - Max connection time: 1 hour
|
|
2904
|
+
*
|
|
2905
|
+
* @param token - SSE authentication token from client.auth.generateSSEToken()
|
|
2906
|
+
* @param options - Subscription options (callbacks, reconnection settings)
|
|
2907
|
+
*
|
|
2908
|
+
* @example
|
|
2909
|
+
* ```ts
|
|
2910
|
+
* // Generate token
|
|
2911
|
+
* const { token } = await client.auth.generateSSEToken();
|
|
2912
|
+
*
|
|
2913
|
+
* // Subscribe with event handlers
|
|
2914
|
+
* const eventSource = client.events.subscribe(token, {
|
|
2915
|
+
* onEvent: (event) => {
|
|
2916
|
+
* // Handle all events
|
|
2917
|
+
* console.log('Event:', event.type);
|
|
2918
|
+
* const data = JSON.parse(event.data);
|
|
2919
|
+
*
|
|
2920
|
+
* if (data.jobId) {
|
|
2921
|
+
* console.log(`Job ${data.jobId}:`, data);
|
|
2922
|
+
* }
|
|
2923
|
+
* },
|
|
2924
|
+
* onError: (error) => {
|
|
2925
|
+
* console.error('SSE error:', error);
|
|
2926
|
+
* },
|
|
2927
|
+
* onOpen: () => {
|
|
2928
|
+
* console.log('SSE connection opened');
|
|
2929
|
+
* }
|
|
2930
|
+
* });
|
|
2931
|
+
*
|
|
2932
|
+
* // Listen for specific event types
|
|
2933
|
+
* eventSource.addEventListener('job.completed', (event) => {
|
|
2934
|
+
* const data = JSON.parse(event.data);
|
|
2935
|
+
* console.log('Job completed:', data);
|
|
2936
|
+
* });
|
|
2937
|
+
*
|
|
2938
|
+
* // Clean up
|
|
2939
|
+
* eventSource.close();
|
|
2940
|
+
* ```
|
|
2941
|
+
*/
|
|
2942
|
+
subscribe(token: string, options?: Omit<SSESubscribeOptions, "lastEventId">): EventSource;
|
|
2943
|
+
/**
|
|
2944
|
+
* Subscribe to events for a specific job
|
|
2945
|
+
*
|
|
2946
|
+
* Opens an SSE connection filtered to a single job.
|
|
2947
|
+
* More efficient than global subscription when tracking one job.
|
|
2948
|
+
*
|
|
2949
|
+
* @param jobId - Job ID to subscribe to
|
|
2950
|
+
* @param token - SSE authentication token
|
|
2951
|
+
* @param options - Subscription options
|
|
2952
|
+
*
|
|
2953
|
+
* @example
|
|
2954
|
+
* ```ts
|
|
2955
|
+
* // Start a scrape job
|
|
2956
|
+
* const job = await client.scrape.create({ url: "https://example.com" });
|
|
2957
|
+
*
|
|
2958
|
+
* // Generate SSE token for this job
|
|
2959
|
+
* const { token } = await client.auth.generateSSEToken({ jobId: job.jobId });
|
|
2960
|
+
*
|
|
2961
|
+
* // Subscribe to job events
|
|
2962
|
+
* const eventSource = client.events.subscribeToJob(job.jobId, token, {
|
|
2963
|
+
* onEvent: (event) => {
|
|
2964
|
+
* const data = JSON.parse(event.data);
|
|
2965
|
+
* console.log(`[${event.type}]`, data);
|
|
2966
|
+
* }
|
|
2967
|
+
* });
|
|
2968
|
+
*
|
|
2969
|
+
* // Listen for completion
|
|
2970
|
+
* eventSource.addEventListener('job.completed', (event) => {
|
|
2971
|
+
* const data = JSON.parse(event.data);
|
|
2972
|
+
* console.log('Scrape completed!', data.summary);
|
|
2973
|
+
* eventSource.close();
|
|
2974
|
+
* });
|
|
2975
|
+
*
|
|
2976
|
+
* // Listen for progress
|
|
2977
|
+
* eventSource.addEventListener('job.progress', (event) => {
|
|
2978
|
+
* const data = JSON.parse(event.data);
|
|
2979
|
+
* console.log(`Progress: ${data.progress}%`);
|
|
2980
|
+
* });
|
|
2981
|
+
*
|
|
2982
|
+
* // Listen for errors
|
|
2983
|
+
* eventSource.addEventListener('job.failed', (event) => {
|
|
2984
|
+
* const data = JSON.parse(event.data);
|
|
2985
|
+
* console.error('Job failed:', data.error);
|
|
2986
|
+
* eventSource.close();
|
|
2987
|
+
* });
|
|
2988
|
+
* ```
|
|
2989
|
+
*/
|
|
2990
|
+
subscribeToJob(jobId: string, token: string, options?: Omit<SSESubscribeOptions, "lastEventId">): EventSource;
|
|
2991
|
+
/**
|
|
2992
|
+
* Helper: Wait for job completion via SSE
|
|
2993
|
+
*
|
|
2994
|
+
* Convenience method that subscribes to a job and resolves when complete.
|
|
2995
|
+
* Automatically handles token generation and cleanup.
|
|
2996
|
+
*
|
|
2997
|
+
* @param jobId - Job ID to wait for
|
|
2998
|
+
* @param onProgress - Optional progress callback
|
|
2999
|
+
*
|
|
3000
|
+
* @example
|
|
3001
|
+
* ```ts
|
|
3002
|
+
* const job = await client.scrape.create({ url: "https://example.com" });
|
|
3003
|
+
*
|
|
3004
|
+
* // Wait for completion with progress updates
|
|
3005
|
+
* const result = await client.events.waitForCompletion(job.jobId, (progress) => {
|
|
3006
|
+
* console.log(`Progress: ${progress}%`);
|
|
3007
|
+
* });
|
|
3008
|
+
*
|
|
3009
|
+
* console.log('Job completed:', result);
|
|
3010
|
+
* ```
|
|
3011
|
+
*/
|
|
3012
|
+
waitForCompletion<T = unknown>(jobId: string, onProgress?: (progress: number) => void): Promise<T>;
|
|
3013
|
+
}
|
|
3014
|
+
|
|
2381
3015
|
/**
|
|
2382
3016
|
* Extract Resource
|
|
2383
3017
|
* Handles LLM-based structured data extraction
|
|
@@ -2682,7 +3316,8 @@ declare class ScrapeResource {
|
|
|
2682
3316
|
* { url: "https://shop1.com/product1" },
|
|
2683
3317
|
* { url: "https://shop2.com/deal", extractDeal: true }
|
|
2684
3318
|
* ],
|
|
2685
|
-
* defaults: { detectSignals: true }
|
|
3319
|
+
* defaults: { detectSignals: true },
|
|
3320
|
+
* ignoreInvalidURLs: true
|
|
2686
3321
|
* });
|
|
2687
3322
|
* console.log(batch.batchId, batch.results);
|
|
2688
3323
|
* ```
|
|
@@ -2713,6 +3348,58 @@ declare class ScrapeResource {
|
|
|
2713
3348
|
batchForDeals(urls: string[], options?: Omit<BatchScrapeOptions, "urls">): Promise<BatchScrapeResponse>;
|
|
2714
3349
|
}
|
|
2715
3350
|
|
|
3351
|
+
/**
|
|
3352
|
+
* Screenshots Resource
|
|
3353
|
+
* Handles screenshot signed URL refresh and TTL limits
|
|
3354
|
+
*/
|
|
3355
|
+
|
|
3356
|
+
/**
|
|
3357
|
+
* Options for refreshing a screenshot signed URL
|
|
3358
|
+
*/
|
|
3359
|
+
interface RefreshScreenshotOptions {
|
|
3360
|
+
/** File path in storage (e.g., "job_abc123/timestamp_id_url.png") */
|
|
3361
|
+
path: string;
|
|
3362
|
+
/** Optional: New TTL in seconds (must be within tier limits) */
|
|
3363
|
+
ttl?: number;
|
|
3364
|
+
/** Optional: Bucket name (defaults to 'screenshots-private') */
|
|
3365
|
+
bucket?: string;
|
|
3366
|
+
}
|
|
3367
|
+
/**
|
|
3368
|
+
* Screenshots resource class
|
|
3369
|
+
* Provides methods for managing screenshot signed URLs
|
|
3370
|
+
*/
|
|
3371
|
+
declare class ScreenshotsResource {
|
|
3372
|
+
private ctx;
|
|
3373
|
+
constructor(ctx: RequestContext);
|
|
3374
|
+
/**
|
|
3375
|
+
* Refresh a signed URL before expiration
|
|
3376
|
+
*
|
|
3377
|
+
* @example
|
|
3378
|
+
* ```ts
|
|
3379
|
+
* const refreshed = await client.screenshots.refresh({
|
|
3380
|
+
* path: "job_abc123/1234567890_nanoid_example.png",
|
|
3381
|
+
* ttl: 604800 // 7 days
|
|
3382
|
+
* });
|
|
3383
|
+
* console.log(refreshed.url); // New signed URL
|
|
3384
|
+
* console.log(refreshed.expiresAt); // "2026-01-25T12:00:00Z"
|
|
3385
|
+
* console.log(refreshed.tierLimits); // { min: 3600, max: 604800, default: 604800 }
|
|
3386
|
+
* ```
|
|
3387
|
+
*/
|
|
3388
|
+
refresh(options: RefreshScreenshotOptions): Promise<ScreenshotRefreshResponse>;
|
|
3389
|
+
/**
|
|
3390
|
+
* Get TTL limits for the current tier
|
|
3391
|
+
*
|
|
3392
|
+
* @example
|
|
3393
|
+
* ```ts
|
|
3394
|
+
* const limits = await client.screenshots.getLimits();
|
|
3395
|
+
* console.log(limits.tier); // "pro"
|
|
3396
|
+
* console.log(limits.limits.max); // 604800 (7 days in seconds)
|
|
3397
|
+
* console.log(limits.formattedLimits.max); // "7 days"
|
|
3398
|
+
* ```
|
|
3399
|
+
*/
|
|
3400
|
+
getLimits(): Promise<ScreenshotLimitsResponse>;
|
|
3401
|
+
}
|
|
3402
|
+
|
|
2716
3403
|
/**
|
|
2717
3404
|
* Search Resource
|
|
2718
3405
|
* Handles web search operations with AI optimization
|
|
@@ -2732,7 +3419,7 @@ declare class SearchResource {
|
|
|
2732
3419
|
* ```ts
|
|
2733
3420
|
* const result = await client.search.create({
|
|
2734
3421
|
* query: "laptop deals black friday",
|
|
2735
|
-
*
|
|
3422
|
+
* limit: 20,
|
|
2736
3423
|
* useDealScoring: true
|
|
2737
3424
|
* });
|
|
2738
3425
|
* ```
|
|
@@ -2768,12 +3455,12 @@ declare class SearchResource {
|
|
|
2768
3455
|
* @example
|
|
2769
3456
|
* ```ts
|
|
2770
3457
|
* const result = await client.search.andScrape("promo codes", {
|
|
2771
|
-
*
|
|
3458
|
+
* maxScrapeResults: 5
|
|
2772
3459
|
* });
|
|
2773
3460
|
* console.log(result.data.scrapedJobIds);
|
|
2774
3461
|
* ```
|
|
2775
3462
|
*/
|
|
2776
|
-
andScrape(query: string, options?: Omit<SearchOptions, "query" | "
|
|
3463
|
+
andScrape(query: string, options?: Omit<SearchOptions, "query" | "scrapeResults">): Promise<SearchJobResponse>;
|
|
2777
3464
|
/**
|
|
2778
3465
|
* Check search API status
|
|
2779
3466
|
* Returns availability and features info
|
|
@@ -3049,6 +3736,9 @@ declare class WebhooksResource {
|
|
|
3049
3736
|
* await client.extract.create({ url: "...", schema: {...} });
|
|
3050
3737
|
* await client.dork.create({ query: "..." });
|
|
3051
3738
|
*
|
|
3739
|
+
* // Utilities
|
|
3740
|
+
* await client.convert.htmlToMarkdown({ html: "..." });
|
|
3741
|
+
*
|
|
3052
3742
|
* // Status & polling
|
|
3053
3743
|
* await client.status.get(jobId);
|
|
3054
3744
|
* await client.waitForResult(jobId);
|
|
@@ -3140,6 +3830,20 @@ declare class DealCrawl {
|
|
|
3140
3830
|
* ```
|
|
3141
3831
|
*/
|
|
3142
3832
|
readonly dork: DorkResource;
|
|
3833
|
+
/**
|
|
3834
|
+
* Convert resource - HTML to Markdown conversion
|
|
3835
|
+
*
|
|
3836
|
+
* @example
|
|
3837
|
+
* ```ts
|
|
3838
|
+
* const result = await client.convert.htmlToMarkdown({
|
|
3839
|
+
* html: "<h1>Title</h1><p>Content</p>",
|
|
3840
|
+
* baseUrl: "https://example.com",
|
|
3841
|
+
* options: { removeNoise: true }
|
|
3842
|
+
* });
|
|
3843
|
+
* console.log(result.data.markdown);
|
|
3844
|
+
* ```
|
|
3845
|
+
*/
|
|
3846
|
+
readonly convert: ConvertResource;
|
|
3143
3847
|
/**
|
|
3144
3848
|
* Agent resource - AI-powered autonomous web navigation
|
|
3145
3849
|
*
|
|
@@ -3221,6 +3925,80 @@ declare class DealCrawl {
|
|
|
3221
3925
|
* ```
|
|
3222
3926
|
*/
|
|
3223
3927
|
readonly account: AccountResource;
|
|
3928
|
+
/**
|
|
3929
|
+
* Screenshots resource - Screenshot signed URL management
|
|
3930
|
+
*
|
|
3931
|
+
* @example
|
|
3932
|
+
* ```ts
|
|
3933
|
+
* // Refresh a signed URL before expiration
|
|
3934
|
+
* const refreshed = await client.screenshots.refresh({
|
|
3935
|
+
* path: "job_abc123/1234567890_nanoid_example.png",
|
|
3936
|
+
* ttl: 604800 // 7 days
|
|
3937
|
+
* });
|
|
3938
|
+
*
|
|
3939
|
+
* // Get tier-specific TTL limits
|
|
3940
|
+
* const limits = await client.screenshots.getLimits();
|
|
3941
|
+
* console.log(limits.formattedLimits.max); // "7 days"
|
|
3942
|
+
* ```
|
|
3943
|
+
*/
|
|
3944
|
+
readonly screenshots: ScreenshotsResource;
|
|
3945
|
+
/**
|
|
3946
|
+
* Auth resource - SSE (Server-Sent Events) authentication
|
|
3947
|
+
*
|
|
3948
|
+
* @example
|
|
3949
|
+
* ```ts
|
|
3950
|
+
* // Generate SSE token for browser EventSource
|
|
3951
|
+
* const { token, expiresAt } = await client.auth.generateSSEToken();
|
|
3952
|
+
*
|
|
3953
|
+
* // Use in browser
|
|
3954
|
+
* const eventSource = new EventSource(`/v1/events?token=${token}`);
|
|
3955
|
+
*
|
|
3956
|
+
* // Generate token for specific job
|
|
3957
|
+
* const jobToken = await client.auth.generateSSEToken({ jobId: "job_123" });
|
|
3958
|
+
*
|
|
3959
|
+
* // Check connection limits
|
|
3960
|
+
* const limits = await client.auth.getLimits();
|
|
3961
|
+
* console.log(`Available connections: ${limits.sse.available}`);
|
|
3962
|
+
* ```
|
|
3963
|
+
*/
|
|
3964
|
+
readonly auth: AuthResource;
|
|
3965
|
+
/**
|
|
3966
|
+
* Events resource - Real-time SSE event streaming (Browser only)
|
|
3967
|
+
*
|
|
3968
|
+
* IMPORTANT: This resource only works in browsers. For Node.js, use polling via client.status.get()
|
|
3969
|
+
*
|
|
3970
|
+
* @example Browser Usage
|
|
3971
|
+
* ```ts
|
|
3972
|
+
* // 1. Generate SSE token
|
|
3973
|
+
* const { token } = await client.auth.generateSSEToken();
|
|
3974
|
+
*
|
|
3975
|
+
* // 2. Subscribe to all events
|
|
3976
|
+
* const eventSource = client.events.subscribe(token, {
|
|
3977
|
+
* onEvent: (event) => {
|
|
3978
|
+
* console.log('Event:', event.type, JSON.parse(event.data));
|
|
3979
|
+
* }
|
|
3980
|
+
* });
|
|
3981
|
+
*
|
|
3982
|
+
* // 3. Or subscribe to specific job
|
|
3983
|
+
* const jobEvents = client.events.subscribeToJob('job_123', token, {
|
|
3984
|
+
* onEvent: (event) => {
|
|
3985
|
+
* const data = JSON.parse(event.data);
|
|
3986
|
+
* console.log(`Progress: ${data.progress}%`);
|
|
3987
|
+
* }
|
|
3988
|
+
* });
|
|
3989
|
+
*
|
|
3990
|
+
* // 4. Listen for specific events
|
|
3991
|
+
* eventSource.addEventListener('job.completed', (event) => {
|
|
3992
|
+
* const data = JSON.parse(event.data);
|
|
3993
|
+
* console.log('Job completed!', data.summary);
|
|
3994
|
+
* eventSource.close();
|
|
3995
|
+
* });
|
|
3996
|
+
*
|
|
3997
|
+
* // 5. Clean up
|
|
3998
|
+
* eventSource.close();
|
|
3999
|
+
* ```
|
|
4000
|
+
*/
|
|
4001
|
+
readonly events: EventsResource;
|
|
3224
4002
|
/**
|
|
3225
4003
|
* Create a new DealCrawl client
|
|
3226
4004
|
*
|
|
@@ -3628,4 +4406,352 @@ declare const ERROR_MESSAGES: Record<ErrorCode, string>;
|
|
|
3628
4406
|
*/
|
|
3629
4407
|
declare function getErrorMessage(code: ErrorCode): string;
|
|
3630
4408
|
|
|
3631
|
-
|
|
4409
|
+
/**
|
|
4410
|
+
* Advanced Job Status System
|
|
4411
|
+
*
|
|
4412
|
+
* Production-grade status model with:
|
|
4413
|
+
* - Primary status (lifecycle stage)
|
|
4414
|
+
* - Sub-status (detailed state within stage)
|
|
4415
|
+
* - Reason codes (why this state was reached)
|
|
4416
|
+
* - State machine transitions (valid state changes)
|
|
4417
|
+
*
|
|
4418
|
+
* @module job-status
|
|
4419
|
+
*/
|
|
4420
|
+
/**
|
|
4421
|
+
* Primary job status - represents the main lifecycle stage
|
|
4422
|
+
*/
|
|
4423
|
+
type JobStatus = "queued" | "scheduled" | "priority_queued" | "initializing" | "active" | "processing" | "extracting" | "uploading" | "finalizing" | "retrying" | "backing_off" | "paused" | "pausing" | "resuming" | "completed" | "partial" | "failed" | "timeout" | "cancelled" | "rejected" | "stale" | "zombie" | "stuck";
|
|
4424
|
+
/**
|
|
4425
|
+
* Sub-status provides granular detail within a primary status
|
|
4426
|
+
*/
|
|
4427
|
+
type JobSubStatus = "waiting_for_worker" | "waiting_for_rate_limit" | "waiting_for_quota" | "waiting_for_retry" | "waiting_for_dependency" | "launching_browser" | "establishing_connection" | "loading_config" | "validating_input" | "fetching_page" | "executing_javascript" | "waiting_for_render" | "detecting_signals" | "crawling_links" | "capturing_screenshot" | "calling_llm" | "parsing_response" | "validating_extraction" | "uploading_screenshot" | "uploading_artifacts" | "generating_signed_url" | "saving_to_database" | "cleaning_resources" | "sending_webhook" | "syncing_to_dealup" | "exponential_backoff" | "rate_limit_backoff" | "circuit_breaker_open" | "network_error" | "timeout_error" | "bot_detected" | "captcha_detected" | "auth_required" | "page_not_found" | "server_error" | "parse_error" | "extraction_failed" | "quota_exceeded" | "invalid_input" | "resource_unavailable" | "no_heartbeat" | "progress_stalled" | "worker_lost";
|
|
4428
|
+
/**
|
|
4429
|
+
* Reason codes - why a status/substatus was reached
|
|
4430
|
+
*/
|
|
4431
|
+
type StatusReason = "job_completed_successfully" | "partial_results_available" | "added_to_queue" | "priority_elevated" | "rate_limit_applied" | "quota_check_pending" | "transient_network_error" | "rate_limit_exceeded" | "timeout_exceeded" | "worker_crashed" | "max_retries_not_reached" | "max_retries_exceeded" | "permanent_error" | "blocked_by_site" | "captcha_required" | "invalid_url_format" | "ssrf_detected" | "quota_exhausted" | "tier_restriction" | "resource_not_available" | "cancelled_by_user" | "cancelled_by_system" | "duplicate_detected" | "policy_violation" | "heartbeat_timeout" | "progress_timeout" | "worker_unresponsive";
|
|
4432
|
+
|
|
4433
|
+
/**
|
|
4434
|
+
* Professional structured logger for DealCrawl
|
|
4435
|
+
* - Environment-aware (verbose in dev, minimal in prod)
|
|
4436
|
+
* - Structured JSON output in production
|
|
4437
|
+
* - Colored console output in development
|
|
4438
|
+
* - Correlation ID support for request tracing
|
|
4439
|
+
*/
|
|
4440
|
+
type LogLevel = "debug" | "info" | "warn" | "error";
|
|
4441
|
+
|
|
4442
|
+
/**
|
|
4443
|
+
* SSE (Server-Sent Events) Event Types
|
|
4444
|
+
*
|
|
4445
|
+
* Defines all events that can be pushed to clients via SSE.
|
|
4446
|
+
*
|
|
4447
|
+
* @module sse-events
|
|
4448
|
+
*/
|
|
4449
|
+
|
|
4450
|
+
/**
|
|
4451
|
+
* SSE event types - used as the "event:" field in SSE messages
|
|
4452
|
+
*/
|
|
4453
|
+
type SSEEventType = "job.created" | "job.queued" | "job.started" | "job.status" | "job.progress" | "job.completed" | "job.failed" | "job.cancelled" | "job.paused" | "job.resumed" | "job.log" | "job.metric" | "job.alert" | "job.checkpoint" | "deal.found" | "deal.validated" | "ping" | "connection.open" | "connection.close" | "error";
|
|
4454
|
+
/**
|
|
4455
|
+
* Base SSE event structure - all events extend this
|
|
4456
|
+
*/
|
|
4457
|
+
interface BaseSSEEvent {
|
|
4458
|
+
/** Event type (used in SSE "event:" field) */
|
|
4459
|
+
type: SSEEventType;
|
|
4460
|
+
/** Unique event ID (for Last-Event-ID replay) */
|
|
4461
|
+
id: string;
|
|
4462
|
+
/** When this event was generated */
|
|
4463
|
+
timestamp: string;
|
|
4464
|
+
/** Job ID this event relates to (null for system events) */
|
|
4465
|
+
jobId: string | null;
|
|
4466
|
+
/** Client ID (for server-side filtering) */
|
|
4467
|
+
clientId: string;
|
|
4468
|
+
}
|
|
4469
|
+
/**
|
|
4470
|
+
* Job created event - sent when job is first created
|
|
4471
|
+
*/
|
|
4472
|
+
interface JobCreatedEvent extends BaseSSEEvent {
|
|
4473
|
+
type: "job.created";
|
|
4474
|
+
jobId: string;
|
|
4475
|
+
data: {
|
|
4476
|
+
jobType: "scrape" | "crawl" | "dork" | "extract" | "agent";
|
|
4477
|
+
priority?: "high" | "medium" | "low";
|
|
4478
|
+
template?: string;
|
|
4479
|
+
};
|
|
4480
|
+
}
|
|
4481
|
+
/**
|
|
4482
|
+
* Job queued event - sent when job enters queue
|
|
4483
|
+
*/
|
|
4484
|
+
interface JobQueuedEvent extends BaseSSEEvent {
|
|
4485
|
+
type: "job.queued";
|
|
4486
|
+
jobId: string;
|
|
4487
|
+
data: {
|
|
4488
|
+
queue: string;
|
|
4489
|
+
position?: number;
|
|
4490
|
+
estimatedWaitMs?: number;
|
|
4491
|
+
};
|
|
4492
|
+
}
|
|
4493
|
+
/**
|
|
4494
|
+
* Job started event - sent when worker picks up job
|
|
4495
|
+
*/
|
|
4496
|
+
interface JobStartedEvent extends BaseSSEEvent {
|
|
4497
|
+
type: "job.started";
|
|
4498
|
+
jobId: string;
|
|
4499
|
+
data: {
|
|
4500
|
+
workerId?: string;
|
|
4501
|
+
startedAt: string;
|
|
4502
|
+
};
|
|
4503
|
+
}
|
|
4504
|
+
/**
|
|
4505
|
+
* Job status change event - sent on every status transition
|
|
4506
|
+
*/
|
|
4507
|
+
interface JobStatusEvent extends BaseSSEEvent {
|
|
4508
|
+
type: "job.status";
|
|
4509
|
+
jobId: string;
|
|
4510
|
+
data: {
|
|
4511
|
+
status: JobStatus;
|
|
4512
|
+
subStatus?: JobSubStatus;
|
|
4513
|
+
reason?: StatusReason;
|
|
4514
|
+
message?: string;
|
|
4515
|
+
previousStatus?: JobStatus;
|
|
4516
|
+
progress: number;
|
|
4517
|
+
};
|
|
4518
|
+
}
|
|
4519
|
+
/**
|
|
4520
|
+
* Job progress event - sent periodically during processing
|
|
4521
|
+
*/
|
|
4522
|
+
interface JobProgressEvent extends BaseSSEEvent {
|
|
4523
|
+
type: "job.progress";
|
|
4524
|
+
jobId: string;
|
|
4525
|
+
data: {
|
|
4526
|
+
progress: number;
|
|
4527
|
+
status: JobStatus;
|
|
4528
|
+
subStatus?: JobSubStatus;
|
|
4529
|
+
message?: string;
|
|
4530
|
+
eta?: {
|
|
4531
|
+
remainingMs: number;
|
|
4532
|
+
remainingFormatted: string;
|
|
4533
|
+
};
|
|
4534
|
+
stats?: {
|
|
4535
|
+
pagesProcessed?: number;
|
|
4536
|
+
dealsFound?: number;
|
|
4537
|
+
signalsDetected?: number;
|
|
4538
|
+
};
|
|
4539
|
+
};
|
|
4540
|
+
}
|
|
4541
|
+
/**
|
|
4542
|
+
* Job completed event - sent when job finishes successfully
|
|
4543
|
+
*/
|
|
4544
|
+
interface JobCompletedEvent extends BaseSSEEvent {
|
|
4545
|
+
type: "job.completed";
|
|
4546
|
+
jobId: string;
|
|
4547
|
+
data: {
|
|
4548
|
+
statusUrl: string;
|
|
4549
|
+
completedAt: string;
|
|
4550
|
+
durationMs: number;
|
|
4551
|
+
summary: {
|
|
4552
|
+
success: boolean;
|
|
4553
|
+
pagesProcessed?: number;
|
|
4554
|
+
dealsFound?: number;
|
|
4555
|
+
signalsDetected?: number;
|
|
4556
|
+
hasScreenshot?: boolean;
|
|
4557
|
+
};
|
|
4558
|
+
result?: {
|
|
4559
|
+
dealScore?: number;
|
|
4560
|
+
topSignals?: string[];
|
|
4561
|
+
dealTitle?: string;
|
|
4562
|
+
};
|
|
4563
|
+
};
|
|
4564
|
+
}
|
|
4565
|
+
/**
|
|
4566
|
+
* Job failed event - sent when job fails
|
|
4567
|
+
*/
|
|
4568
|
+
interface JobFailedEvent extends BaseSSEEvent {
|
|
4569
|
+
type: "job.failed";
|
|
4570
|
+
jobId: string;
|
|
4571
|
+
data: {
|
|
4572
|
+
statusUrl: string;
|
|
4573
|
+
failedAt: string;
|
|
4574
|
+
durationMs: number;
|
|
4575
|
+
error: {
|
|
4576
|
+
code: string;
|
|
4577
|
+
message: string;
|
|
4578
|
+
recoverable: boolean;
|
|
4579
|
+
};
|
|
4580
|
+
retry?: {
|
|
4581
|
+
attempt: number;
|
|
4582
|
+
maxAttempts: number;
|
|
4583
|
+
willRetry: boolean;
|
|
4584
|
+
nextRetryAt?: string;
|
|
4585
|
+
};
|
|
4586
|
+
};
|
|
4587
|
+
}
|
|
4588
|
+
/**
|
|
4589
|
+
* Job cancelled event - sent when job is cancelled
|
|
4590
|
+
*/
|
|
4591
|
+
interface JobCancelledEvent extends BaseSSEEvent {
|
|
4592
|
+
type: "job.cancelled";
|
|
4593
|
+
jobId: string;
|
|
4594
|
+
data: {
|
|
4595
|
+
cancelledAt: string;
|
|
4596
|
+
cancelledBy: "user" | "system";
|
|
4597
|
+
reason?: string;
|
|
4598
|
+
};
|
|
4599
|
+
}
|
|
4600
|
+
/**
|
|
4601
|
+
* Job paused event
|
|
4602
|
+
*/
|
|
4603
|
+
interface JobPausedEvent extends BaseSSEEvent {
|
|
4604
|
+
type: "job.paused";
|
|
4605
|
+
jobId: string;
|
|
4606
|
+
data: {
|
|
4607
|
+
pausedAt: string;
|
|
4608
|
+
checkpoint?: {
|
|
4609
|
+
lastProcessedUrl: string;
|
|
4610
|
+
urlsProcessed: number;
|
|
4611
|
+
urlsRemaining: number;
|
|
4612
|
+
};
|
|
4613
|
+
};
|
|
4614
|
+
}
|
|
4615
|
+
/**
|
|
4616
|
+
* Job resumed event
|
|
4617
|
+
*/
|
|
4618
|
+
interface JobResumedEvent extends BaseSSEEvent {
|
|
4619
|
+
type: "job.resumed";
|
|
4620
|
+
jobId: string;
|
|
4621
|
+
data: {
|
|
4622
|
+
resumedAt: string;
|
|
4623
|
+
fromCheckpoint: boolean;
|
|
4624
|
+
};
|
|
4625
|
+
}
|
|
4626
|
+
/**
|
|
4627
|
+
* Job log event - sent for important log messages
|
|
4628
|
+
*/
|
|
4629
|
+
interface JobLogEvent extends BaseSSEEvent {
|
|
4630
|
+
type: "job.log";
|
|
4631
|
+
jobId: string;
|
|
4632
|
+
data: {
|
|
4633
|
+
level: LogLevel;
|
|
4634
|
+
message: string;
|
|
4635
|
+
metadata?: Record<string, unknown>;
|
|
4636
|
+
};
|
|
4637
|
+
}
|
|
4638
|
+
/**
|
|
4639
|
+
* Job metric event - sent for performance/business metrics
|
|
4640
|
+
*/
|
|
4641
|
+
interface JobMetricEvent extends BaseSSEEvent {
|
|
4642
|
+
type: "job.metric";
|
|
4643
|
+
jobId: string;
|
|
4644
|
+
data: {
|
|
4645
|
+
metric: string;
|
|
4646
|
+
value: number;
|
|
4647
|
+
unit?: string;
|
|
4648
|
+
};
|
|
4649
|
+
}
|
|
4650
|
+
/**
|
|
4651
|
+
* Job alert event - sent for important alerts (quota warnings, etc.)
|
|
4652
|
+
*/
|
|
4653
|
+
interface JobAlertEvent extends BaseSSEEvent {
|
|
4654
|
+
type: "job.alert";
|
|
4655
|
+
jobId: string;
|
|
4656
|
+
data: {
|
|
4657
|
+
severity: "info" | "warning" | "error" | "critical";
|
|
4658
|
+
title: string;
|
|
4659
|
+
message: string;
|
|
4660
|
+
actionRequired?: boolean;
|
|
4661
|
+
};
|
|
4662
|
+
}
|
|
4663
|
+
/**
|
|
4664
|
+
* Job checkpoint event - sent when checkpoint is saved
|
|
4665
|
+
*/
|
|
4666
|
+
interface JobCheckpointEvent extends BaseSSEEvent {
|
|
4667
|
+
type: "job.checkpoint";
|
|
4668
|
+
jobId: string;
|
|
4669
|
+
data: {
|
|
4670
|
+
savedAt: string;
|
|
4671
|
+
lastProcessedUrl: string;
|
|
4672
|
+
urlsProcessed: number;
|
|
4673
|
+
urlsRemaining: number;
|
|
4674
|
+
canResume: boolean;
|
|
4675
|
+
};
|
|
4676
|
+
}
|
|
4677
|
+
/**
|
|
4678
|
+
* Deal found event - sent when a deal is detected
|
|
4679
|
+
*/
|
|
4680
|
+
interface DealFoundEvent extends BaseSSEEvent {
|
|
4681
|
+
type: "deal.found";
|
|
4682
|
+
jobId: string;
|
|
4683
|
+
data: {
|
|
4684
|
+
dealId?: string;
|
|
4685
|
+
url: string;
|
|
4686
|
+
title: string;
|
|
4687
|
+
score: number;
|
|
4688
|
+
price?: number;
|
|
4689
|
+
discount?: number;
|
|
4690
|
+
category?: string;
|
|
4691
|
+
signals: string[];
|
|
4692
|
+
};
|
|
4693
|
+
}
|
|
4694
|
+
/**
|
|
4695
|
+
* Deal validated event - sent when deal is validated/scored
|
|
4696
|
+
*/
|
|
4697
|
+
interface DealValidatedEvent extends BaseSSEEvent {
|
|
4698
|
+
type: "deal.validated";
|
|
4699
|
+
jobId: string;
|
|
4700
|
+
data: {
|
|
4701
|
+
dealId: string;
|
|
4702
|
+
score: number;
|
|
4703
|
+
confidence: number;
|
|
4704
|
+
accepted: boolean;
|
|
4705
|
+
rejectionReason?: string;
|
|
4706
|
+
};
|
|
4707
|
+
}
|
|
4708
|
+
/**
|
|
4709
|
+
* Ping event - keepalive to prevent connection timeout
|
|
4710
|
+
*/
|
|
4711
|
+
interface PingEvent extends BaseSSEEvent {
|
|
4712
|
+
type: "ping";
|
|
4713
|
+
jobId: null;
|
|
4714
|
+
data: {
|
|
4715
|
+
serverTime: string;
|
|
4716
|
+
};
|
|
4717
|
+
}
|
|
4718
|
+
/**
|
|
4719
|
+
* Connection open event - sent when SSE connection established
|
|
4720
|
+
*/
|
|
4721
|
+
interface ConnectionOpenEvent extends BaseSSEEvent {
|
|
4722
|
+
type: "connection.open";
|
|
4723
|
+
jobId: null;
|
|
4724
|
+
data: {
|
|
4725
|
+
clientId: string;
|
|
4726
|
+
reconnected: boolean;
|
|
4727
|
+
lastEventId?: string;
|
|
4728
|
+
};
|
|
4729
|
+
}
|
|
4730
|
+
/**
|
|
4731
|
+
* Connection close event - sent before connection closes
|
|
4732
|
+
*/
|
|
4733
|
+
interface ConnectionCloseEvent extends BaseSSEEvent {
|
|
4734
|
+
type: "connection.close";
|
|
4735
|
+
jobId: null;
|
|
4736
|
+
data: {
|
|
4737
|
+
reason: string;
|
|
4738
|
+
};
|
|
4739
|
+
}
|
|
4740
|
+
/**
|
|
4741
|
+
* Error event - sent when an error occurs
|
|
4742
|
+
*/
|
|
4743
|
+
interface ErrorEvent extends BaseSSEEvent {
|
|
4744
|
+
type: "error";
|
|
4745
|
+
jobId: string | null;
|
|
4746
|
+
data: {
|
|
4747
|
+
code: string;
|
|
4748
|
+
message: string;
|
|
4749
|
+
recoverable: boolean;
|
|
4750
|
+
};
|
|
4751
|
+
}
|
|
4752
|
+
/**
|
|
4753
|
+
* All possible SSE events (discriminated union)
|
|
4754
|
+
*/
|
|
4755
|
+
type SSEEvent = JobCreatedEvent | JobQueuedEvent | JobStartedEvent | JobStatusEvent | JobProgressEvent | JobCompletedEvent | JobFailedEvent | JobCancelledEvent | JobPausedEvent | JobResumedEvent | JobLogEvent | JobMetricEvent | JobAlertEvent | JobCheckpointEvent | DealFoundEvent | DealValidatedEvent | PingEvent | ConnectionOpenEvent | ConnectionCloseEvent | ErrorEvent;
|
|
4756
|
+
|
|
4757
|
+
export { ALL_API_KEY_SCOPES, type AccountInfoResponse, type AccountMetricsResponse, AccountResource, type ActionBaseOptions, type ActionInput, type AgentAction, type AgentActionType, type AgentCompletionReason, type AgentJobResponse, type AgentModel, type AgentOptions, AgentResource, type AgentResultResponse, type AgentStatusResponse, type AgentStepResponse, type ApiError, type ApiKeyInfo, type ApiKeyScope, type ApiResponse, AuthResource, type BaseSSEEvent, type BatchScrapeDefaults, type BatchScrapeItem, type BatchScrapeOptions, type BatchScrapeResponse, type BatchScrapeResultItem, type BatchStatusResponse, type CancelJobResponse, type CheckpointInfo, type ClickAction, type ClientPreferences, type ClientStatsResponse, type ConnectionCloseEvent, type ConnectionOpenEvent, type ConversionMetadata, type ConvertOptions, ConvertResource, type ConvertResponse, type CrawlAnalysisResponse, type CrawlError, type CrawlJobResponse, type CrawlMode, type CrawlOptions, type CrawlRecommendation, CrawlResource, type CrawlResult, type CrawlStats, type CrawlTemplate, type CrawlTemplateId, type CreateApiKeyOptions, type CreateKeyResponse, type CreateWebhookOptions, type CreateWebhookResponse, type CreatedApiKey, DEFAULT_API_KEY_SCOPES, DEFAULT_CONFIG, DataResource, DealCrawl, type DealCrawlConfig, DealCrawlError, type DealDetails, type DealFoundEvent, type DealItem, type DealMetrics, type DealScoreSummary, type DealSummary, type DealUpMetrics, type DealUpMetricsResponse, type DealValidatedEvent, type DeleteKeyResponse, type DeleteWebhookResponse, type DiscountSignal, type DorkJobResponse, type DorkOptions, DorkResource, type DorkResult, ERROR_CODES, ERROR_MESSAGES, type EngineSelection, type EngineType, type ErrorCode, type ErrorEvent, EventsResource, type ExportDealsOptions, type ExportFormat, type ExportJobsOptions, type ExtractJobResponse, type ExtractModel, type ExtractOptions, ExtractResource, type ExtractedDeal, type ExtractionErrorDetails, type FallbackConfig, type FallbackMetadata, type FallbackResult, type FallbackSource, type GenerateSSETokenOptions, type GetApiKeyStatsOptions, type GetDealsOptions, type HoverAction, type JobAlertEvent, type JobCancelledEvent, type JobCheckpointEvent, type JobCompletedEvent, type JobCreatedEvent, type JobDealsResponse, type JobFailedEvent, type JobLogEvent, type JobMetricEvent, type JobMetricsResponse, type JobPausedEvent, type JobProgressEvent, type JobQueuedEvent, type JobResponse, type JobResumedEvent, type JobStartedEvent, type JobStatus$1 as JobStatus, type JobStatusEvent, type JobStatusFilter, type JobStatusResponse, type JobSummary, type JobTypeFilter, type KeyStatsResponse, KeysResource, type ListApiKeysOptions, type ListDealsOptions, type ListDealsResponse, type ListJobsOptions, type ListJobsResponse, type ListKeysResponse, type ListWebhooksResponse, type PaginatedResponse, type PaginationInfo, type ParsedPage, type PingEvent, type PreferencesResponse, type PressAction, type PriceSignal, type PricingInfo, type ProductCategory, type ProductInfo, type RateLimitInfo, type RecommendationsResponse, type RefreshScreenshotOptions, type RequestContext, type ResumeJobResponse, type RevokeApiKeyOptions, type RotateApiKeyOptions, type RotateKeyResponse, type SSEConnectionLimitsResponse, type SSEEvent, type SSEEventType, type SSESubscribeOptions, type SSETokenResponse, type ScrapeJobResponse, type ScrapeOptions, ScrapeResource, type ScrapeResult, type ScreenshotAction, type ScreenshotAgentAction, type ScreenshotLimitsResponse, type ScreenshotOptions, type ScreenshotRefreshResponse, type ScreenshotResult, ScreenshotsResource, type ScrollAction, type SearchAiModel, type SearchAiProvider, type SearchData, type SearchDateRange, type SearchFilters, type SearchJobResponse, type SearchOptions, SearchResource, type SearchResultItem, type SearchStatusResponse, type SelectAction, type ClickAction$1 as SharedClickAction, type HoverAction$1 as SharedHoverAction, type PressAction$1 as SharedPressAction, type ScrollAction$1 as SharedScrollAction, type SelectAction$1 as SharedSelectAction, type WaitAction$1 as SharedWaitAction, type WriteAction$1 as SharedWriteAction, type Signal, type SortOrder, StatusResource, type TestWebhookResponse, type UpdatePreferencesOptions, type UpdatePreferencesResponse, type UpdateWebhookOptions, type UpdateWebhookResponse, type UrgencyLevel, type UsageStats, type ValidationError, type ValidationResult, type WaitAction, type WaitOptions, type WaitResult, type WebhookEvent, type WebhookItem, WebhooksResource, type WriteAction, DealCrawl as default, getErrorMessage, pollUntil, waitForAll, waitForAny, waitForResult };
|