@dealcrawl/sdk 2.10.0 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +223 -37
- package/dist/index.d.mts +1145 -26
- package/dist/index.d.ts +1145 -26
- package/dist/index.js +631 -9
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +628 -10
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.d.mts
CHANGED
|
@@ -19,7 +19,7 @@ interface ResponseMeta {
|
|
|
19
19
|
timestamp: string;
|
|
20
20
|
duration?: number;
|
|
21
21
|
}
|
|
22
|
-
type JobStatus = "pending" | "active" | "completed" | "failed" | "delayed" | "paused";
|
|
22
|
+
type JobStatus$1 = "pending" | "active" | "completed" | "failed" | "delayed" | "paused";
|
|
23
23
|
interface PaginatedResponse<T> {
|
|
24
24
|
data: T[];
|
|
25
25
|
pagination: {
|
|
@@ -130,10 +130,19 @@ interface DealScoreSummary {
|
|
|
130
130
|
};
|
|
131
131
|
}
|
|
132
132
|
interface ScreenshotResult {
|
|
133
|
+
/** Screenshot URL - private signed URL (default) or public URL (Enterprise opt-in) */
|
|
133
134
|
url: string;
|
|
135
|
+
/** Whether this is a public URL (false = private signed URL with TTL) */
|
|
136
|
+
isPublic: boolean;
|
|
137
|
+
/** Expiration timestamp for signed URLs (ISO 8601, only for private URLs) */
|
|
138
|
+
expiresAt?: string;
|
|
139
|
+
/** Screenshot width in pixels */
|
|
134
140
|
width: number;
|
|
141
|
+
/** Screenshot height in pixels */
|
|
135
142
|
height: number;
|
|
143
|
+
/** Image format */
|
|
136
144
|
format: "png" | "jpeg" | "webp";
|
|
145
|
+
/** File size in bytes */
|
|
137
146
|
sizeBytes: number;
|
|
138
147
|
}
|
|
139
148
|
interface AIExtraction {
|
|
@@ -438,7 +447,7 @@ interface JobResponse {
|
|
|
438
447
|
/** Unique job identifier */
|
|
439
448
|
jobId: string;
|
|
440
449
|
/** Current job status */
|
|
441
|
-
status: JobStatus;
|
|
450
|
+
status: JobStatus$1;
|
|
442
451
|
/** URL to check job status */
|
|
443
452
|
statusUrl: string;
|
|
444
453
|
/** Estimated time to completion */
|
|
@@ -468,6 +477,15 @@ interface BatchScrapeResultItem {
|
|
|
468
477
|
/** Error message if job creation failed */
|
|
469
478
|
error?: string;
|
|
470
479
|
}
|
|
480
|
+
/** Invalid URL entry (returned when ignoreInvalidURLs=true) */
|
|
481
|
+
interface InvalidURLEntry {
|
|
482
|
+
/** The invalid URL */
|
|
483
|
+
url: string;
|
|
484
|
+
/** Reference ID if provided */
|
|
485
|
+
ref?: string;
|
|
486
|
+
/** Reason for rejection */
|
|
487
|
+
reason: string;
|
|
488
|
+
}
|
|
471
489
|
/** Batch scrape response */
|
|
472
490
|
interface BatchScrapeResponse {
|
|
473
491
|
/** Batch ID for tracking */
|
|
@@ -482,6 +500,11 @@ interface BatchScrapeResponse {
|
|
|
482
500
|
results: BatchScrapeResultItem[];
|
|
483
501
|
/** Timestamp when batch was created */
|
|
484
502
|
createdAt: string;
|
|
503
|
+
/**
|
|
504
|
+
* Invalid URLs that were skipped (only present when ignoreInvalidURLs=true)
|
|
505
|
+
* Firecrawl-compatible field
|
|
506
|
+
*/
|
|
507
|
+
invalidURLs?: InvalidURLEntry[];
|
|
485
508
|
}
|
|
486
509
|
/** Batch status response */
|
|
487
510
|
interface BatchStatusResponse {
|
|
@@ -502,7 +525,7 @@ interface BatchStatusResponse {
|
|
|
502
525
|
/** Individual job statuses */
|
|
503
526
|
jobs: Array<{
|
|
504
527
|
id: string;
|
|
505
|
-
status: JobStatus;
|
|
528
|
+
status: JobStatus$1;
|
|
506
529
|
result?: unknown;
|
|
507
530
|
error?: string;
|
|
508
531
|
}>;
|
|
@@ -612,7 +635,7 @@ interface CheckpointInfo {
|
|
|
612
635
|
/** Job status response */
|
|
613
636
|
interface JobStatusResponse {
|
|
614
637
|
jobId: string;
|
|
615
|
-
status: JobStatus;
|
|
638
|
+
status: JobStatus$1;
|
|
616
639
|
progress?: number;
|
|
617
640
|
createdAt: string;
|
|
618
641
|
completedAt?: string;
|
|
@@ -652,7 +675,7 @@ interface ResumeJobResponse {
|
|
|
652
675
|
/** Job metrics response */
|
|
653
676
|
interface JobMetricsResponse {
|
|
654
677
|
jobId: string;
|
|
655
|
-
status: JobStatus;
|
|
678
|
+
status: JobStatus$1;
|
|
656
679
|
metrics: {
|
|
657
680
|
totalUrls: number;
|
|
658
681
|
successfulUrls: number;
|
|
@@ -671,7 +694,7 @@ interface CancelJobResponse {
|
|
|
671
694
|
interface JobSummary {
|
|
672
695
|
id: string;
|
|
673
696
|
type: "scrape" | "crawl" | "dork" | "extract";
|
|
674
|
-
status: JobStatus;
|
|
697
|
+
status: JobStatus$1;
|
|
675
698
|
input: Record<string, unknown>;
|
|
676
699
|
dealsFound?: number;
|
|
677
700
|
avgDealScore?: number;
|
|
@@ -964,7 +987,7 @@ interface AgentStepResponse {
|
|
|
964
987
|
currentUrl?: string;
|
|
965
988
|
}
|
|
966
989
|
/** Agent completion reason */
|
|
967
|
-
type AgentCompletionReason = "goal_achieved" | "max_steps_reached" | "stuck" | "error"
|
|
990
|
+
type AgentCompletionReason = "goal_achieved" | "max_steps_reached" | "stuck" | "error";
|
|
968
991
|
/** Agent result (from completed job) */
|
|
969
992
|
interface AgentResultResponse {
|
|
970
993
|
/** Extracted/collected data (matches provided schema if given) */
|
|
@@ -1013,6 +1036,113 @@ interface SchemaGenerationResponse {
|
|
|
1013
1036
|
/** Confidence score (0-1) in the generated schema */
|
|
1014
1037
|
confidence: number;
|
|
1015
1038
|
}
|
|
1039
|
+
/** SSE token response */
|
|
1040
|
+
interface SSETokenResponse {
|
|
1041
|
+
/** Short-lived SSE authentication token */
|
|
1042
|
+
token: string;
|
|
1043
|
+
/** Expiration timestamp (ISO 8601) */
|
|
1044
|
+
expiresAt: string;
|
|
1045
|
+
/** TTL in seconds (default: 300 = 5 minutes) */
|
|
1046
|
+
expiresIn: number;
|
|
1047
|
+
/** Token type identifier */
|
|
1048
|
+
type: "sse_token";
|
|
1049
|
+
/** Usage instructions for the token */
|
|
1050
|
+
usage: string;
|
|
1051
|
+
}
|
|
1052
|
+
/** SSE connection limits response */
|
|
1053
|
+
interface SSEConnectionLimitsResponse {
|
|
1054
|
+
/** Client tier */
|
|
1055
|
+
tier: "free" | "pro" | "enterprise";
|
|
1056
|
+
/** SSE connection limits */
|
|
1057
|
+
sse: {
|
|
1058
|
+
/** Maximum concurrent connections allowed */
|
|
1059
|
+
maxConnections: number;
|
|
1060
|
+
/** Current active connections */
|
|
1061
|
+
currentConnections: number;
|
|
1062
|
+
/** Available connection slots */
|
|
1063
|
+
available: number;
|
|
1064
|
+
};
|
|
1065
|
+
}
|
|
1066
|
+
/** Tier-specific TTL limits */
|
|
1067
|
+
interface TtlLimits {
|
|
1068
|
+
/** Minimum TTL in seconds */
|
|
1069
|
+
min: number;
|
|
1070
|
+
/** Maximum TTL in seconds */
|
|
1071
|
+
max: number;
|
|
1072
|
+
/** Default TTL in seconds */
|
|
1073
|
+
default: number;
|
|
1074
|
+
}
|
|
1075
|
+
/** Formatted TTL limits (human-readable) */
|
|
1076
|
+
interface FormattedTtlLimits {
|
|
1077
|
+
/** Minimum TTL formatted (e.g., "1 hour") */
|
|
1078
|
+
min: string;
|
|
1079
|
+
/** Maximum TTL formatted (e.g., "30 days") */
|
|
1080
|
+
max: string;
|
|
1081
|
+
/** Default TTL formatted (e.g., "7 days") */
|
|
1082
|
+
default: string;
|
|
1083
|
+
}
|
|
1084
|
+
/** Screenshot signed URL refresh response */
|
|
1085
|
+
interface ScreenshotRefreshResponse {
|
|
1086
|
+
/** Success status */
|
|
1087
|
+
success: boolean;
|
|
1088
|
+
/** New signed URL */
|
|
1089
|
+
url: string;
|
|
1090
|
+
/** Expiration timestamp (ISO 8601) */
|
|
1091
|
+
expiresAt: string;
|
|
1092
|
+
/** TTL in seconds */
|
|
1093
|
+
ttl: number;
|
|
1094
|
+
/** Tier-specific limits for reference */
|
|
1095
|
+
tierLimits: TtlLimits;
|
|
1096
|
+
/** Additional metadata */
|
|
1097
|
+
metadata: {
|
|
1098
|
+
/** File path in storage */
|
|
1099
|
+
path: string;
|
|
1100
|
+
/** Bucket name */
|
|
1101
|
+
bucket: string;
|
|
1102
|
+
/** Timestamp when URL was refreshed */
|
|
1103
|
+
refreshedAt: string;
|
|
1104
|
+
};
|
|
1105
|
+
}
|
|
1106
|
+
/** Screenshot TTL limits response */
|
|
1107
|
+
interface ScreenshotLimitsResponse {
|
|
1108
|
+
/** Client tier */
|
|
1109
|
+
tier: "free" | "pro" | "enterprise";
|
|
1110
|
+
/** TTL limits in seconds */
|
|
1111
|
+
limits: TtlLimits;
|
|
1112
|
+
/** Human-readable formatted limits */
|
|
1113
|
+
formattedLimits: FormattedTtlLimits;
|
|
1114
|
+
}
|
|
1115
|
+
/**
|
|
1116
|
+
* Metadata extracted during HTML to Markdown conversion
|
|
1117
|
+
*/
|
|
1118
|
+
interface ConversionMetadata {
|
|
1119
|
+
/** Page title if found */
|
|
1120
|
+
title?: string;
|
|
1121
|
+
/** Word count in resulting markdown */
|
|
1122
|
+
wordCount: number;
|
|
1123
|
+
/** Number of links in markdown */
|
|
1124
|
+
linkCount: number;
|
|
1125
|
+
/** Number of images in markdown */
|
|
1126
|
+
imageCount: number;
|
|
1127
|
+
/** Conversion duration in milliseconds */
|
|
1128
|
+
conversionTimeMs: number;
|
|
1129
|
+
}
|
|
1130
|
+
/**
|
|
1131
|
+
* HTML to Markdown conversion response
|
|
1132
|
+
*/
|
|
1133
|
+
interface ConvertResponse {
|
|
1134
|
+
/** Success status */
|
|
1135
|
+
success: boolean;
|
|
1136
|
+
/** Conversion result data */
|
|
1137
|
+
data: {
|
|
1138
|
+
/** Generated markdown content */
|
|
1139
|
+
markdown: string;
|
|
1140
|
+
/** Extraction metadata */
|
|
1141
|
+
metadata: ConversionMetadata;
|
|
1142
|
+
/** Warnings (truncation, excluded elements, etc.) */
|
|
1143
|
+
warnings?: string[];
|
|
1144
|
+
};
|
|
1145
|
+
}
|
|
1016
1146
|
|
|
1017
1147
|
/**
|
|
1018
1148
|
* Polling Utilities
|
|
@@ -1092,10 +1222,19 @@ interface ScreenshotOptions {
|
|
|
1092
1222
|
/**
|
|
1093
1223
|
* Request public URL (Enterprise only, default: false)
|
|
1094
1224
|
* SECURITY WARNING: Public URLs can expose personal data, copyrighted content, and tokens
|
|
1095
|
-
* By default, screenshots use private signed URLs with
|
|
1225
|
+
* By default, screenshots use private signed URLs with tier-specific expiration
|
|
1096
1226
|
* Requires Enterprise tier
|
|
1097
1227
|
*/
|
|
1098
1228
|
publicUrl?: boolean;
|
|
1229
|
+
/**
|
|
1230
|
+
* TTL for signed URLs in seconds (only for private screenshots)
|
|
1231
|
+
* Tier limits:
|
|
1232
|
+
* - Free: 3600 (1h) - 86400 (24h), default: 86400
|
|
1233
|
+
* - Pro: 3600 (1h) - 604800 (7d), default: 604800
|
|
1234
|
+
* - Enterprise: 3600 (1h) - 2592000 (30d), default: 604800
|
|
1235
|
+
* If not specified, uses tier-specific default
|
|
1236
|
+
*/
|
|
1237
|
+
signedUrlTtl?: number;
|
|
1099
1238
|
}
|
|
1100
1239
|
/** Options for scraping a single page */
|
|
1101
1240
|
interface ScrapeOptions {
|
|
@@ -1158,8 +1297,6 @@ interface BatchScrapeItem {
|
|
|
1158
1297
|
outputMarkdown?: boolean;
|
|
1159
1298
|
/** Override markdownBaseUrl for this URL */
|
|
1160
1299
|
markdownBaseUrl?: string;
|
|
1161
|
-
/** Override actions for this URL */
|
|
1162
|
-
actions?: ActionInput[];
|
|
1163
1300
|
}
|
|
1164
1301
|
/** Default options applied to all URLs in a batch */
|
|
1165
1302
|
interface BatchScrapeDefaults {
|
|
@@ -1195,8 +1332,6 @@ interface BatchScrapeDefaults {
|
|
|
1195
1332
|
outputMarkdown?: boolean;
|
|
1196
1333
|
/** Base URL for resolving relative URLs in markdown */
|
|
1197
1334
|
markdownBaseUrl?: string;
|
|
1198
|
-
/** Browser actions to execute before scraping */
|
|
1199
|
-
actions?: ActionInput[];
|
|
1200
1335
|
}
|
|
1201
1336
|
/** Options for batch scraping multiple URLs */
|
|
1202
1337
|
interface BatchScrapeOptions {
|
|
@@ -1208,8 +1343,13 @@ interface BatchScrapeOptions {
|
|
|
1208
1343
|
webhookUrl?: string;
|
|
1209
1344
|
/** Priority for batch jobs (1-10, higher = faster) */
|
|
1210
1345
|
priority?: number;
|
|
1211
|
-
/** Delay between job submissions in ms (0-5000) */
|
|
1212
|
-
|
|
1346
|
+
/** Delay between job submissions in ms (0-5000, default: 0) */
|
|
1347
|
+
delayMs?: number;
|
|
1348
|
+
/**
|
|
1349
|
+
* Continue processing even if some URLs are invalid (default: false)
|
|
1350
|
+
* Invalid URLs will be returned in the response with status: "failed"
|
|
1351
|
+
*/
|
|
1352
|
+
ignoreInvalidURLs?: boolean;
|
|
1213
1353
|
}
|
|
1214
1354
|
/** AI provider for search optimization */
|
|
1215
1355
|
type SearchAiProvider = "openai" | "anthropic";
|
|
@@ -1237,11 +1377,11 @@ interface SearchOptions {
|
|
|
1237
1377
|
/** Search query (required) */
|
|
1238
1378
|
query: string;
|
|
1239
1379
|
/** Maximum number of results (1-100, default: 10) */
|
|
1240
|
-
|
|
1241
|
-
/** Auto-scrape
|
|
1242
|
-
|
|
1243
|
-
/**
|
|
1244
|
-
|
|
1380
|
+
limit?: number;
|
|
1381
|
+
/** Auto-scrape search results (default: false) */
|
|
1382
|
+
scrapeResults?: boolean;
|
|
1383
|
+
/** Maximum results to auto-scrape when scrapeResults=true (1-10, default: 5) */
|
|
1384
|
+
maxScrapeResults?: number;
|
|
1245
1385
|
/** Use AI to optimize the search query */
|
|
1246
1386
|
useAiOptimization?: boolean;
|
|
1247
1387
|
/** AI provider for query optimization */
|
|
@@ -1352,7 +1492,7 @@ interface CrawlTemplate {
|
|
|
1352
1492
|
defaultOptions: Partial<CrawlOptions>;
|
|
1353
1493
|
}
|
|
1354
1494
|
/** LLM model options for extraction */
|
|
1355
|
-
type ExtractModel = "gpt-4o-mini" | "gpt-4o" | "claude-3-haiku" | "claude-3-sonnet";
|
|
1495
|
+
type ExtractModel = "gpt-4o-mini" | "gpt-4o" | "gpt-4o-2024-11-20" | "o1-mini" | "claude-3-5-haiku-20241022" | "claude-3-5-sonnet-20241022" | "claude-3-opus-20240229";
|
|
1356
1496
|
/** Options for structured data extraction */
|
|
1357
1497
|
interface ExtractOptions {
|
|
1358
1498
|
/** URL to extract data from (required) */
|
|
@@ -1513,7 +1653,7 @@ interface UpdateWebhookOptions {
|
|
|
1513
1653
|
* API key scope - Must match @dealcrawl/shared/src/types/api-key.types.ts
|
|
1514
1654
|
* These are the actual scopes enforced by the backend via requireScope() middleware
|
|
1515
1655
|
*/
|
|
1516
|
-
type ApiKeyScope = "scrape" | "
|
|
1656
|
+
type ApiKeyScope = "scrape" | "crawl" | "dork" | "extract" | "agent" | "status" | "data:read" | "data:export" | "keys:manage" | "webhooks:manage";
|
|
1517
1657
|
/**
|
|
1518
1658
|
* All available scopes (for reference and validation)
|
|
1519
1659
|
*/
|
|
@@ -1691,6 +1831,71 @@ interface SchemaGenerationOptions {
|
|
|
1691
1831
|
/** LLM provider for generation (default: openai) */
|
|
1692
1832
|
model?: AgentModel;
|
|
1693
1833
|
}
|
|
1834
|
+
/**
|
|
1835
|
+
* Options for subscribing to SSE events
|
|
1836
|
+
*/
|
|
1837
|
+
interface SSESubscribeOptions {
|
|
1838
|
+
/**
|
|
1839
|
+
* Last event ID for reconnection/replay
|
|
1840
|
+
* If provided, server will replay missed events since this ID
|
|
1841
|
+
*/
|
|
1842
|
+
lastEventId?: string;
|
|
1843
|
+
/**
|
|
1844
|
+
* Callback for received events
|
|
1845
|
+
* Called for each event received from the server
|
|
1846
|
+
*/
|
|
1847
|
+
onEvent?: (event: MessageEvent) => void;
|
|
1848
|
+
/**
|
|
1849
|
+
* Callback for errors
|
|
1850
|
+
* Called when connection error occurs
|
|
1851
|
+
*/
|
|
1852
|
+
onError?: (error: Error) => void;
|
|
1853
|
+
/**
|
|
1854
|
+
* Callback for connection open
|
|
1855
|
+
* Called when SSE connection is established
|
|
1856
|
+
*/
|
|
1857
|
+
onOpen?: (event: Event) => void;
|
|
1858
|
+
/**
|
|
1859
|
+
* Auto-reconnect on connection loss (default: true)
|
|
1860
|
+
* If true, EventSource will automatically reconnect
|
|
1861
|
+
*/
|
|
1862
|
+
autoReconnect?: boolean;
|
|
1863
|
+
/**
|
|
1864
|
+
* Reconnection delay in milliseconds (default: 3000)
|
|
1865
|
+
* How long to wait before attempting reconnection
|
|
1866
|
+
*/
|
|
1867
|
+
reconnectDelay?: number;
|
|
1868
|
+
}
|
|
1869
|
+
/**
|
|
1870
|
+
* Options for HTML to Markdown conversion
|
|
1871
|
+
*/
|
|
1872
|
+
interface ConvertOptions {
|
|
1873
|
+
/** HTML content to convert (required, max 10MB) */
|
|
1874
|
+
html: string;
|
|
1875
|
+
/** Base URL for resolving relative URLs */
|
|
1876
|
+
baseUrl?: string;
|
|
1877
|
+
/** Conversion options */
|
|
1878
|
+
options?: {
|
|
1879
|
+
/** Enable GFM tables (default: true) */
|
|
1880
|
+
gfmTables?: boolean;
|
|
1881
|
+
/** Enable GFM strikethrough (default: true) */
|
|
1882
|
+
gfmStrikethrough?: boolean;
|
|
1883
|
+
/** Enable GFM task lists (default: true) */
|
|
1884
|
+
gfmTaskLists?: boolean;
|
|
1885
|
+
/** Remove noise elements like nav, footer, ads (default: true) */
|
|
1886
|
+
removeNoise?: boolean;
|
|
1887
|
+
/** CSS selectors to exclude from conversion */
|
|
1888
|
+
excludeSelectors?: string[];
|
|
1889
|
+
/** Resolve relative URLs to absolute (default: true) */
|
|
1890
|
+
absoluteUrls?: boolean;
|
|
1891
|
+
/** Maximum output length in characters (0 = unlimited, max: 1,000,000) */
|
|
1892
|
+
maxLength?: number;
|
|
1893
|
+
/** Include images in output (default: true) */
|
|
1894
|
+
includeImages?: boolean;
|
|
1895
|
+
/** Include links in output (default: true) */
|
|
1896
|
+
includeLinks?: boolean;
|
|
1897
|
+
};
|
|
1898
|
+
}
|
|
1694
1899
|
|
|
1695
1900
|
/**
|
|
1696
1901
|
* Account Resource
|
|
@@ -2035,6 +2240,257 @@ declare class AgentResource {
|
|
|
2035
2240
|
generateSchema(options: SchemaGenerationOptions): Promise<SchemaGenerationResponse>;
|
|
2036
2241
|
}
|
|
2037
2242
|
|
|
2243
|
+
/**
|
|
2244
|
+
* Auth Resource
|
|
2245
|
+
* Handles SSE (Server-Sent Events) authentication
|
|
2246
|
+
*/
|
|
2247
|
+
|
|
2248
|
+
/**
|
|
2249
|
+
* Options for generating an SSE token
|
|
2250
|
+
*/
|
|
2251
|
+
interface GenerateSSETokenOptions {
|
|
2252
|
+
/**
|
|
2253
|
+
* Optional job ID to restrict token to a specific job
|
|
2254
|
+
* If provided, token can only be used for /v1/events/:jobId
|
|
2255
|
+
* If omitted, token can be used for /v1/events (all events)
|
|
2256
|
+
*/
|
|
2257
|
+
jobId?: string;
|
|
2258
|
+
}
|
|
2259
|
+
/**
|
|
2260
|
+
* Auth resource class
|
|
2261
|
+
* Provides methods for SSE authentication
|
|
2262
|
+
*
|
|
2263
|
+
* @example
|
|
2264
|
+
* ```ts
|
|
2265
|
+
* // Generate SSE token for browser EventSource
|
|
2266
|
+
* const { token, expiresAt } = await client.auth.generateSSEToken();
|
|
2267
|
+
*
|
|
2268
|
+
* // Use in browser
|
|
2269
|
+
* const eventSource = new EventSource(`/v1/events?token=${token}`);
|
|
2270
|
+
*
|
|
2271
|
+
* // Token for specific job
|
|
2272
|
+
* const jobToken = await client.auth.generateSSEToken({ jobId: "job_123" });
|
|
2273
|
+
* const jobEvents = new EventSource(`/v1/events/job_123?token=${jobToken.token}`);
|
|
2274
|
+
* ```
|
|
2275
|
+
*/
|
|
2276
|
+
declare class AuthResource {
|
|
2277
|
+
private ctx;
|
|
2278
|
+
constructor(ctx: RequestContext);
|
|
2279
|
+
/**
|
|
2280
|
+
* Generate SSE authentication token
|
|
2281
|
+
*
|
|
2282
|
+
* Required for browser-based SSE connections because EventSource API
|
|
2283
|
+
* doesn't support custom headers. Token is short-lived (5 minutes).
|
|
2284
|
+
*
|
|
2285
|
+
* Security:
|
|
2286
|
+
* - Requires valid API key (Bearer token)
|
|
2287
|
+
* - Token expires in 5 minutes
|
|
2288
|
+
* - Token can be restricted to specific job
|
|
2289
|
+
* - Token stored in Redis (revocable)
|
|
2290
|
+
*
|
|
2291
|
+
* @example
|
|
2292
|
+
* ```ts
|
|
2293
|
+
* // 1. Generate token
|
|
2294
|
+
* const { token, expiresAt } = await client.auth.generateSSEToken();
|
|
2295
|
+
* console.log(`Token expires at: ${expiresAt}`);
|
|
2296
|
+
*
|
|
2297
|
+
* // 2. Use in browser EventSource
|
|
2298
|
+
* const eventSource = new EventSource(`/v1/events?token=${token}`);
|
|
2299
|
+
*
|
|
2300
|
+
* eventSource.addEventListener('job.completed', (event) => {
|
|
2301
|
+
* const data = JSON.parse(event.data);
|
|
2302
|
+
* console.log('Job completed:', data);
|
|
2303
|
+
* });
|
|
2304
|
+
*
|
|
2305
|
+
* // 3. For specific job only
|
|
2306
|
+
* const jobToken = await client.auth.generateSSEToken({ jobId: "job_abc123" });
|
|
2307
|
+
* const jobEvents = new EventSource(`/v1/events/job_abc123?token=${jobToken.token}`);
|
|
2308
|
+
* ```
|
|
2309
|
+
*/
|
|
2310
|
+
generateSSEToken(options?: GenerateSSETokenOptions): Promise<SSETokenResponse>;
|
|
2311
|
+
/**
|
|
2312
|
+
* Get SSE connection limits for current tier
|
|
2313
|
+
*
|
|
2314
|
+
* Shows how many concurrent SSE connections are allowed
|
|
2315
|
+
* and how many are currently active.
|
|
2316
|
+
*
|
|
2317
|
+
* Tier limits:
|
|
2318
|
+
* - Free: 2 concurrent connections
|
|
2319
|
+
* - Pro: 10 concurrent connections
|
|
2320
|
+
* - Enterprise: 50 concurrent connections
|
|
2321
|
+
*
|
|
2322
|
+
* @example
|
|
2323
|
+
* ```ts
|
|
2324
|
+
* const limits = await client.auth.getLimits();
|
|
2325
|
+
*
|
|
2326
|
+
* console.log(`Tier: ${limits.tier}`);
|
|
2327
|
+
* console.log(`Max connections: ${limits.sse.maxConnections}`);
|
|
2328
|
+
* console.log(`Current connections: ${limits.sse.currentConnections}`);
|
|
2329
|
+
* console.log(`Available: ${limits.sse.available}`);
|
|
2330
|
+
*
|
|
2331
|
+
* // Check before opening new connection
|
|
2332
|
+
* if (limits.sse.available > 0) {
|
|
2333
|
+
* const token = await client.auth.generateSSEToken();
|
|
2334
|
+
* const eventSource = new EventSource(`/v1/events?token=${token.token}`);
|
|
2335
|
+
* } else {
|
|
2336
|
+
* console.error('No available SSE connection slots');
|
|
2337
|
+
* }
|
|
2338
|
+
* ```
|
|
2339
|
+
*/
|
|
2340
|
+
getLimits(): Promise<SSEConnectionLimitsResponse>;
|
|
2341
|
+
}
|
|
2342
|
+
|
|
2343
|
+
/**
|
|
2344
|
+
* Convert Resource
|
|
2345
|
+
* Handles HTML to Markdown conversion
|
|
2346
|
+
*/
|
|
2347
|
+
|
|
2348
|
+
/**
|
|
2349
|
+
* Convert resource class
|
|
2350
|
+
* Provides methods for HTML to Markdown conversion
|
|
2351
|
+
*
|
|
2352
|
+
* @example
|
|
2353
|
+
* ```ts
|
|
2354
|
+
* // Convert HTML to markdown
|
|
2355
|
+
* const result = await client.convert.htmlToMarkdown({
|
|
2356
|
+
* html: "<h1>Title</h1><p>Content</p>",
|
|
2357
|
+
* baseUrl: "https://example.com",
|
|
2358
|
+
* options: {
|
|
2359
|
+
* gfmTables: true,
|
|
2360
|
+
* removeNoise: true
|
|
2361
|
+
* }
|
|
2362
|
+
* });
|
|
2363
|
+
*
|
|
2364
|
+
* console.log(result.data.markdown);
|
|
2365
|
+
* console.log(result.data.metadata.wordCount);
|
|
2366
|
+
* ```
|
|
2367
|
+
*/
|
|
2368
|
+
declare class ConvertResource {
|
|
2369
|
+
private ctx;
|
|
2370
|
+
constructor(ctx: RequestContext);
|
|
2371
|
+
/**
|
|
2372
|
+
* Convert HTML to Markdown
|
|
2373
|
+
*
|
|
2374
|
+
* Transforms raw HTML content into clean, readable Markdown using GitHub Flavored Markdown (GFM).
|
|
2375
|
+
* Useful for:
|
|
2376
|
+
* - Converting scraped HTML to markdown for LLM processing
|
|
2377
|
+
* - Cleaning up messy HTML from web pages
|
|
2378
|
+
* - Extracting main content while removing noise (ads, nav, footer)
|
|
2379
|
+
* - Creating documentation from HTML sources
|
|
2380
|
+
*
|
|
2381
|
+
* Features:
|
|
2382
|
+
* - GFM table, strikethrough, and task list support
|
|
2383
|
+
* - Automatic noise removal (scripts, ads, navigation)
|
|
2384
|
+
* - Relative URL resolution
|
|
2385
|
+
* - Custom element exclusion via CSS selectors
|
|
2386
|
+
* - Output length limiting
|
|
2387
|
+
*
|
|
2388
|
+
* @param options - Conversion options
|
|
2389
|
+
* @returns Conversion result with markdown, metadata, and warnings
|
|
2390
|
+
*
|
|
2391
|
+
* @example Basic usage
|
|
2392
|
+
* ```ts
|
|
2393
|
+
* const result = await client.convert.htmlToMarkdown({
|
|
2394
|
+
* html: "<h1>Product</h1><p>Price: $99</p>"
|
|
2395
|
+
* });
|
|
2396
|
+
* console.log(result.data.markdown);
|
|
2397
|
+
* ```
|
|
2398
|
+
*
|
|
2399
|
+
* @example With all options
|
|
2400
|
+
* ```ts
|
|
2401
|
+
* const result = await client.convert.htmlToMarkdown({
|
|
2402
|
+
* html: htmlContent,
|
|
2403
|
+
* baseUrl: "https://shop.example.com",
|
|
2404
|
+
* options: {
|
|
2405
|
+
* gfmTables: true,
|
|
2406
|
+
* removeNoise: true,
|
|
2407
|
+
* excludeSelectors: [".advertisement", "#sidebar"],
|
|
2408
|
+
* absoluteUrls: true,
|
|
2409
|
+
* maxLength: 100000,
|
|
2410
|
+
* includeImages: true,
|
|
2411
|
+
* includeLinks: true
|
|
2412
|
+
* }
|
|
2413
|
+
* });
|
|
2414
|
+
*
|
|
2415
|
+
* // Check metadata
|
|
2416
|
+
* console.log(`Words: ${result.data.metadata.wordCount}`);
|
|
2417
|
+
* console.log(`Links: ${result.data.metadata.linkCount}`);
|
|
2418
|
+
* console.log(`Images: ${result.data.metadata.imageCount}`);
|
|
2419
|
+
* console.log(`Conversion time: ${result.data.metadata.conversionTimeMs}ms`);
|
|
2420
|
+
*
|
|
2421
|
+
* // Check for warnings
|
|
2422
|
+
* if (result.data.warnings?.length) {
|
|
2423
|
+
* console.warn("Conversion warnings:", result.data.warnings);
|
|
2424
|
+
* }
|
|
2425
|
+
* ```
|
|
2426
|
+
*
|
|
2427
|
+
* @example Converting scraped HTML
|
|
2428
|
+
* ```ts
|
|
2429
|
+
* // First scrape a page
|
|
2430
|
+
* const scrapeJob = await client.scrape.create({
|
|
2431
|
+
* url: "https://example.com/article"
|
|
2432
|
+
* });
|
|
2433
|
+
* const scrapeResult = await client.waitForResult(scrapeJob.jobId);
|
|
2434
|
+
*
|
|
2435
|
+
* // Then convert HTML to markdown
|
|
2436
|
+
* const markdown = await client.convert.htmlToMarkdown({
|
|
2437
|
+
* html: scrapeResult.data.html,
|
|
2438
|
+
* baseUrl: scrapeResult.data.url,
|
|
2439
|
+
* options: {
|
|
2440
|
+
* removeNoise: true,
|
|
2441
|
+
* onlyMainContent: true
|
|
2442
|
+
* }
|
|
2443
|
+
* });
|
|
2444
|
+
* ```
|
|
2445
|
+
*/
|
|
2446
|
+
htmlToMarkdown(options: ConvertOptions): Promise<ConvertResponse>;
|
|
2447
|
+
/**
|
|
2448
|
+
* Alias for htmlToMarkdown() for convenience
|
|
2449
|
+
*
|
|
2450
|
+
* @example
|
|
2451
|
+
* ```ts
|
|
2452
|
+
* const result = await client.convert.toMarkdown({
|
|
2453
|
+
* html: "<h1>Hello</h1>"
|
|
2454
|
+
* });
|
|
2455
|
+
* ```
|
|
2456
|
+
*/
|
|
2457
|
+
toMarkdown(options: ConvertOptions): Promise<ConvertResponse>;
|
|
2458
|
+
/**
|
|
2459
|
+
* Convert HTML with minimal options (just the HTML content)
|
|
2460
|
+
* Uses all default settings
|
|
2461
|
+
*
|
|
2462
|
+
* @param html - HTML content to convert
|
|
2463
|
+
* @param baseUrl - Optional base URL for resolving relative links
|
|
2464
|
+
* @returns Conversion result
|
|
2465
|
+
*
|
|
2466
|
+
* @example
|
|
2467
|
+
* ```ts
|
|
2468
|
+
* const result = await client.convert.quick(
|
|
2469
|
+
* "<h1>Title</h1><p>Content</p>",
|
|
2470
|
+
* "https://example.com"
|
|
2471
|
+
* );
|
|
2472
|
+
* console.log(result.data.markdown);
|
|
2473
|
+
* ```
|
|
2474
|
+
*/
|
|
2475
|
+
quick(html: string, baseUrl?: string): Promise<ConvertResponse>;
|
|
2476
|
+
/**
|
|
2477
|
+
* Convert HTML with noise removal enabled
|
|
2478
|
+
* Removes navigation, footer, ads, scripts, and other clutter
|
|
2479
|
+
*
|
|
2480
|
+
* @param html - HTML content to convert
|
|
2481
|
+
* @param baseUrl - Optional base URL
|
|
2482
|
+
* @returns Conversion result with clean markdown
|
|
2483
|
+
*
|
|
2484
|
+
* @example
|
|
2485
|
+
* ```ts
|
|
2486
|
+
* // Extract just the main content from a messy page
|
|
2487
|
+
* const result = await client.convert.clean(messyHtml, "https://example.com");
|
|
2488
|
+
* console.log(result.data.markdown); // Clean, readable markdown
|
|
2489
|
+
* ```
|
|
2490
|
+
*/
|
|
2491
|
+
clean(html: string, baseUrl?: string): Promise<ConvertResponse>;
|
|
2492
|
+
}
|
|
2493
|
+
|
|
2038
2494
|
/**
|
|
2039
2495
|
* Crawl Resource
|
|
2040
2496
|
* Handles website crawling operations
|
|
@@ -2385,6 +2841,177 @@ declare class DorkResource {
|
|
|
2385
2841
|
buildQuery(options: DorkOptions): string;
|
|
2386
2842
|
}
|
|
2387
2843
|
|
|
2844
|
+
/**
|
|
2845
|
+
* Events Resource
|
|
2846
|
+
* Handles real-time SSE (Server-Sent Events) streaming
|
|
2847
|
+
*/
|
|
2848
|
+
|
|
2849
|
+
/**
|
|
2850
|
+
* Events resource class
|
|
2851
|
+
* Provides methods for subscribing to real-time job events via SSE
|
|
2852
|
+
*
|
|
2853
|
+
* IMPORTANT: Browser Support Only
|
|
2854
|
+
* This resource uses the EventSource API which is only available in browsers.
|
|
2855
|
+
* For Node.js, use polling via client.status.get() instead.
|
|
2856
|
+
*
|
|
2857
|
+
* @example Browser Usage
|
|
2858
|
+
* ```ts
|
|
2859
|
+
* // 1. Generate SSE token (required for EventSource)
|
|
2860
|
+
* const { token } = await client.auth.generateSSEToken();
|
|
2861
|
+
*
|
|
2862
|
+
* // 2. Subscribe to all events
|
|
2863
|
+
* const eventSource = client.events.subscribe({
|
|
2864
|
+
* token,
|
|
2865
|
+
* onEvent: (event) => {
|
|
2866
|
+
* console.log('Event:', event.type, event.data);
|
|
2867
|
+
* }
|
|
2868
|
+
* });
|
|
2869
|
+
*
|
|
2870
|
+
* // 3. Or subscribe to specific job
|
|
2871
|
+
* const jobEvents = client.events.subscribeToJob('job_123', {
|
|
2872
|
+
* token,
|
|
2873
|
+
* onEvent: (event) => {
|
|
2874
|
+
* const data = JSON.parse(event.data);
|
|
2875
|
+
* console.log('Job event:', data);
|
|
2876
|
+
* }
|
|
2877
|
+
* });
|
|
2878
|
+
*
|
|
2879
|
+
* // 4. Clean up when done
|
|
2880
|
+
* eventSource.close();
|
|
2881
|
+
* ```
|
|
2882
|
+
*/
|
|
2883
|
+
declare class EventsResource {
|
|
2884
|
+
private ctx;
|
|
2885
|
+
constructor(ctx: RequestContext);
|
|
2886
|
+
/**
|
|
2887
|
+
* Subscribe to all events for authenticated client
|
|
2888
|
+
*
|
|
2889
|
+
* Opens an SSE connection to receive real-time events for all jobs.
|
|
2890
|
+
* Requires an SSE token obtained via client.auth.generateSSEToken().
|
|
2891
|
+
*
|
|
2892
|
+
* Event Types:
|
|
2893
|
+
* - Job lifecycle: job.created, job.queued, job.started, job.progress,
|
|
2894
|
+
* job.completed, job.failed, job.cancelled
|
|
2895
|
+
* - Job details: job.log, job.metric, job.alert, job.checkpoint
|
|
2896
|
+
* - Deals: deal.found, deal.validated
|
|
2897
|
+
* - System: ping, connection.open, connection.close, error
|
|
2898
|
+
*
|
|
2899
|
+
* Features:
|
|
2900
|
+
* - Automatic reconnection on disconnect
|
|
2901
|
+
* - Event replay via Last-Event-ID
|
|
2902
|
+
* - Keepalive pings every 15 seconds
|
|
2903
|
+
* - Max connection time: 1 hour
|
|
2904
|
+
*
|
|
2905
|
+
* @param token - SSE authentication token from client.auth.generateSSEToken()
|
|
2906
|
+
* @param options - Subscription options (callbacks, reconnection settings)
|
|
2907
|
+
*
|
|
2908
|
+
* @example
|
|
2909
|
+
* ```ts
|
|
2910
|
+
* // Generate token
|
|
2911
|
+
* const { token } = await client.auth.generateSSEToken();
|
|
2912
|
+
*
|
|
2913
|
+
* // Subscribe with event handlers
|
|
2914
|
+
* const eventSource = client.events.subscribe(token, {
|
|
2915
|
+
* onEvent: (event) => {
|
|
2916
|
+
* // Handle all events
|
|
2917
|
+
* console.log('Event:', event.type);
|
|
2918
|
+
* const data = JSON.parse(event.data);
|
|
2919
|
+
*
|
|
2920
|
+
* if (data.jobId) {
|
|
2921
|
+
* console.log(`Job ${data.jobId}:`, data);
|
|
2922
|
+
* }
|
|
2923
|
+
* },
|
|
2924
|
+
* onError: (error) => {
|
|
2925
|
+
* console.error('SSE error:', error);
|
|
2926
|
+
* },
|
|
2927
|
+
* onOpen: () => {
|
|
2928
|
+
* console.log('SSE connection opened');
|
|
2929
|
+
* }
|
|
2930
|
+
* });
|
|
2931
|
+
*
|
|
2932
|
+
* // Listen for specific event types
|
|
2933
|
+
* eventSource.addEventListener('job.completed', (event) => {
|
|
2934
|
+
* const data = JSON.parse(event.data);
|
|
2935
|
+
* console.log('Job completed:', data);
|
|
2936
|
+
* });
|
|
2937
|
+
*
|
|
2938
|
+
* // Clean up
|
|
2939
|
+
* eventSource.close();
|
|
2940
|
+
* ```
|
|
2941
|
+
*/
|
|
2942
|
+
subscribe(token: string, options?: Omit<SSESubscribeOptions, "lastEventId">): EventSource;
|
|
2943
|
+
/**
|
|
2944
|
+
* Subscribe to events for a specific job
|
|
2945
|
+
*
|
|
2946
|
+
* Opens an SSE connection filtered to a single job.
|
|
2947
|
+
* More efficient than global subscription when tracking one job.
|
|
2948
|
+
*
|
|
2949
|
+
* @param jobId - Job ID to subscribe to
|
|
2950
|
+
* @param token - SSE authentication token
|
|
2951
|
+
* @param options - Subscription options
|
|
2952
|
+
*
|
|
2953
|
+
* @example
|
|
2954
|
+
* ```ts
|
|
2955
|
+
* // Start a scrape job
|
|
2956
|
+
* const job = await client.scrape.create({ url: "https://example.com" });
|
|
2957
|
+
*
|
|
2958
|
+
* // Generate SSE token for this job
|
|
2959
|
+
* const { token } = await client.auth.generateSSEToken({ jobId: job.jobId });
|
|
2960
|
+
*
|
|
2961
|
+
* // Subscribe to job events
|
|
2962
|
+
* const eventSource = client.events.subscribeToJob(job.jobId, token, {
|
|
2963
|
+
* onEvent: (event) => {
|
|
2964
|
+
* const data = JSON.parse(event.data);
|
|
2965
|
+
* console.log(`[${event.type}]`, data);
|
|
2966
|
+
* }
|
|
2967
|
+
* });
|
|
2968
|
+
*
|
|
2969
|
+
* // Listen for completion
|
|
2970
|
+
* eventSource.addEventListener('job.completed', (event) => {
|
|
2971
|
+
* const data = JSON.parse(event.data);
|
|
2972
|
+
* console.log('Scrape completed!', data.summary);
|
|
2973
|
+
* eventSource.close();
|
|
2974
|
+
* });
|
|
2975
|
+
*
|
|
2976
|
+
* // Listen for progress
|
|
2977
|
+
* eventSource.addEventListener('job.progress', (event) => {
|
|
2978
|
+
* const data = JSON.parse(event.data);
|
|
2979
|
+
* console.log(`Progress: ${data.progress}%`);
|
|
2980
|
+
* });
|
|
2981
|
+
*
|
|
2982
|
+
* // Listen for errors
|
|
2983
|
+
* eventSource.addEventListener('job.failed', (event) => {
|
|
2984
|
+
* const data = JSON.parse(event.data);
|
|
2985
|
+
* console.error('Job failed:', data.error);
|
|
2986
|
+
* eventSource.close();
|
|
2987
|
+
* });
|
|
2988
|
+
* ```
|
|
2989
|
+
*/
|
|
2990
|
+
subscribeToJob(jobId: string, token: string, options?: Omit<SSESubscribeOptions, "lastEventId">): EventSource;
|
|
2991
|
+
/**
|
|
2992
|
+
* Helper: Wait for job completion via SSE
|
|
2993
|
+
*
|
|
2994
|
+
* Convenience method that subscribes to a job and resolves when complete.
|
|
2995
|
+
* Automatically handles token generation and cleanup.
|
|
2996
|
+
*
|
|
2997
|
+
* @param jobId - Job ID to wait for
|
|
2998
|
+
* @param onProgress - Optional progress callback
|
|
2999
|
+
*
|
|
3000
|
+
* @example
|
|
3001
|
+
* ```ts
|
|
3002
|
+
* const job = await client.scrape.create({ url: "https://example.com" });
|
|
3003
|
+
*
|
|
3004
|
+
* // Wait for completion with progress updates
|
|
3005
|
+
* const result = await client.events.waitForCompletion(job.jobId, (progress) => {
|
|
3006
|
+
* console.log(`Progress: ${progress}%`);
|
|
3007
|
+
* });
|
|
3008
|
+
*
|
|
3009
|
+
* console.log('Job completed:', result);
|
|
3010
|
+
* ```
|
|
3011
|
+
*/
|
|
3012
|
+
waitForCompletion<T = unknown>(jobId: string, onProgress?: (progress: number) => void): Promise<T>;
|
|
3013
|
+
}
|
|
3014
|
+
|
|
2388
3015
|
/**
|
|
2389
3016
|
* Extract Resource
|
|
2390
3017
|
* Handles LLM-based structured data extraction
|
|
@@ -2689,7 +3316,8 @@ declare class ScrapeResource {
|
|
|
2689
3316
|
* { url: "https://shop1.com/product1" },
|
|
2690
3317
|
* { url: "https://shop2.com/deal", extractDeal: true }
|
|
2691
3318
|
* ],
|
|
2692
|
-
* defaults: { detectSignals: true }
|
|
3319
|
+
* defaults: { detectSignals: true },
|
|
3320
|
+
* ignoreInvalidURLs: true
|
|
2693
3321
|
* });
|
|
2694
3322
|
* console.log(batch.batchId, batch.results);
|
|
2695
3323
|
* ```
|
|
@@ -2720,6 +3348,58 @@ declare class ScrapeResource {
|
|
|
2720
3348
|
batchForDeals(urls: string[], options?: Omit<BatchScrapeOptions, "urls">): Promise<BatchScrapeResponse>;
|
|
2721
3349
|
}
|
|
2722
3350
|
|
|
3351
|
+
/**
|
|
3352
|
+
* Screenshots Resource
|
|
3353
|
+
* Handles screenshot signed URL refresh and TTL limits
|
|
3354
|
+
*/
|
|
3355
|
+
|
|
3356
|
+
/**
|
|
3357
|
+
* Options for refreshing a screenshot signed URL
|
|
3358
|
+
*/
|
|
3359
|
+
interface RefreshScreenshotOptions {
|
|
3360
|
+
/** File path in storage (e.g., "job_abc123/timestamp_id_url.png") */
|
|
3361
|
+
path: string;
|
|
3362
|
+
/** Optional: New TTL in seconds (must be within tier limits) */
|
|
3363
|
+
ttl?: number;
|
|
3364
|
+
/** Optional: Bucket name (defaults to 'screenshots-private') */
|
|
3365
|
+
bucket?: string;
|
|
3366
|
+
}
|
|
3367
|
+
/**
|
|
3368
|
+
* Screenshots resource class
|
|
3369
|
+
* Provides methods for managing screenshot signed URLs
|
|
3370
|
+
*/
|
|
3371
|
+
declare class ScreenshotsResource {
|
|
3372
|
+
private ctx;
|
|
3373
|
+
constructor(ctx: RequestContext);
|
|
3374
|
+
/**
|
|
3375
|
+
* Refresh a signed URL before expiration
|
|
3376
|
+
*
|
|
3377
|
+
* @example
|
|
3378
|
+
* ```ts
|
|
3379
|
+
* const refreshed = await client.screenshots.refresh({
|
|
3380
|
+
* path: "job_abc123/1234567890_nanoid_example.png",
|
|
3381
|
+
* ttl: 604800 // 7 days
|
|
3382
|
+
* });
|
|
3383
|
+
* console.log(refreshed.url); // New signed URL
|
|
3384
|
+
* console.log(refreshed.expiresAt); // "2026-01-25T12:00:00Z"
|
|
3385
|
+
* console.log(refreshed.tierLimits); // { min: 3600, max: 604800, default: 604800 }
|
|
3386
|
+
* ```
|
|
3387
|
+
*/
|
|
3388
|
+
refresh(options: RefreshScreenshotOptions): Promise<ScreenshotRefreshResponse>;
|
|
3389
|
+
/**
|
|
3390
|
+
* Get TTL limits for the current tier
|
|
3391
|
+
*
|
|
3392
|
+
* @example
|
|
3393
|
+
* ```ts
|
|
3394
|
+
* const limits = await client.screenshots.getLimits();
|
|
3395
|
+
* console.log(limits.tier); // "pro"
|
|
3396
|
+
* console.log(limits.limits.max); // 604800 (7 days in seconds)
|
|
3397
|
+
* console.log(limits.formattedLimits.max); // "7 days"
|
|
3398
|
+
* ```
|
|
3399
|
+
*/
|
|
3400
|
+
getLimits(): Promise<ScreenshotLimitsResponse>;
|
|
3401
|
+
}
|
|
3402
|
+
|
|
2723
3403
|
/**
|
|
2724
3404
|
* Search Resource
|
|
2725
3405
|
* Handles web search operations with AI optimization
|
|
@@ -2739,7 +3419,7 @@ declare class SearchResource {
|
|
|
2739
3419
|
* ```ts
|
|
2740
3420
|
* const result = await client.search.create({
|
|
2741
3421
|
* query: "laptop deals black friday",
|
|
2742
|
-
*
|
|
3422
|
+
* limit: 20,
|
|
2743
3423
|
* useDealScoring: true
|
|
2744
3424
|
* });
|
|
2745
3425
|
* ```
|
|
@@ -2775,12 +3455,12 @@ declare class SearchResource {
|
|
|
2775
3455
|
* @example
|
|
2776
3456
|
* ```ts
|
|
2777
3457
|
* const result = await client.search.andScrape("promo codes", {
|
|
2778
|
-
*
|
|
3458
|
+
* maxScrapeResults: 5
|
|
2779
3459
|
* });
|
|
2780
3460
|
* console.log(result.data.scrapedJobIds);
|
|
2781
3461
|
* ```
|
|
2782
3462
|
*/
|
|
2783
|
-
andScrape(query: string, options?: Omit<SearchOptions, "query" | "
|
|
3463
|
+
andScrape(query: string, options?: Omit<SearchOptions, "query" | "scrapeResults">): Promise<SearchJobResponse>;
|
|
2784
3464
|
/**
|
|
2785
3465
|
* Check search API status
|
|
2786
3466
|
* Returns availability and features info
|
|
@@ -3056,6 +3736,9 @@ declare class WebhooksResource {
|
|
|
3056
3736
|
* await client.extract.create({ url: "...", schema: {...} });
|
|
3057
3737
|
* await client.dork.create({ query: "..." });
|
|
3058
3738
|
*
|
|
3739
|
+
* // Utilities
|
|
3740
|
+
* await client.convert.htmlToMarkdown({ html: "..." });
|
|
3741
|
+
*
|
|
3059
3742
|
* // Status & polling
|
|
3060
3743
|
* await client.status.get(jobId);
|
|
3061
3744
|
* await client.waitForResult(jobId);
|
|
@@ -3147,6 +3830,20 @@ declare class DealCrawl {
|
|
|
3147
3830
|
* ```
|
|
3148
3831
|
*/
|
|
3149
3832
|
readonly dork: DorkResource;
|
|
3833
|
+
/**
|
|
3834
|
+
* Convert resource - HTML to Markdown conversion
|
|
3835
|
+
*
|
|
3836
|
+
* @example
|
|
3837
|
+
* ```ts
|
|
3838
|
+
* const result = await client.convert.htmlToMarkdown({
|
|
3839
|
+
* html: "<h1>Title</h1><p>Content</p>",
|
|
3840
|
+
* baseUrl: "https://example.com",
|
|
3841
|
+
* options: { removeNoise: true }
|
|
3842
|
+
* });
|
|
3843
|
+
* console.log(result.data.markdown);
|
|
3844
|
+
* ```
|
|
3845
|
+
*/
|
|
3846
|
+
readonly convert: ConvertResource;
|
|
3150
3847
|
/**
|
|
3151
3848
|
* Agent resource - AI-powered autonomous web navigation
|
|
3152
3849
|
*
|
|
@@ -3228,6 +3925,80 @@ declare class DealCrawl {
|
|
|
3228
3925
|
* ```
|
|
3229
3926
|
*/
|
|
3230
3927
|
readonly account: AccountResource;
|
|
3928
|
+
/**
|
|
3929
|
+
* Screenshots resource - Screenshot signed URL management
|
|
3930
|
+
*
|
|
3931
|
+
* @example
|
|
3932
|
+
* ```ts
|
|
3933
|
+
* // Refresh a signed URL before expiration
|
|
3934
|
+
* const refreshed = await client.screenshots.refresh({
|
|
3935
|
+
* path: "job_abc123/1234567890_nanoid_example.png",
|
|
3936
|
+
* ttl: 604800 // 7 days
|
|
3937
|
+
* });
|
|
3938
|
+
*
|
|
3939
|
+
* // Get tier-specific TTL limits
|
|
3940
|
+
* const limits = await client.screenshots.getLimits();
|
|
3941
|
+
* console.log(limits.formattedLimits.max); // "7 days"
|
|
3942
|
+
* ```
|
|
3943
|
+
*/
|
|
3944
|
+
readonly screenshots: ScreenshotsResource;
|
|
3945
|
+
/**
|
|
3946
|
+
* Auth resource - SSE (Server-Sent Events) authentication
|
|
3947
|
+
*
|
|
3948
|
+
* @example
|
|
3949
|
+
* ```ts
|
|
3950
|
+
* // Generate SSE token for browser EventSource
|
|
3951
|
+
* const { token, expiresAt } = await client.auth.generateSSEToken();
|
|
3952
|
+
*
|
|
3953
|
+
* // Use in browser
|
|
3954
|
+
* const eventSource = new EventSource(`/v1/events?token=${token}`);
|
|
3955
|
+
*
|
|
3956
|
+
* // Generate token for specific job
|
|
3957
|
+
* const jobToken = await client.auth.generateSSEToken({ jobId: "job_123" });
|
|
3958
|
+
*
|
|
3959
|
+
* // Check connection limits
|
|
3960
|
+
* const limits = await client.auth.getLimits();
|
|
3961
|
+
* console.log(`Available connections: ${limits.sse.available}`);
|
|
3962
|
+
* ```
|
|
3963
|
+
*/
|
|
3964
|
+
readonly auth: AuthResource;
|
|
3965
|
+
/**
|
|
3966
|
+
* Events resource - Real-time SSE event streaming (Browser only)
|
|
3967
|
+
*
|
|
3968
|
+
* IMPORTANT: This resource only works in browsers. For Node.js, use polling via client.status.get()
|
|
3969
|
+
*
|
|
3970
|
+
* @example Browser Usage
|
|
3971
|
+
* ```ts
|
|
3972
|
+
* // 1. Generate SSE token
|
|
3973
|
+
* const { token } = await client.auth.generateSSEToken();
|
|
3974
|
+
*
|
|
3975
|
+
* // 2. Subscribe to all events
|
|
3976
|
+
* const eventSource = client.events.subscribe(token, {
|
|
3977
|
+
* onEvent: (event) => {
|
|
3978
|
+
* console.log('Event:', event.type, JSON.parse(event.data));
|
|
3979
|
+
* }
|
|
3980
|
+
* });
|
|
3981
|
+
*
|
|
3982
|
+
* // 3. Or subscribe to specific job
|
|
3983
|
+
* const jobEvents = client.events.subscribeToJob('job_123', token, {
|
|
3984
|
+
* onEvent: (event) => {
|
|
3985
|
+
* const data = JSON.parse(event.data);
|
|
3986
|
+
* console.log(`Progress: ${data.progress}%`);
|
|
3987
|
+
* }
|
|
3988
|
+
* });
|
|
3989
|
+
*
|
|
3990
|
+
* // 4. Listen for specific events
|
|
3991
|
+
* eventSource.addEventListener('job.completed', (event) => {
|
|
3992
|
+
* const data = JSON.parse(event.data);
|
|
3993
|
+
* console.log('Job completed!', data.summary);
|
|
3994
|
+
* eventSource.close();
|
|
3995
|
+
* });
|
|
3996
|
+
*
|
|
3997
|
+
* // 5. Clean up
|
|
3998
|
+
* eventSource.close();
|
|
3999
|
+
* ```
|
|
4000
|
+
*/
|
|
4001
|
+
readonly events: EventsResource;
|
|
3231
4002
|
/**
|
|
3232
4003
|
* Create a new DealCrawl client
|
|
3233
4004
|
*
|
|
@@ -3635,4 +4406,352 @@ declare const ERROR_MESSAGES: Record<ErrorCode, string>;
|
|
|
3635
4406
|
*/
|
|
3636
4407
|
declare function getErrorMessage(code: ErrorCode): string;
|
|
3637
4408
|
|
|
3638
|
-
|
|
4409
|
+
/**
|
|
4410
|
+
* Advanced Job Status System
|
|
4411
|
+
*
|
|
4412
|
+
* Production-grade status model with:
|
|
4413
|
+
* - Primary status (lifecycle stage)
|
|
4414
|
+
* - Sub-status (detailed state within stage)
|
|
4415
|
+
* - Reason codes (why this state was reached)
|
|
4416
|
+
* - State machine transitions (valid state changes)
|
|
4417
|
+
*
|
|
4418
|
+
* @module job-status
|
|
4419
|
+
*/
|
|
4420
|
+
/**
|
|
4421
|
+
* Primary job status - represents the main lifecycle stage
|
|
4422
|
+
*/
|
|
4423
|
+
type JobStatus = "queued" | "scheduled" | "priority_queued" | "initializing" | "active" | "processing" | "extracting" | "uploading" | "finalizing" | "retrying" | "backing_off" | "paused" | "pausing" | "resuming" | "completed" | "partial" | "failed" | "timeout" | "cancelled" | "rejected" | "stale" | "zombie" | "stuck";
|
|
4424
|
+
/**
|
|
4425
|
+
* Sub-status provides granular detail within a primary status
|
|
4426
|
+
*/
|
|
4427
|
+
type JobSubStatus = "waiting_for_worker" | "waiting_for_rate_limit" | "waiting_for_quota" | "waiting_for_retry" | "waiting_for_dependency" | "launching_browser" | "establishing_connection" | "loading_config" | "validating_input" | "fetching_page" | "executing_javascript" | "waiting_for_render" | "detecting_signals" | "crawling_links" | "capturing_screenshot" | "calling_llm" | "parsing_response" | "validating_extraction" | "uploading_screenshot" | "uploading_artifacts" | "generating_signed_url" | "saving_to_database" | "cleaning_resources" | "sending_webhook" | "syncing_to_dealup" | "exponential_backoff" | "rate_limit_backoff" | "circuit_breaker_open" | "network_error" | "timeout_error" | "bot_detected" | "captcha_detected" | "auth_required" | "page_not_found" | "server_error" | "parse_error" | "extraction_failed" | "quota_exceeded" | "invalid_input" | "resource_unavailable" | "no_heartbeat" | "progress_stalled" | "worker_lost";
|
|
4428
|
+
/**
|
|
4429
|
+
* Reason codes - why a status/substatus was reached
|
|
4430
|
+
*/
|
|
4431
|
+
type StatusReason = "job_completed_successfully" | "partial_results_available" | "added_to_queue" | "priority_elevated" | "rate_limit_applied" | "quota_check_pending" | "transient_network_error" | "rate_limit_exceeded" | "timeout_exceeded" | "worker_crashed" | "max_retries_not_reached" | "max_retries_exceeded" | "permanent_error" | "blocked_by_site" | "captcha_required" | "invalid_url_format" | "ssrf_detected" | "quota_exhausted" | "tier_restriction" | "resource_not_available" | "cancelled_by_user" | "cancelled_by_system" | "duplicate_detected" | "policy_violation" | "heartbeat_timeout" | "progress_timeout" | "worker_unresponsive";
|
|
4432
|
+
|
|
4433
|
+
/**
|
|
4434
|
+
* Professional structured logger for DealCrawl
|
|
4435
|
+
* - Environment-aware (verbose in dev, minimal in prod)
|
|
4436
|
+
* - Structured JSON output in production
|
|
4437
|
+
* - Colored console output in development
|
|
4438
|
+
* - Correlation ID support for request tracing
|
|
4439
|
+
*/
|
|
4440
|
+
type LogLevel = "debug" | "info" | "warn" | "error";
|
|
4441
|
+
|
|
4442
|
+
/**
|
|
4443
|
+
* SSE (Server-Sent Events) Event Types
|
|
4444
|
+
*
|
|
4445
|
+
* Defines all events that can be pushed to clients via SSE.
|
|
4446
|
+
*
|
|
4447
|
+
* @module sse-events
|
|
4448
|
+
*/
|
|
4449
|
+
|
|
4450
|
+
/**
|
|
4451
|
+
* SSE event types - used as the "event:" field in SSE messages
|
|
4452
|
+
*/
|
|
4453
|
+
type SSEEventType = "job.created" | "job.queued" | "job.started" | "job.status" | "job.progress" | "job.completed" | "job.failed" | "job.cancelled" | "job.paused" | "job.resumed" | "job.log" | "job.metric" | "job.alert" | "job.checkpoint" | "deal.found" | "deal.validated" | "ping" | "connection.open" | "connection.close" | "error";
|
|
4454
|
+
/**
|
|
4455
|
+
* Base SSE event structure - all events extend this
|
|
4456
|
+
*/
|
|
4457
|
+
interface BaseSSEEvent {
|
|
4458
|
+
/** Event type (used in SSE "event:" field) */
|
|
4459
|
+
type: SSEEventType;
|
|
4460
|
+
/** Unique event ID (for Last-Event-ID replay) */
|
|
4461
|
+
id: string;
|
|
4462
|
+
/** When this event was generated */
|
|
4463
|
+
timestamp: string;
|
|
4464
|
+
/** Job ID this event relates to (null for system events) */
|
|
4465
|
+
jobId: string | null;
|
|
4466
|
+
/** Client ID (for server-side filtering) */
|
|
4467
|
+
clientId: string;
|
|
4468
|
+
}
|
|
4469
|
+
/**
|
|
4470
|
+
* Job created event - sent when job is first created
|
|
4471
|
+
*/
|
|
4472
|
+
interface JobCreatedEvent extends BaseSSEEvent {
|
|
4473
|
+
type: "job.created";
|
|
4474
|
+
jobId: string;
|
|
4475
|
+
data: {
|
|
4476
|
+
jobType: "scrape" | "crawl" | "dork" | "extract" | "agent";
|
|
4477
|
+
priority?: "high" | "medium" | "low";
|
|
4478
|
+
template?: string;
|
|
4479
|
+
};
|
|
4480
|
+
}
|
|
4481
|
+
/**
|
|
4482
|
+
* Job queued event - sent when job enters queue
|
|
4483
|
+
*/
|
|
4484
|
+
interface JobQueuedEvent extends BaseSSEEvent {
|
|
4485
|
+
type: "job.queued";
|
|
4486
|
+
jobId: string;
|
|
4487
|
+
data: {
|
|
4488
|
+
queue: string;
|
|
4489
|
+
position?: number;
|
|
4490
|
+
estimatedWaitMs?: number;
|
|
4491
|
+
};
|
|
4492
|
+
}
|
|
4493
|
+
/**
|
|
4494
|
+
* Job started event - sent when worker picks up job
|
|
4495
|
+
*/
|
|
4496
|
+
interface JobStartedEvent extends BaseSSEEvent {
|
|
4497
|
+
type: "job.started";
|
|
4498
|
+
jobId: string;
|
|
4499
|
+
data: {
|
|
4500
|
+
workerId?: string;
|
|
4501
|
+
startedAt: string;
|
|
4502
|
+
};
|
|
4503
|
+
}
|
|
4504
|
+
/**
|
|
4505
|
+
* Job status change event - sent on every status transition
|
|
4506
|
+
*/
|
|
4507
|
+
interface JobStatusEvent extends BaseSSEEvent {
|
|
4508
|
+
type: "job.status";
|
|
4509
|
+
jobId: string;
|
|
4510
|
+
data: {
|
|
4511
|
+
status: JobStatus;
|
|
4512
|
+
subStatus?: JobSubStatus;
|
|
4513
|
+
reason?: StatusReason;
|
|
4514
|
+
message?: string;
|
|
4515
|
+
previousStatus?: JobStatus;
|
|
4516
|
+
progress: number;
|
|
4517
|
+
};
|
|
4518
|
+
}
|
|
4519
|
+
/**
|
|
4520
|
+
* Job progress event - sent periodically during processing
|
|
4521
|
+
*/
|
|
4522
|
+
interface JobProgressEvent extends BaseSSEEvent {
|
|
4523
|
+
type: "job.progress";
|
|
4524
|
+
jobId: string;
|
|
4525
|
+
data: {
|
|
4526
|
+
progress: number;
|
|
4527
|
+
status: JobStatus;
|
|
4528
|
+
subStatus?: JobSubStatus;
|
|
4529
|
+
message?: string;
|
|
4530
|
+
eta?: {
|
|
4531
|
+
remainingMs: number;
|
|
4532
|
+
remainingFormatted: string;
|
|
4533
|
+
};
|
|
4534
|
+
stats?: {
|
|
4535
|
+
pagesProcessed?: number;
|
|
4536
|
+
dealsFound?: number;
|
|
4537
|
+
signalsDetected?: number;
|
|
4538
|
+
};
|
|
4539
|
+
};
|
|
4540
|
+
}
|
|
4541
|
+
/**
|
|
4542
|
+
* Job completed event - sent when job finishes successfully
|
|
4543
|
+
*/
|
|
4544
|
+
interface JobCompletedEvent extends BaseSSEEvent {
|
|
4545
|
+
type: "job.completed";
|
|
4546
|
+
jobId: string;
|
|
4547
|
+
data: {
|
|
4548
|
+
statusUrl: string;
|
|
4549
|
+
completedAt: string;
|
|
4550
|
+
durationMs: number;
|
|
4551
|
+
summary: {
|
|
4552
|
+
success: boolean;
|
|
4553
|
+
pagesProcessed?: number;
|
|
4554
|
+
dealsFound?: number;
|
|
4555
|
+
signalsDetected?: number;
|
|
4556
|
+
hasScreenshot?: boolean;
|
|
4557
|
+
};
|
|
4558
|
+
result?: {
|
|
4559
|
+
dealScore?: number;
|
|
4560
|
+
topSignals?: string[];
|
|
4561
|
+
dealTitle?: string;
|
|
4562
|
+
};
|
|
4563
|
+
};
|
|
4564
|
+
}
|
|
4565
|
+
/**
|
|
4566
|
+
* Job failed event - sent when job fails
|
|
4567
|
+
*/
|
|
4568
|
+
interface JobFailedEvent extends BaseSSEEvent {
|
|
4569
|
+
type: "job.failed";
|
|
4570
|
+
jobId: string;
|
|
4571
|
+
data: {
|
|
4572
|
+
statusUrl: string;
|
|
4573
|
+
failedAt: string;
|
|
4574
|
+
durationMs: number;
|
|
4575
|
+
error: {
|
|
4576
|
+
code: string;
|
|
4577
|
+
message: string;
|
|
4578
|
+
recoverable: boolean;
|
|
4579
|
+
};
|
|
4580
|
+
retry?: {
|
|
4581
|
+
attempt: number;
|
|
4582
|
+
maxAttempts: number;
|
|
4583
|
+
willRetry: boolean;
|
|
4584
|
+
nextRetryAt?: string;
|
|
4585
|
+
};
|
|
4586
|
+
};
|
|
4587
|
+
}
|
|
4588
|
+
/**
|
|
4589
|
+
* Job cancelled event - sent when job is cancelled
|
|
4590
|
+
*/
|
|
4591
|
+
interface JobCancelledEvent extends BaseSSEEvent {
|
|
4592
|
+
type: "job.cancelled";
|
|
4593
|
+
jobId: string;
|
|
4594
|
+
data: {
|
|
4595
|
+
cancelledAt: string;
|
|
4596
|
+
cancelledBy: "user" | "system";
|
|
4597
|
+
reason?: string;
|
|
4598
|
+
};
|
|
4599
|
+
}
|
|
4600
|
+
/**
|
|
4601
|
+
* Job paused event
|
|
4602
|
+
*/
|
|
4603
|
+
interface JobPausedEvent extends BaseSSEEvent {
|
|
4604
|
+
type: "job.paused";
|
|
4605
|
+
jobId: string;
|
|
4606
|
+
data: {
|
|
4607
|
+
pausedAt: string;
|
|
4608
|
+
checkpoint?: {
|
|
4609
|
+
lastProcessedUrl: string;
|
|
4610
|
+
urlsProcessed: number;
|
|
4611
|
+
urlsRemaining: number;
|
|
4612
|
+
};
|
|
4613
|
+
};
|
|
4614
|
+
}
|
|
4615
|
+
/**
|
|
4616
|
+
* Job resumed event
|
|
4617
|
+
*/
|
|
4618
|
+
interface JobResumedEvent extends BaseSSEEvent {
|
|
4619
|
+
type: "job.resumed";
|
|
4620
|
+
jobId: string;
|
|
4621
|
+
data: {
|
|
4622
|
+
resumedAt: string;
|
|
4623
|
+
fromCheckpoint: boolean;
|
|
4624
|
+
};
|
|
4625
|
+
}
|
|
4626
|
+
/**
|
|
4627
|
+
* Job log event - sent for important log messages
|
|
4628
|
+
*/
|
|
4629
|
+
interface JobLogEvent extends BaseSSEEvent {
|
|
4630
|
+
type: "job.log";
|
|
4631
|
+
jobId: string;
|
|
4632
|
+
data: {
|
|
4633
|
+
level: LogLevel;
|
|
4634
|
+
message: string;
|
|
4635
|
+
metadata?: Record<string, unknown>;
|
|
4636
|
+
};
|
|
4637
|
+
}
|
|
4638
|
+
/**
|
|
4639
|
+
* Job metric event - sent for performance/business metrics
|
|
4640
|
+
*/
|
|
4641
|
+
interface JobMetricEvent extends BaseSSEEvent {
|
|
4642
|
+
type: "job.metric";
|
|
4643
|
+
jobId: string;
|
|
4644
|
+
data: {
|
|
4645
|
+
metric: string;
|
|
4646
|
+
value: number;
|
|
4647
|
+
unit?: string;
|
|
4648
|
+
};
|
|
4649
|
+
}
|
|
4650
|
+
/**
|
|
4651
|
+
* Job alert event - sent for important alerts (quota warnings, etc.)
|
|
4652
|
+
*/
|
|
4653
|
+
interface JobAlertEvent extends BaseSSEEvent {
|
|
4654
|
+
type: "job.alert";
|
|
4655
|
+
jobId: string;
|
|
4656
|
+
data: {
|
|
4657
|
+
severity: "info" | "warning" | "error" | "critical";
|
|
4658
|
+
title: string;
|
|
4659
|
+
message: string;
|
|
4660
|
+
actionRequired?: boolean;
|
|
4661
|
+
};
|
|
4662
|
+
}
|
|
4663
|
+
/**
|
|
4664
|
+
* Job checkpoint event - sent when checkpoint is saved
|
|
4665
|
+
*/
|
|
4666
|
+
interface JobCheckpointEvent extends BaseSSEEvent {
|
|
4667
|
+
type: "job.checkpoint";
|
|
4668
|
+
jobId: string;
|
|
4669
|
+
data: {
|
|
4670
|
+
savedAt: string;
|
|
4671
|
+
lastProcessedUrl: string;
|
|
4672
|
+
urlsProcessed: number;
|
|
4673
|
+
urlsRemaining: number;
|
|
4674
|
+
canResume: boolean;
|
|
4675
|
+
};
|
|
4676
|
+
}
|
|
4677
|
+
/**
|
|
4678
|
+
* Deal found event - sent when a deal is detected
|
|
4679
|
+
*/
|
|
4680
|
+
interface DealFoundEvent extends BaseSSEEvent {
|
|
4681
|
+
type: "deal.found";
|
|
4682
|
+
jobId: string;
|
|
4683
|
+
data: {
|
|
4684
|
+
dealId?: string;
|
|
4685
|
+
url: string;
|
|
4686
|
+
title: string;
|
|
4687
|
+
score: number;
|
|
4688
|
+
price?: number;
|
|
4689
|
+
discount?: number;
|
|
4690
|
+
category?: string;
|
|
4691
|
+
signals: string[];
|
|
4692
|
+
};
|
|
4693
|
+
}
|
|
4694
|
+
/**
|
|
4695
|
+
* Deal validated event - sent when deal is validated/scored
|
|
4696
|
+
*/
|
|
4697
|
+
interface DealValidatedEvent extends BaseSSEEvent {
|
|
4698
|
+
type: "deal.validated";
|
|
4699
|
+
jobId: string;
|
|
4700
|
+
data: {
|
|
4701
|
+
dealId: string;
|
|
4702
|
+
score: number;
|
|
4703
|
+
confidence: number;
|
|
4704
|
+
accepted: boolean;
|
|
4705
|
+
rejectionReason?: string;
|
|
4706
|
+
};
|
|
4707
|
+
}
|
|
4708
|
+
/**
|
|
4709
|
+
* Ping event - keepalive to prevent connection timeout
|
|
4710
|
+
*/
|
|
4711
|
+
interface PingEvent extends BaseSSEEvent {
|
|
4712
|
+
type: "ping";
|
|
4713
|
+
jobId: null;
|
|
4714
|
+
data: {
|
|
4715
|
+
serverTime: string;
|
|
4716
|
+
};
|
|
4717
|
+
}
|
|
4718
|
+
/**
|
|
4719
|
+
* Connection open event - sent when SSE connection established
|
|
4720
|
+
*/
|
|
4721
|
+
interface ConnectionOpenEvent extends BaseSSEEvent {
|
|
4722
|
+
type: "connection.open";
|
|
4723
|
+
jobId: null;
|
|
4724
|
+
data: {
|
|
4725
|
+
clientId: string;
|
|
4726
|
+
reconnected: boolean;
|
|
4727
|
+
lastEventId?: string;
|
|
4728
|
+
};
|
|
4729
|
+
}
|
|
4730
|
+
/**
|
|
4731
|
+
* Connection close event - sent before connection closes
|
|
4732
|
+
*/
|
|
4733
|
+
interface ConnectionCloseEvent extends BaseSSEEvent {
|
|
4734
|
+
type: "connection.close";
|
|
4735
|
+
jobId: null;
|
|
4736
|
+
data: {
|
|
4737
|
+
reason: string;
|
|
4738
|
+
};
|
|
4739
|
+
}
|
|
4740
|
+
/**
|
|
4741
|
+
* Error event - sent when an error occurs
|
|
4742
|
+
*/
|
|
4743
|
+
interface ErrorEvent extends BaseSSEEvent {
|
|
4744
|
+
type: "error";
|
|
4745
|
+
jobId: string | null;
|
|
4746
|
+
data: {
|
|
4747
|
+
code: string;
|
|
4748
|
+
message: string;
|
|
4749
|
+
recoverable: boolean;
|
|
4750
|
+
};
|
|
4751
|
+
}
|
|
4752
|
+
/**
|
|
4753
|
+
* All possible SSE events (discriminated union)
|
|
4754
|
+
*/
|
|
4755
|
+
type SSEEvent = JobCreatedEvent | JobQueuedEvent | JobStartedEvent | JobStatusEvent | JobProgressEvent | JobCompletedEvent | JobFailedEvent | JobCancelledEvent | JobPausedEvent | JobResumedEvent | JobLogEvent | JobMetricEvent | JobAlertEvent | JobCheckpointEvent | DealFoundEvent | DealValidatedEvent | PingEvent | ConnectionOpenEvent | ConnectionCloseEvent | ErrorEvent;
|
|
4756
|
+
|
|
4757
|
+
export { ALL_API_KEY_SCOPES, type AccountInfoResponse, type AccountMetricsResponse, AccountResource, type ActionBaseOptions, type ActionInput, type AgentAction, type AgentActionType, type AgentCompletionReason, type AgentJobResponse, type AgentModel, type AgentOptions, AgentResource, type AgentResultResponse, type AgentStatusResponse, type AgentStepResponse, type ApiError, type ApiKeyInfo, type ApiKeyScope, type ApiResponse, AuthResource, type BaseSSEEvent, type BatchScrapeDefaults, type BatchScrapeItem, type BatchScrapeOptions, type BatchScrapeResponse, type BatchScrapeResultItem, type BatchStatusResponse, type CancelJobResponse, type CheckpointInfo, type ClickAction, type ClientPreferences, type ClientStatsResponse, type ConnectionCloseEvent, type ConnectionOpenEvent, type ConversionMetadata, type ConvertOptions, ConvertResource, type ConvertResponse, type CrawlAnalysisResponse, type CrawlError, type CrawlJobResponse, type CrawlMode, type CrawlOptions, type CrawlRecommendation, CrawlResource, type CrawlResult, type CrawlStats, type CrawlTemplate, type CrawlTemplateId, type CreateApiKeyOptions, type CreateKeyResponse, type CreateWebhookOptions, type CreateWebhookResponse, type CreatedApiKey, DEFAULT_API_KEY_SCOPES, DEFAULT_CONFIG, DataResource, DealCrawl, type DealCrawlConfig, DealCrawlError, type DealDetails, type DealFoundEvent, type DealItem, type DealMetrics, type DealScoreSummary, type DealSummary, type DealUpMetrics, type DealUpMetricsResponse, type DealValidatedEvent, type DeleteKeyResponse, type DeleteWebhookResponse, type DiscountSignal, type DorkJobResponse, type DorkOptions, DorkResource, type DorkResult, ERROR_CODES, ERROR_MESSAGES, type EngineSelection, type EngineType, type ErrorCode, type ErrorEvent, EventsResource, type ExportDealsOptions, type ExportFormat, type ExportJobsOptions, type ExtractJobResponse, type ExtractModel, type ExtractOptions, ExtractResource, type ExtractedDeal, type ExtractionErrorDetails, type FallbackConfig, type FallbackMetadata, type FallbackResult, type FallbackSource, type GenerateSSETokenOptions, type GetApiKeyStatsOptions, type GetDealsOptions, type HoverAction, type JobAlertEvent, type JobCancelledEvent, type JobCheckpointEvent, type JobCompletedEvent, type JobCreatedEvent, type JobDealsResponse, type JobFailedEvent, type JobLogEvent, type JobMetricEvent, type JobMetricsResponse, type JobPausedEvent, type JobProgressEvent, type JobQueuedEvent, type JobResponse, type JobResumedEvent, type JobStartedEvent, type JobStatus$1 as JobStatus, type JobStatusEvent, type JobStatusFilter, type JobStatusResponse, type JobSummary, type JobTypeFilter, type KeyStatsResponse, KeysResource, type ListApiKeysOptions, type ListDealsOptions, type ListDealsResponse, type ListJobsOptions, type ListJobsResponse, type ListKeysResponse, type ListWebhooksResponse, type PaginatedResponse, type PaginationInfo, type ParsedPage, type PingEvent, type PreferencesResponse, type PressAction, type PriceSignal, type PricingInfo, type ProductCategory, type ProductInfo, type RateLimitInfo, type RecommendationsResponse, type RefreshScreenshotOptions, type RequestContext, type ResumeJobResponse, type RevokeApiKeyOptions, type RotateApiKeyOptions, type RotateKeyResponse, type SSEConnectionLimitsResponse, type SSEEvent, type SSEEventType, type SSESubscribeOptions, type SSETokenResponse, type ScrapeJobResponse, type ScrapeOptions, ScrapeResource, type ScrapeResult, type ScreenshotAction, type ScreenshotAgentAction, type ScreenshotLimitsResponse, type ScreenshotOptions, type ScreenshotRefreshResponse, type ScreenshotResult, ScreenshotsResource, type ScrollAction, type SearchAiModel, type SearchAiProvider, type SearchData, type SearchDateRange, type SearchFilters, type SearchJobResponse, type SearchOptions, SearchResource, type SearchResultItem, type SearchStatusResponse, type SelectAction, type ClickAction$1 as SharedClickAction, type HoverAction$1 as SharedHoverAction, type PressAction$1 as SharedPressAction, type ScrollAction$1 as SharedScrollAction, type SelectAction$1 as SharedSelectAction, type WaitAction$1 as SharedWaitAction, type WriteAction$1 as SharedWriteAction, type Signal, type SortOrder, StatusResource, type TestWebhookResponse, type UpdatePreferencesOptions, type UpdatePreferencesResponse, type UpdateWebhookOptions, type UpdateWebhookResponse, type UrgencyLevel, type UsageStats, type ValidationError, type ValidationResult, type WaitAction, type WaitOptions, type WaitResult, type WebhookEvent, type WebhookItem, WebhooksResource, type WriteAction, DealCrawl as default, getErrorMessage, pollUntil, waitForAll, waitForAny, waitForResult };
|