@dealcrawl/sdk 2.10.0 → 2.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +246 -37
- package/dist/index.d.mts +1174 -50
- package/dist/index.d.ts +1174 -50
- package/dist/index.js +668 -52
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +665 -53
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.d.mts
CHANGED
|
@@ -19,7 +19,7 @@ interface ResponseMeta {
|
|
|
19
19
|
timestamp: string;
|
|
20
20
|
duration?: number;
|
|
21
21
|
}
|
|
22
|
-
type JobStatus = "pending" | "active" | "completed" | "failed" | "delayed" | "paused";
|
|
22
|
+
type JobStatus$1 = "pending" | "active" | "completed" | "failed" | "delayed" | "paused";
|
|
23
23
|
interface PaginatedResponse<T> {
|
|
24
24
|
data: T[];
|
|
25
25
|
pagination: {
|
|
@@ -130,10 +130,19 @@ interface DealScoreSummary {
|
|
|
130
130
|
};
|
|
131
131
|
}
|
|
132
132
|
interface ScreenshotResult {
|
|
133
|
+
/** Screenshot URL - private signed URL (default) or public URL (Enterprise opt-in) */
|
|
133
134
|
url: string;
|
|
135
|
+
/** Whether this is a public URL (false = private signed URL with TTL) */
|
|
136
|
+
isPublic: boolean;
|
|
137
|
+
/** Expiration timestamp for signed URLs (ISO 8601, only for private URLs) */
|
|
138
|
+
expiresAt?: string;
|
|
139
|
+
/** Screenshot width in pixels */
|
|
134
140
|
width: number;
|
|
141
|
+
/** Screenshot height in pixels */
|
|
135
142
|
height: number;
|
|
143
|
+
/** Image format */
|
|
136
144
|
format: "png" | "jpeg" | "webp";
|
|
145
|
+
/** File size in bytes */
|
|
137
146
|
sizeBytes: number;
|
|
138
147
|
}
|
|
139
148
|
interface AIExtraction {
|
|
@@ -438,7 +447,7 @@ interface JobResponse {
|
|
|
438
447
|
/** Unique job identifier */
|
|
439
448
|
jobId: string;
|
|
440
449
|
/** Current job status */
|
|
441
|
-
status: JobStatus;
|
|
450
|
+
status: JobStatus$1;
|
|
442
451
|
/** URL to check job status */
|
|
443
452
|
statusUrl: string;
|
|
444
453
|
/** Estimated time to completion */
|
|
@@ -468,6 +477,15 @@ interface BatchScrapeResultItem {
|
|
|
468
477
|
/** Error message if job creation failed */
|
|
469
478
|
error?: string;
|
|
470
479
|
}
|
|
480
|
+
/** Invalid URL entry (returned when ignoreInvalidURLs=true) */
|
|
481
|
+
interface InvalidURLEntry {
|
|
482
|
+
/** The invalid URL */
|
|
483
|
+
url: string;
|
|
484
|
+
/** Reference ID if provided */
|
|
485
|
+
ref?: string;
|
|
486
|
+
/** Reason for rejection */
|
|
487
|
+
reason: string;
|
|
488
|
+
}
|
|
471
489
|
/** Batch scrape response */
|
|
472
490
|
interface BatchScrapeResponse {
|
|
473
491
|
/** Batch ID for tracking */
|
|
@@ -482,6 +500,11 @@ interface BatchScrapeResponse {
|
|
|
482
500
|
results: BatchScrapeResultItem[];
|
|
483
501
|
/** Timestamp when batch was created */
|
|
484
502
|
createdAt: string;
|
|
503
|
+
/**
|
|
504
|
+
* Invalid URLs that were skipped (only present when ignoreInvalidURLs=true)
|
|
505
|
+
* Firecrawl-compatible field
|
|
506
|
+
*/
|
|
507
|
+
invalidURLs?: InvalidURLEntry[];
|
|
485
508
|
}
|
|
486
509
|
/** Batch status response */
|
|
487
510
|
interface BatchStatusResponse {
|
|
@@ -502,7 +525,7 @@ interface BatchStatusResponse {
|
|
|
502
525
|
/** Individual job statuses */
|
|
503
526
|
jobs: Array<{
|
|
504
527
|
id: string;
|
|
505
|
-
status: JobStatus;
|
|
528
|
+
status: JobStatus$1;
|
|
506
529
|
result?: unknown;
|
|
507
530
|
error?: string;
|
|
508
531
|
}>;
|
|
@@ -612,7 +635,7 @@ interface CheckpointInfo {
|
|
|
612
635
|
/** Job status response */
|
|
613
636
|
interface JobStatusResponse {
|
|
614
637
|
jobId: string;
|
|
615
|
-
status: JobStatus;
|
|
638
|
+
status: JobStatus$1;
|
|
616
639
|
progress?: number;
|
|
617
640
|
createdAt: string;
|
|
618
641
|
completedAt?: string;
|
|
@@ -652,7 +675,7 @@ interface ResumeJobResponse {
|
|
|
652
675
|
/** Job metrics response */
|
|
653
676
|
interface JobMetricsResponse {
|
|
654
677
|
jobId: string;
|
|
655
|
-
status: JobStatus;
|
|
678
|
+
status: JobStatus$1;
|
|
656
679
|
metrics: {
|
|
657
680
|
totalUrls: number;
|
|
658
681
|
successfulUrls: number;
|
|
@@ -671,7 +694,7 @@ interface CancelJobResponse {
|
|
|
671
694
|
interface JobSummary {
|
|
672
695
|
id: string;
|
|
673
696
|
type: "scrape" | "crawl" | "dork" | "extract";
|
|
674
|
-
status: JobStatus;
|
|
697
|
+
status: JobStatus$1;
|
|
675
698
|
input: Record<string, unknown>;
|
|
676
699
|
dealsFound?: number;
|
|
677
700
|
avgDealScore?: number;
|
|
@@ -769,10 +792,17 @@ interface CreateWebhookResponse {
|
|
|
769
792
|
minDealScore: number;
|
|
770
793
|
active: boolean;
|
|
771
794
|
}
|
|
772
|
-
/** List webhooks response */
|
|
795
|
+
/** List webhooks response (standardized list format) */
|
|
773
796
|
interface ListWebhooksResponse {
|
|
774
|
-
|
|
775
|
-
|
|
797
|
+
data: WebhookItem[];
|
|
798
|
+
pagination: {
|
|
799
|
+
page: number;
|
|
800
|
+
limit: number;
|
|
801
|
+
total: number;
|
|
802
|
+
totalPages: number;
|
|
803
|
+
hasMore: boolean;
|
|
804
|
+
};
|
|
805
|
+
meta?: Record<string, unknown>;
|
|
776
806
|
}
|
|
777
807
|
/** Update webhook response */
|
|
778
808
|
interface UpdateWebhookResponse {
|
|
@@ -964,7 +994,7 @@ interface AgentStepResponse {
|
|
|
964
994
|
currentUrl?: string;
|
|
965
995
|
}
|
|
966
996
|
/** Agent completion reason */
|
|
967
|
-
type AgentCompletionReason = "goal_achieved" | "max_steps_reached" | "stuck" | "error"
|
|
997
|
+
type AgentCompletionReason = "goal_achieved" | "max_steps_reached" | "stuck" | "error";
|
|
968
998
|
/** Agent result (from completed job) */
|
|
969
999
|
interface AgentResultResponse {
|
|
970
1000
|
/** Extracted/collected data (matches provided schema if given) */
|
|
@@ -1013,6 +1043,113 @@ interface SchemaGenerationResponse {
|
|
|
1013
1043
|
/** Confidence score (0-1) in the generated schema */
|
|
1014
1044
|
confidence: number;
|
|
1015
1045
|
}
|
|
1046
|
+
/** SSE token response */
|
|
1047
|
+
interface SSETokenResponse {
|
|
1048
|
+
/** Short-lived SSE authentication token */
|
|
1049
|
+
token: string;
|
|
1050
|
+
/** Expiration timestamp (ISO 8601) */
|
|
1051
|
+
expiresAt: string;
|
|
1052
|
+
/** TTL in seconds (default: 300 = 5 minutes) */
|
|
1053
|
+
expiresIn: number;
|
|
1054
|
+
/** Token type identifier */
|
|
1055
|
+
type: "sse_token";
|
|
1056
|
+
/** Usage instructions for the token */
|
|
1057
|
+
usage: string;
|
|
1058
|
+
}
|
|
1059
|
+
/** SSE connection limits response */
|
|
1060
|
+
interface SSEConnectionLimitsResponse {
|
|
1061
|
+
/** Client tier */
|
|
1062
|
+
tier: "free" | "pro" | "enterprise";
|
|
1063
|
+
/** SSE connection limits */
|
|
1064
|
+
sse: {
|
|
1065
|
+
/** Maximum concurrent connections allowed */
|
|
1066
|
+
maxConnections: number;
|
|
1067
|
+
/** Current active connections */
|
|
1068
|
+
currentConnections: number;
|
|
1069
|
+
/** Available connection slots */
|
|
1070
|
+
available: number;
|
|
1071
|
+
};
|
|
1072
|
+
}
|
|
1073
|
+
/** Tier-specific TTL limits */
|
|
1074
|
+
interface TtlLimits {
|
|
1075
|
+
/** Minimum TTL in seconds */
|
|
1076
|
+
min: number;
|
|
1077
|
+
/** Maximum TTL in seconds */
|
|
1078
|
+
max: number;
|
|
1079
|
+
/** Default TTL in seconds */
|
|
1080
|
+
default: number;
|
|
1081
|
+
}
|
|
1082
|
+
/** Formatted TTL limits (human-readable) */
|
|
1083
|
+
interface FormattedTtlLimits {
|
|
1084
|
+
/** Minimum TTL formatted (e.g., "1 hour") */
|
|
1085
|
+
min: string;
|
|
1086
|
+
/** Maximum TTL formatted (e.g., "30 days") */
|
|
1087
|
+
max: string;
|
|
1088
|
+
/** Default TTL formatted (e.g., "7 days") */
|
|
1089
|
+
default: string;
|
|
1090
|
+
}
|
|
1091
|
+
/** Screenshot signed URL refresh response */
|
|
1092
|
+
interface ScreenshotRefreshResponse {
|
|
1093
|
+
/** Success status */
|
|
1094
|
+
success: boolean;
|
|
1095
|
+
/** New signed URL */
|
|
1096
|
+
url: string;
|
|
1097
|
+
/** Expiration timestamp (ISO 8601) */
|
|
1098
|
+
expiresAt: string;
|
|
1099
|
+
/** TTL in seconds */
|
|
1100
|
+
ttl: number;
|
|
1101
|
+
/** Tier-specific limits for reference */
|
|
1102
|
+
tierLimits: TtlLimits;
|
|
1103
|
+
/** Additional metadata */
|
|
1104
|
+
metadata: {
|
|
1105
|
+
/** File path in storage */
|
|
1106
|
+
path: string;
|
|
1107
|
+
/** Bucket name */
|
|
1108
|
+
bucket: string;
|
|
1109
|
+
/** Timestamp when URL was refreshed */
|
|
1110
|
+
refreshedAt: string;
|
|
1111
|
+
};
|
|
1112
|
+
}
|
|
1113
|
+
/** Screenshot TTL limits response */
|
|
1114
|
+
interface ScreenshotLimitsResponse {
|
|
1115
|
+
/** Client tier */
|
|
1116
|
+
tier: "free" | "pro" | "enterprise";
|
|
1117
|
+
/** TTL limits in seconds */
|
|
1118
|
+
limits: TtlLimits;
|
|
1119
|
+
/** Human-readable formatted limits */
|
|
1120
|
+
formattedLimits: FormattedTtlLimits;
|
|
1121
|
+
}
|
|
1122
|
+
/**
|
|
1123
|
+
* Metadata extracted during HTML to Markdown conversion
|
|
1124
|
+
*/
|
|
1125
|
+
interface ConversionMetadata {
|
|
1126
|
+
/** Page title if found */
|
|
1127
|
+
title?: string;
|
|
1128
|
+
/** Word count in resulting markdown */
|
|
1129
|
+
wordCount: number;
|
|
1130
|
+
/** Number of links in markdown */
|
|
1131
|
+
linkCount: number;
|
|
1132
|
+
/** Number of images in markdown */
|
|
1133
|
+
imageCount: number;
|
|
1134
|
+
/** Conversion duration in milliseconds */
|
|
1135
|
+
conversionTimeMs: number;
|
|
1136
|
+
}
|
|
1137
|
+
/**
|
|
1138
|
+
* HTML to Markdown conversion response
|
|
1139
|
+
*/
|
|
1140
|
+
interface ConvertResponse {
|
|
1141
|
+
/** Success status */
|
|
1142
|
+
success: boolean;
|
|
1143
|
+
/** Conversion result data */
|
|
1144
|
+
data: {
|
|
1145
|
+
/** Generated markdown content */
|
|
1146
|
+
markdown: string;
|
|
1147
|
+
/** Extraction metadata */
|
|
1148
|
+
metadata: ConversionMetadata;
|
|
1149
|
+
/** Warnings (truncation, excluded elements, etc.) */
|
|
1150
|
+
warnings?: string[];
|
|
1151
|
+
};
|
|
1152
|
+
}
|
|
1016
1153
|
|
|
1017
1154
|
/**
|
|
1018
1155
|
* Polling Utilities
|
|
@@ -1092,10 +1229,19 @@ interface ScreenshotOptions {
|
|
|
1092
1229
|
/**
|
|
1093
1230
|
* Request public URL (Enterprise only, default: false)
|
|
1094
1231
|
* SECURITY WARNING: Public URLs can expose personal data, copyrighted content, and tokens
|
|
1095
|
-
* By default, screenshots use private signed URLs with
|
|
1232
|
+
* By default, screenshots use private signed URLs with tier-specific expiration
|
|
1096
1233
|
* Requires Enterprise tier
|
|
1097
1234
|
*/
|
|
1098
1235
|
publicUrl?: boolean;
|
|
1236
|
+
/**
|
|
1237
|
+
* TTL for signed URLs in seconds (only for private screenshots)
|
|
1238
|
+
* Tier limits:
|
|
1239
|
+
* - Free: 3600 (1h) - 86400 (24h), default: 86400
|
|
1240
|
+
* - Pro: 3600 (1h) - 604800 (7d), default: 604800
|
|
1241
|
+
* - Enterprise: 3600 (1h) - 2592000 (30d), default: 604800
|
|
1242
|
+
* If not specified, uses tier-specific default
|
|
1243
|
+
*/
|
|
1244
|
+
signedUrlTtl?: number;
|
|
1099
1245
|
}
|
|
1100
1246
|
/** Options for scraping a single page */
|
|
1101
1247
|
interface ScrapeOptions {
|
|
@@ -1158,8 +1304,6 @@ interface BatchScrapeItem {
|
|
|
1158
1304
|
outputMarkdown?: boolean;
|
|
1159
1305
|
/** Override markdownBaseUrl for this URL */
|
|
1160
1306
|
markdownBaseUrl?: string;
|
|
1161
|
-
/** Override actions for this URL */
|
|
1162
|
-
actions?: ActionInput[];
|
|
1163
1307
|
}
|
|
1164
1308
|
/** Default options applied to all URLs in a batch */
|
|
1165
1309
|
interface BatchScrapeDefaults {
|
|
@@ -1195,8 +1339,6 @@ interface BatchScrapeDefaults {
|
|
|
1195
1339
|
outputMarkdown?: boolean;
|
|
1196
1340
|
/** Base URL for resolving relative URLs in markdown */
|
|
1197
1341
|
markdownBaseUrl?: string;
|
|
1198
|
-
/** Browser actions to execute before scraping */
|
|
1199
|
-
actions?: ActionInput[];
|
|
1200
1342
|
}
|
|
1201
1343
|
/** Options for batch scraping multiple URLs */
|
|
1202
1344
|
interface BatchScrapeOptions {
|
|
@@ -1208,8 +1350,13 @@ interface BatchScrapeOptions {
|
|
|
1208
1350
|
webhookUrl?: string;
|
|
1209
1351
|
/** Priority for batch jobs (1-10, higher = faster) */
|
|
1210
1352
|
priority?: number;
|
|
1211
|
-
/** Delay between job submissions in ms (0-5000) */
|
|
1212
|
-
|
|
1353
|
+
/** Delay between job submissions in ms (0-5000, default: 0) */
|
|
1354
|
+
delayMs?: number;
|
|
1355
|
+
/**
|
|
1356
|
+
* Continue processing even if some URLs are invalid (default: false)
|
|
1357
|
+
* Invalid URLs will be returned in the response with status: "failed"
|
|
1358
|
+
*/
|
|
1359
|
+
ignoreInvalidURLs?: boolean;
|
|
1213
1360
|
}
|
|
1214
1361
|
/** AI provider for search optimization */
|
|
1215
1362
|
type SearchAiProvider = "openai" | "anthropic";
|
|
@@ -1237,11 +1384,11 @@ interface SearchOptions {
|
|
|
1237
1384
|
/** Search query (required) */
|
|
1238
1385
|
query: string;
|
|
1239
1386
|
/** Maximum number of results (1-100, default: 10) */
|
|
1240
|
-
|
|
1241
|
-
/** Auto-scrape
|
|
1242
|
-
|
|
1243
|
-
/**
|
|
1244
|
-
|
|
1387
|
+
limit?: number;
|
|
1388
|
+
/** Auto-scrape search results (default: false) */
|
|
1389
|
+
scrapeResults?: boolean;
|
|
1390
|
+
/** Maximum results to auto-scrape when scrapeResults=true (1-10, default: 5) */
|
|
1391
|
+
maxScrapeResults?: number;
|
|
1245
1392
|
/** Use AI to optimize the search query */
|
|
1246
1393
|
useAiOptimization?: boolean;
|
|
1247
1394
|
/** AI provider for query optimization */
|
|
@@ -1275,7 +1422,7 @@ interface PriceRange {
|
|
|
1275
1422
|
interface CrawlOptions {
|
|
1276
1423
|
/** Starting URL for the crawl (required) */
|
|
1277
1424
|
url: string;
|
|
1278
|
-
/**
|
|
1425
|
+
/** @deprecated Use 'url' instead. Kept for backward compatibility. */
|
|
1279
1426
|
startUrl?: string;
|
|
1280
1427
|
/** Prompt to guide the crawl (optional) */
|
|
1281
1428
|
prompt?: string;
|
|
@@ -1352,7 +1499,7 @@ interface CrawlTemplate {
|
|
|
1352
1499
|
defaultOptions: Partial<CrawlOptions>;
|
|
1353
1500
|
}
|
|
1354
1501
|
/** LLM model options for extraction */
|
|
1355
|
-
type ExtractModel = "gpt-4o-mini" | "gpt-4o" | "claude-3-haiku" | "claude-3-sonnet";
|
|
1502
|
+
type ExtractModel = "gpt-4o-mini" | "gpt-4o" | "gpt-4o-2024-11-20" | "o1-mini" | "claude-3-5-haiku-20241022" | "claude-3-5-sonnet-20241022" | "claude-3-opus-20240229";
|
|
1356
1503
|
/** Options for structured data extraction */
|
|
1357
1504
|
interface ExtractOptions {
|
|
1358
1505
|
/** URL to extract data from (required) */
|
|
@@ -1395,8 +1542,6 @@ interface DorkOptions {
|
|
|
1395
1542
|
inTitle?: string;
|
|
1396
1543
|
/** Maximum results to return (default: 10, max: 100) */
|
|
1397
1544
|
maxResults?: number;
|
|
1398
|
-
/** Region code (2 letter ISO code) */
|
|
1399
|
-
region?: string;
|
|
1400
1545
|
}
|
|
1401
1546
|
/** Options for getting job deals */
|
|
1402
1547
|
interface GetDealsOptions {
|
|
@@ -1472,19 +1617,25 @@ interface ExportDealsOptions {
|
|
|
1472
1617
|
format?: ExportFormat;
|
|
1473
1618
|
/** Minimum deal score */
|
|
1474
1619
|
minScore?: number;
|
|
1475
|
-
/** Maximum price */
|
|
1476
|
-
maxPrice?: number;
|
|
1477
1620
|
/** Filter by category */
|
|
1478
1621
|
category?: string;
|
|
1479
|
-
/**
|
|
1480
|
-
|
|
1622
|
+
/** Filter by sync status to DealUp */
|
|
1623
|
+
synced?: boolean;
|
|
1624
|
+
/** Filter from date (ISO string) */
|
|
1625
|
+
fromDate?: string;
|
|
1626
|
+
/** Filter to date (ISO string) */
|
|
1627
|
+
toDate?: string;
|
|
1628
|
+
/** Maximum number of deals to export (default: 500, max: 1000) */
|
|
1629
|
+
limit?: number;
|
|
1481
1630
|
}
|
|
1482
1631
|
/** Webhook event types */
|
|
1483
1632
|
type WebhookEvent = "deal.found" | "deal.synced" | "crawl.completed" | "crawl.failed";
|
|
1484
1633
|
/** Options for creating a webhook */
|
|
1485
1634
|
interface CreateWebhookOptions {
|
|
1486
|
-
/** Event
|
|
1487
|
-
|
|
1635
|
+
/** Event types to subscribe to (array) */
|
|
1636
|
+
events?: WebhookEvent[];
|
|
1637
|
+
/** @deprecated Use 'events' array instead */
|
|
1638
|
+
event?: WebhookEvent;
|
|
1488
1639
|
/** URL to receive webhook notifications */
|
|
1489
1640
|
url: string;
|
|
1490
1641
|
/** Secret for signature verification */
|
|
@@ -1513,7 +1664,7 @@ interface UpdateWebhookOptions {
|
|
|
1513
1664
|
* API key scope - Must match @dealcrawl/shared/src/types/api-key.types.ts
|
|
1514
1665
|
* These are the actual scopes enforced by the backend via requireScope() middleware
|
|
1515
1666
|
*/
|
|
1516
|
-
type ApiKeyScope = "scrape" | "
|
|
1667
|
+
type ApiKeyScope = "scrape" | "crawl" | "dork" | "extract" | "agent" | "status" | "data:read" | "data:export" | "keys:manage" | "webhooks:manage";
|
|
1517
1668
|
/**
|
|
1518
1669
|
* All available scopes (for reference and validation)
|
|
1519
1670
|
*/
|
|
@@ -1691,6 +1842,71 @@ interface SchemaGenerationOptions {
|
|
|
1691
1842
|
/** LLM provider for generation (default: openai) */
|
|
1692
1843
|
model?: AgentModel;
|
|
1693
1844
|
}
|
|
1845
|
+
/**
|
|
1846
|
+
* Options for subscribing to SSE events
|
|
1847
|
+
*/
|
|
1848
|
+
interface SSESubscribeOptions {
|
|
1849
|
+
/**
|
|
1850
|
+
* Last event ID for reconnection/replay
|
|
1851
|
+
* If provided, server will replay missed events since this ID
|
|
1852
|
+
*/
|
|
1853
|
+
lastEventId?: string;
|
|
1854
|
+
/**
|
|
1855
|
+
* Callback for received events
|
|
1856
|
+
* Called for each event received from the server
|
|
1857
|
+
*/
|
|
1858
|
+
onEvent?: (event: MessageEvent) => void;
|
|
1859
|
+
/**
|
|
1860
|
+
* Callback for errors
|
|
1861
|
+
* Called when connection error occurs
|
|
1862
|
+
*/
|
|
1863
|
+
onError?: (error: Error) => void;
|
|
1864
|
+
/**
|
|
1865
|
+
* Callback for connection open
|
|
1866
|
+
* Called when SSE connection is established
|
|
1867
|
+
*/
|
|
1868
|
+
onOpen?: (event: Event) => void;
|
|
1869
|
+
/**
|
|
1870
|
+
* Auto-reconnect on connection loss (default: true)
|
|
1871
|
+
* If true, EventSource will automatically reconnect
|
|
1872
|
+
*/
|
|
1873
|
+
autoReconnect?: boolean;
|
|
1874
|
+
/**
|
|
1875
|
+
* Reconnection delay in milliseconds (default: 3000)
|
|
1876
|
+
* How long to wait before attempting reconnection
|
|
1877
|
+
*/
|
|
1878
|
+
reconnectDelay?: number;
|
|
1879
|
+
}
|
|
1880
|
+
/**
|
|
1881
|
+
* Options for HTML to Markdown conversion
|
|
1882
|
+
*/
|
|
1883
|
+
interface ConvertOptions {
|
|
1884
|
+
/** HTML content to convert (required, max 10MB) */
|
|
1885
|
+
html: string;
|
|
1886
|
+
/** Base URL for resolving relative URLs */
|
|
1887
|
+
baseUrl?: string;
|
|
1888
|
+
/** Conversion options */
|
|
1889
|
+
options?: {
|
|
1890
|
+
/** Enable GFM tables (default: true) */
|
|
1891
|
+
gfmTables?: boolean;
|
|
1892
|
+
/** Enable GFM strikethrough (default: true) */
|
|
1893
|
+
gfmStrikethrough?: boolean;
|
|
1894
|
+
/** Enable GFM task lists (default: true) */
|
|
1895
|
+
gfmTaskLists?: boolean;
|
|
1896
|
+
/** Remove noise elements like nav, footer, ads (default: true) */
|
|
1897
|
+
removeNoise?: boolean;
|
|
1898
|
+
/** CSS selectors to exclude from conversion */
|
|
1899
|
+
excludeSelectors?: string[];
|
|
1900
|
+
/** Resolve relative URLs to absolute (default: true) */
|
|
1901
|
+
absoluteUrls?: boolean;
|
|
1902
|
+
/** Maximum output length in characters (0 = unlimited, max: 1,000,000) */
|
|
1903
|
+
maxLength?: number;
|
|
1904
|
+
/** Include images in output (default: true) */
|
|
1905
|
+
includeImages?: boolean;
|
|
1906
|
+
/** Include links in output (default: true) */
|
|
1907
|
+
includeLinks?: boolean;
|
|
1908
|
+
};
|
|
1909
|
+
}
|
|
1694
1910
|
|
|
1695
1911
|
/**
|
|
1696
1912
|
* Account Resource
|
|
@@ -2035,6 +2251,257 @@ declare class AgentResource {
|
|
|
2035
2251
|
generateSchema(options: SchemaGenerationOptions): Promise<SchemaGenerationResponse>;
|
|
2036
2252
|
}
|
|
2037
2253
|
|
|
2254
|
+
/**
|
|
2255
|
+
* Auth Resource
|
|
2256
|
+
* Handles SSE (Server-Sent Events) authentication
|
|
2257
|
+
*/
|
|
2258
|
+
|
|
2259
|
+
/**
|
|
2260
|
+
* Options for generating an SSE token
|
|
2261
|
+
*/
|
|
2262
|
+
interface GenerateSSETokenOptions {
|
|
2263
|
+
/**
|
|
2264
|
+
* Optional job ID to restrict token to a specific job
|
|
2265
|
+
* If provided, token can only be used for /v1/events/:jobId
|
|
2266
|
+
* If omitted, token can be used for /v1/events (all events)
|
|
2267
|
+
*/
|
|
2268
|
+
jobId?: string;
|
|
2269
|
+
}
|
|
2270
|
+
/**
|
|
2271
|
+
* Auth resource class
|
|
2272
|
+
* Provides methods for SSE authentication
|
|
2273
|
+
*
|
|
2274
|
+
* @example
|
|
2275
|
+
* ```ts
|
|
2276
|
+
* // Generate SSE token for browser EventSource
|
|
2277
|
+
* const { token, expiresAt } = await client.auth.generateSSEToken();
|
|
2278
|
+
*
|
|
2279
|
+
* // Use in browser
|
|
2280
|
+
* const eventSource = new EventSource(`/v1/events?token=${token}`);
|
|
2281
|
+
*
|
|
2282
|
+
* // Token for specific job
|
|
2283
|
+
* const jobToken = await client.auth.generateSSEToken({ jobId: "job_123" });
|
|
2284
|
+
* const jobEvents = new EventSource(`/v1/events/job_123?token=${jobToken.token}`);
|
|
2285
|
+
* ```
|
|
2286
|
+
*/
|
|
2287
|
+
declare class AuthResource {
|
|
2288
|
+
private ctx;
|
|
2289
|
+
constructor(ctx: RequestContext);
|
|
2290
|
+
/**
|
|
2291
|
+
* Generate SSE authentication token
|
|
2292
|
+
*
|
|
2293
|
+
* Required for browser-based SSE connections because EventSource API
|
|
2294
|
+
* doesn't support custom headers. Token is short-lived (5 minutes).
|
|
2295
|
+
*
|
|
2296
|
+
* Security:
|
|
2297
|
+
* - Requires valid API key (Bearer token)
|
|
2298
|
+
* - Token expires in 5 minutes
|
|
2299
|
+
* - Token can be restricted to specific job
|
|
2300
|
+
* - Token stored in Redis (revocable)
|
|
2301
|
+
*
|
|
2302
|
+
* @example
|
|
2303
|
+
* ```ts
|
|
2304
|
+
* // 1. Generate token
|
|
2305
|
+
* const { token, expiresAt } = await client.auth.generateSSEToken();
|
|
2306
|
+
* console.log(`Token expires at: ${expiresAt}`);
|
|
2307
|
+
*
|
|
2308
|
+
* // 2. Use in browser EventSource
|
|
2309
|
+
* const eventSource = new EventSource(`/v1/events?token=${token}`);
|
|
2310
|
+
*
|
|
2311
|
+
* eventSource.addEventListener('job.completed', (event) => {
|
|
2312
|
+
* const data = JSON.parse(event.data);
|
|
2313
|
+
* console.log('Job completed:', data);
|
|
2314
|
+
* });
|
|
2315
|
+
*
|
|
2316
|
+
* // 3. For specific job only
|
|
2317
|
+
* const jobToken = await client.auth.generateSSEToken({ jobId: "job_abc123" });
|
|
2318
|
+
* const jobEvents = new EventSource(`/v1/events/job_abc123?token=${jobToken.token}`);
|
|
2319
|
+
* ```
|
|
2320
|
+
*/
|
|
2321
|
+
generateSSEToken(options?: GenerateSSETokenOptions): Promise<SSETokenResponse>;
|
|
2322
|
+
/**
|
|
2323
|
+
* Get SSE connection limits for current tier
|
|
2324
|
+
*
|
|
2325
|
+
* Shows how many concurrent SSE connections are allowed
|
|
2326
|
+
* and how many are currently active.
|
|
2327
|
+
*
|
|
2328
|
+
* Tier limits:
|
|
2329
|
+
* - Free: 10 concurrent connections
|
|
2330
|
+
* - Pro: 50 concurrent connections
|
|
2331
|
+
* - Enterprise: 200 concurrent connections
|
|
2332
|
+
*
|
|
2333
|
+
* @example
|
|
2334
|
+
* ```ts
|
|
2335
|
+
* const limits = await client.auth.getLimits();
|
|
2336
|
+
*
|
|
2337
|
+
* console.log(`Tier: ${limits.tier}`);
|
|
2338
|
+
* console.log(`Max connections: ${limits.sse.maxConnections}`);
|
|
2339
|
+
* console.log(`Current connections: ${limits.sse.currentConnections}`);
|
|
2340
|
+
* console.log(`Available: ${limits.sse.available}`);
|
|
2341
|
+
*
|
|
2342
|
+
* // Check before opening new connection
|
|
2343
|
+
* if (limits.sse.available > 0) {
|
|
2344
|
+
* const token = await client.auth.generateSSEToken();
|
|
2345
|
+
* const eventSource = new EventSource(`/v1/events?token=${token.token}`);
|
|
2346
|
+
* } else {
|
|
2347
|
+
* console.error('No available SSE connection slots');
|
|
2348
|
+
* }
|
|
2349
|
+
* ```
|
|
2350
|
+
*/
|
|
2351
|
+
getLimits(): Promise<SSEConnectionLimitsResponse>;
|
|
2352
|
+
}
|
|
2353
|
+
|
|
2354
|
+
/**
|
|
2355
|
+
* Convert Resource
|
|
2356
|
+
* Handles HTML to Markdown conversion
|
|
2357
|
+
*/
|
|
2358
|
+
|
|
2359
|
+
/**
|
|
2360
|
+
* Convert resource class
|
|
2361
|
+
* Provides methods for HTML to Markdown conversion
|
|
2362
|
+
*
|
|
2363
|
+
* @example
|
|
2364
|
+
* ```ts
|
|
2365
|
+
* // Convert HTML to markdown
|
|
2366
|
+
* const result = await client.convert.htmlToMarkdown({
|
|
2367
|
+
* html: "<h1>Title</h1><p>Content</p>",
|
|
2368
|
+
* baseUrl: "https://example.com",
|
|
2369
|
+
* options: {
|
|
2370
|
+
* gfmTables: true,
|
|
2371
|
+
* removeNoise: true
|
|
2372
|
+
* }
|
|
2373
|
+
* });
|
|
2374
|
+
*
|
|
2375
|
+
* console.log(result.data.markdown);
|
|
2376
|
+
* console.log(result.data.metadata.wordCount);
|
|
2377
|
+
* ```
|
|
2378
|
+
*/
|
|
2379
|
+
declare class ConvertResource {
|
|
2380
|
+
private ctx;
|
|
2381
|
+
constructor(ctx: RequestContext);
|
|
2382
|
+
/**
|
|
2383
|
+
* Convert HTML to Markdown
|
|
2384
|
+
*
|
|
2385
|
+
* Transforms raw HTML content into clean, readable Markdown using GitHub Flavored Markdown (GFM).
|
|
2386
|
+
* Useful for:
|
|
2387
|
+
* - Converting scraped HTML to markdown for LLM processing
|
|
2388
|
+
* - Cleaning up messy HTML from web pages
|
|
2389
|
+
* - Extracting main content while removing noise (ads, nav, footer)
|
|
2390
|
+
* - Creating documentation from HTML sources
|
|
2391
|
+
*
|
|
2392
|
+
* Features:
|
|
2393
|
+
* - GFM table, strikethrough, and task list support
|
|
2394
|
+
* - Automatic noise removal (scripts, ads, navigation)
|
|
2395
|
+
* - Relative URL resolution
|
|
2396
|
+
* - Custom element exclusion via CSS selectors
|
|
2397
|
+
* - Output length limiting
|
|
2398
|
+
*
|
|
2399
|
+
* @param options - Conversion options
|
|
2400
|
+
* @returns Conversion result with markdown, metadata, and warnings
|
|
2401
|
+
*
|
|
2402
|
+
* @example Basic usage
|
|
2403
|
+
* ```ts
|
|
2404
|
+
* const result = await client.convert.htmlToMarkdown({
|
|
2405
|
+
* html: "<h1>Product</h1><p>Price: $99</p>"
|
|
2406
|
+
* });
|
|
2407
|
+
* console.log(result.data.markdown);
|
|
2408
|
+
* ```
|
|
2409
|
+
*
|
|
2410
|
+
* @example With all options
|
|
2411
|
+
* ```ts
|
|
2412
|
+
* const result = await client.convert.htmlToMarkdown({
|
|
2413
|
+
* html: htmlContent,
|
|
2414
|
+
* baseUrl: "https://shop.example.com",
|
|
2415
|
+
* options: {
|
|
2416
|
+
* gfmTables: true,
|
|
2417
|
+
* removeNoise: true,
|
|
2418
|
+
* excludeSelectors: [".advertisement", "#sidebar"],
|
|
2419
|
+
* absoluteUrls: true,
|
|
2420
|
+
* maxLength: 100000,
|
|
2421
|
+
* includeImages: true,
|
|
2422
|
+
* includeLinks: true
|
|
2423
|
+
* }
|
|
2424
|
+
* });
|
|
2425
|
+
*
|
|
2426
|
+
* // Check metadata
|
|
2427
|
+
* console.log(`Words: ${result.data.metadata.wordCount}`);
|
|
2428
|
+
* console.log(`Links: ${result.data.metadata.linkCount}`);
|
|
2429
|
+
* console.log(`Images: ${result.data.metadata.imageCount}`);
|
|
2430
|
+
* console.log(`Conversion time: ${result.data.metadata.conversionTimeMs}ms`);
|
|
2431
|
+
*
|
|
2432
|
+
* // Check for warnings
|
|
2433
|
+
* if (result.data.warnings?.length) {
|
|
2434
|
+
* console.warn("Conversion warnings:", result.data.warnings);
|
|
2435
|
+
* }
|
|
2436
|
+
* ```
|
|
2437
|
+
*
|
|
2438
|
+
* @example Converting scraped HTML
|
|
2439
|
+
* ```ts
|
|
2440
|
+
* // First scrape a page
|
|
2441
|
+
* const scrapeJob = await client.scrape.create({
|
|
2442
|
+
* url: "https://example.com/article"
|
|
2443
|
+
* });
|
|
2444
|
+
* const scrapeResult = await client.waitForResult(scrapeJob.jobId);
|
|
2445
|
+
*
|
|
2446
|
+
* // Then convert HTML to markdown
|
|
2447
|
+
* const markdown = await client.convert.htmlToMarkdown({
|
|
2448
|
+
* html: scrapeResult.data.html,
|
|
2449
|
+
* baseUrl: scrapeResult.data.url,
|
|
2450
|
+
* options: {
|
|
2451
|
+
* removeNoise: true,
|
|
2452
|
+
* onlyMainContent: true
|
|
2453
|
+
* }
|
|
2454
|
+
* });
|
|
2455
|
+
* ```
|
|
2456
|
+
*/
|
|
2457
|
+
htmlToMarkdown(options: ConvertOptions): Promise<ConvertResponse>;
|
|
2458
|
+
/**
|
|
2459
|
+
* Alias for htmlToMarkdown() for convenience
|
|
2460
|
+
*
|
|
2461
|
+
* @example
|
|
2462
|
+
* ```ts
|
|
2463
|
+
* const result = await client.convert.toMarkdown({
|
|
2464
|
+
* html: "<h1>Hello</h1>"
|
|
2465
|
+
* });
|
|
2466
|
+
* ```
|
|
2467
|
+
*/
|
|
2468
|
+
toMarkdown(options: ConvertOptions): Promise<ConvertResponse>;
|
|
2469
|
+
/**
|
|
2470
|
+
* Convert HTML with minimal options (just the HTML content)
|
|
2471
|
+
* Uses all default settings
|
|
2472
|
+
*
|
|
2473
|
+
* @param html - HTML content to convert
|
|
2474
|
+
* @param baseUrl - Optional base URL for resolving relative links
|
|
2475
|
+
* @returns Conversion result
|
|
2476
|
+
*
|
|
2477
|
+
* @example
|
|
2478
|
+
* ```ts
|
|
2479
|
+
* const result = await client.convert.quick(
|
|
2480
|
+
* "<h1>Title</h1><p>Content</p>",
|
|
2481
|
+
* "https://example.com"
|
|
2482
|
+
* );
|
|
2483
|
+
* console.log(result.data.markdown);
|
|
2484
|
+
* ```
|
|
2485
|
+
*/
|
|
2486
|
+
quick(html: string, baseUrl?: string): Promise<ConvertResponse>;
|
|
2487
|
+
/**
|
|
2488
|
+
* Convert HTML with noise removal enabled
|
|
2489
|
+
* Removes navigation, footer, ads, scripts, and other clutter
|
|
2490
|
+
*
|
|
2491
|
+
* @param html - HTML content to convert
|
|
2492
|
+
* @param baseUrl - Optional base URL
|
|
2493
|
+
* @returns Conversion result with clean markdown
|
|
2494
|
+
*
|
|
2495
|
+
* @example
|
|
2496
|
+
* ```ts
|
|
2497
|
+
* // Extract just the main content from a messy page
|
|
2498
|
+
* const result = await client.convert.clean(messyHtml, "https://example.com");
|
|
2499
|
+
* console.log(result.data.markdown); // Clean, readable markdown
|
|
2500
|
+
* ```
|
|
2501
|
+
*/
|
|
2502
|
+
clean(html: string, baseUrl?: string): Promise<ConvertResponse>;
|
|
2503
|
+
}
|
|
2504
|
+
|
|
2038
2505
|
/**
|
|
2039
2506
|
* Crawl Resource
|
|
2040
2507
|
* Handles website crawling operations
|
|
@@ -2220,17 +2687,9 @@ declare class DataResource {
|
|
|
2220
2687
|
* @example
|
|
2221
2688
|
* ```ts
|
|
2222
2689
|
* const electronicsDeals = await client.data.getDealsByCategory("electronics");
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
): Promise<ListDealsResponse> {
|
|
2227
|
-
if (!category || !category.trim()) {
|
|
2228
|
-
throw new Error("category is required and cannot be empty");
|
|
2229
|
-
}
|
|
2230
|
-
return this.listDeals({ category, ...options });
|
|
2231
|
-
} return this.listDeals({ category, ...options });
|
|
2232
|
-
}
|
|
2233
|
-
|
|
2690
|
+
* ```
|
|
2691
|
+
*/
|
|
2692
|
+
getDealsByCategory(category: string, options?: Omit<ListDealsOptions, "category">): Promise<ListDealsResponse>;
|
|
2234
2693
|
/**
|
|
2235
2694
|
* Get unsynced deals (not yet sent to DealUp)
|
|
2236
2695
|
*
|
|
@@ -2385,6 +2844,177 @@ declare class DorkResource {
|
|
|
2385
2844
|
buildQuery(options: DorkOptions): string;
|
|
2386
2845
|
}
|
|
2387
2846
|
|
|
2847
|
+
/**
|
|
2848
|
+
* Events Resource
|
|
2849
|
+
* Handles real-time SSE (Server-Sent Events) streaming
|
|
2850
|
+
*/
|
|
2851
|
+
|
|
2852
|
+
/**
|
|
2853
|
+
* Events resource class
|
|
2854
|
+
* Provides methods for subscribing to real-time job events via SSE
|
|
2855
|
+
*
|
|
2856
|
+
* IMPORTANT: Browser Support Only
|
|
2857
|
+
* This resource uses the EventSource API which is only available in browsers.
|
|
2858
|
+
* For Node.js, use polling via client.status.get() instead.
|
|
2859
|
+
*
|
|
2860
|
+
* @example Browser Usage
|
|
2861
|
+
* ```ts
|
|
2862
|
+
* // 1. Generate SSE token (required for EventSource)
|
|
2863
|
+
* const { token } = await client.auth.generateSSEToken();
|
|
2864
|
+
*
|
|
2865
|
+
* // 2. Subscribe to all events
|
|
2866
|
+
* const eventSource = client.events.subscribe({
|
|
2867
|
+
* token,
|
|
2868
|
+
* onEvent: (event) => {
|
|
2869
|
+
* console.log('Event:', event.type, event.data);
|
|
2870
|
+
* }
|
|
2871
|
+
* });
|
|
2872
|
+
*
|
|
2873
|
+
* // 3. Or subscribe to specific job
|
|
2874
|
+
* const jobEvents = client.events.subscribeToJob('job_123', {
|
|
2875
|
+
* token,
|
|
2876
|
+
* onEvent: (event) => {
|
|
2877
|
+
* const data = JSON.parse(event.data);
|
|
2878
|
+
* console.log('Job event:', data);
|
|
2879
|
+
* }
|
|
2880
|
+
* });
|
|
2881
|
+
*
|
|
2882
|
+
* // 4. Clean up when done
|
|
2883
|
+
* eventSource.close();
|
|
2884
|
+
* ```
|
|
2885
|
+
*/
|
|
2886
|
+
declare class EventsResource {
|
|
2887
|
+
private ctx;
|
|
2888
|
+
constructor(ctx: RequestContext);
|
|
2889
|
+
/**
|
|
2890
|
+
* Subscribe to all events for authenticated client
|
|
2891
|
+
*
|
|
2892
|
+
* Opens an SSE connection to receive real-time events for all jobs.
|
|
2893
|
+
* Requires an SSE token obtained via client.auth.generateSSEToken().
|
|
2894
|
+
*
|
|
2895
|
+
* Event Types:
|
|
2896
|
+
* - Job lifecycle: job.created, job.queued, job.started, job.progress,
|
|
2897
|
+
* job.completed, job.failed, job.cancelled
|
|
2898
|
+
* - Job details: job.log, job.metric, job.alert, job.checkpoint
|
|
2899
|
+
* - Deals: deal.found, deal.validated
|
|
2900
|
+
* - System: ping, connection.open, connection.close, error
|
|
2901
|
+
*
|
|
2902
|
+
* Features:
|
|
2903
|
+
* - Automatic reconnection on disconnect
|
|
2904
|
+
* - Event replay via Last-Event-ID
|
|
2905
|
+
* - Keepalive pings every 15 seconds
|
|
2906
|
+
* - Max connection time: 1 hour
|
|
2907
|
+
*
|
|
2908
|
+
* @param token - SSE authentication token from client.auth.generateSSEToken()
|
|
2909
|
+
* @param options - Subscription options (callbacks, reconnection settings)
|
|
2910
|
+
*
|
|
2911
|
+
* @example
|
|
2912
|
+
* ```ts
|
|
2913
|
+
* // Generate token
|
|
2914
|
+
* const { token } = await client.auth.generateSSEToken();
|
|
2915
|
+
*
|
|
2916
|
+
* // Subscribe with event handlers
|
|
2917
|
+
* const eventSource = client.events.subscribe(token, {
|
|
2918
|
+
* onEvent: (event) => {
|
|
2919
|
+
* // Handle all events
|
|
2920
|
+
* console.log('Event:', event.type);
|
|
2921
|
+
* const data = JSON.parse(event.data);
|
|
2922
|
+
*
|
|
2923
|
+
* if (data.jobId) {
|
|
2924
|
+
* console.log(`Job ${data.jobId}:`, data);
|
|
2925
|
+
* }
|
|
2926
|
+
* },
|
|
2927
|
+
* onError: (error) => {
|
|
2928
|
+
* console.error('SSE error:', error);
|
|
2929
|
+
* },
|
|
2930
|
+
* onOpen: () => {
|
|
2931
|
+
* console.log('SSE connection opened');
|
|
2932
|
+
* }
|
|
2933
|
+
* });
|
|
2934
|
+
*
|
|
2935
|
+
* // Listen for specific event types
|
|
2936
|
+
* eventSource.addEventListener('job.completed', (event) => {
|
|
2937
|
+
* const data = JSON.parse(event.data);
|
|
2938
|
+
* console.log('Job completed:', data);
|
|
2939
|
+
* });
|
|
2940
|
+
*
|
|
2941
|
+
* // Clean up
|
|
2942
|
+
* eventSource.close();
|
|
2943
|
+
* ```
|
|
2944
|
+
*/
|
|
2945
|
+
subscribe(token: string, options?: Omit<SSESubscribeOptions, "lastEventId">): EventSource;
|
|
2946
|
+
/**
|
|
2947
|
+
* Subscribe to events for a specific job
|
|
2948
|
+
*
|
|
2949
|
+
* Opens an SSE connection filtered to a single job.
|
|
2950
|
+
* More efficient than global subscription when tracking one job.
|
|
2951
|
+
*
|
|
2952
|
+
* @param jobId - Job ID to subscribe to
|
|
2953
|
+
* @param token - SSE authentication token
|
|
2954
|
+
* @param options - Subscription options
|
|
2955
|
+
*
|
|
2956
|
+
* @example
|
|
2957
|
+
* ```ts
|
|
2958
|
+
* // Start a scrape job
|
|
2959
|
+
* const job = await client.scrape.create({ url: "https://example.com" });
|
|
2960
|
+
*
|
|
2961
|
+
* // Generate SSE token for this job
|
|
2962
|
+
* const { token } = await client.auth.generateSSEToken({ jobId: job.jobId });
|
|
2963
|
+
*
|
|
2964
|
+
* // Subscribe to job events
|
|
2965
|
+
* const eventSource = client.events.subscribeToJob(job.jobId, token, {
|
|
2966
|
+
* onEvent: (event) => {
|
|
2967
|
+
* const data = JSON.parse(event.data);
|
|
2968
|
+
* console.log(`[${event.type}]`, data);
|
|
2969
|
+
* }
|
|
2970
|
+
* });
|
|
2971
|
+
*
|
|
2972
|
+
* // Listen for completion
|
|
2973
|
+
* eventSource.addEventListener('job.completed', (event) => {
|
|
2974
|
+
* const data = JSON.parse(event.data);
|
|
2975
|
+
* console.log('Scrape completed!', data.summary);
|
|
2976
|
+
* eventSource.close();
|
|
2977
|
+
* });
|
|
2978
|
+
*
|
|
2979
|
+
* // Listen for progress
|
|
2980
|
+
* eventSource.addEventListener('job.progress', (event) => {
|
|
2981
|
+
* const data = JSON.parse(event.data);
|
|
2982
|
+
* console.log(`Progress: ${data.progress}%`);
|
|
2983
|
+
* });
|
|
2984
|
+
*
|
|
2985
|
+
* // Listen for errors
|
|
2986
|
+
* eventSource.addEventListener('job.failed', (event) => {
|
|
2987
|
+
* const data = JSON.parse(event.data);
|
|
2988
|
+
* console.error('Job failed:', data.error);
|
|
2989
|
+
* eventSource.close();
|
|
2990
|
+
* });
|
|
2991
|
+
* ```
|
|
2992
|
+
*/
|
|
2993
|
+
subscribeToJob(jobId: string, token: string, options?: Omit<SSESubscribeOptions, "lastEventId">): EventSource;
|
|
2994
|
+
/**
|
|
2995
|
+
* Helper: Wait for job completion via SSE
|
|
2996
|
+
*
|
|
2997
|
+
* Convenience method that subscribes to a job and resolves when complete.
|
|
2998
|
+
* Automatically handles token generation and cleanup.
|
|
2999
|
+
*
|
|
3000
|
+
* @param jobId - Job ID to wait for
|
|
3001
|
+
* @param onProgress - Optional progress callback
|
|
3002
|
+
*
|
|
3003
|
+
* @example
|
|
3004
|
+
* ```ts
|
|
3005
|
+
* const job = await client.scrape.create({ url: "https://example.com" });
|
|
3006
|
+
*
|
|
3007
|
+
* // Wait for completion with progress updates
|
|
3008
|
+
* const result = await client.events.waitForCompletion(job.jobId, (progress) => {
|
|
3009
|
+
* console.log(`Progress: ${progress}%`);
|
|
3010
|
+
* });
|
|
3011
|
+
*
|
|
3012
|
+
* console.log('Job completed:', result);
|
|
3013
|
+
* ```
|
|
3014
|
+
*/
|
|
3015
|
+
waitForCompletion<T = unknown>(jobId: string, onProgress?: (progress: number) => void): Promise<T>;
|
|
3016
|
+
}
|
|
3017
|
+
|
|
2388
3018
|
/**
|
|
2389
3019
|
* Extract Resource
|
|
2390
3020
|
* Handles LLM-based structured data extraction
|
|
@@ -2594,7 +3224,9 @@ declare class KeysResource {
|
|
|
2594
3224
|
*/
|
|
2595
3225
|
revokeAll(): Promise<{
|
|
2596
3226
|
success: boolean;
|
|
2597
|
-
|
|
3227
|
+
revokedCount: number;
|
|
3228
|
+
message: string;
|
|
3229
|
+
warning: string;
|
|
2598
3230
|
}>;
|
|
2599
3231
|
/**
|
|
2600
3232
|
* Get active keys only
|
|
@@ -2689,7 +3321,8 @@ declare class ScrapeResource {
|
|
|
2689
3321
|
* { url: "https://shop1.com/product1" },
|
|
2690
3322
|
* { url: "https://shop2.com/deal", extractDeal: true }
|
|
2691
3323
|
* ],
|
|
2692
|
-
* defaults: { detectSignals: true }
|
|
3324
|
+
* defaults: { detectSignals: true },
|
|
3325
|
+
* ignoreInvalidURLs: true
|
|
2693
3326
|
* });
|
|
2694
3327
|
* console.log(batch.batchId, batch.results);
|
|
2695
3328
|
* ```
|
|
@@ -2720,6 +3353,58 @@ declare class ScrapeResource {
|
|
|
2720
3353
|
batchForDeals(urls: string[], options?: Omit<BatchScrapeOptions, "urls">): Promise<BatchScrapeResponse>;
|
|
2721
3354
|
}
|
|
2722
3355
|
|
|
3356
|
+
/**
|
|
3357
|
+
* Screenshots Resource
|
|
3358
|
+
* Handles screenshot signed URL refresh and TTL limits
|
|
3359
|
+
*/
|
|
3360
|
+
|
|
3361
|
+
/**
|
|
3362
|
+
* Options for refreshing a screenshot signed URL
|
|
3363
|
+
*/
|
|
3364
|
+
interface RefreshScreenshotOptions {
|
|
3365
|
+
/** File path in storage (e.g., "job_abc123/timestamp_id_url.png") */
|
|
3366
|
+
path: string;
|
|
3367
|
+
/** Optional: New TTL in seconds (must be within tier limits) */
|
|
3368
|
+
ttl?: number;
|
|
3369
|
+
/** Optional: Bucket name (defaults to 'screenshots-private') */
|
|
3370
|
+
bucket?: string;
|
|
3371
|
+
}
|
|
3372
|
+
/**
|
|
3373
|
+
* Screenshots resource class
|
|
3374
|
+
* Provides methods for managing screenshot signed URLs
|
|
3375
|
+
*/
|
|
3376
|
+
declare class ScreenshotsResource {
|
|
3377
|
+
private ctx;
|
|
3378
|
+
constructor(ctx: RequestContext);
|
|
3379
|
+
/**
|
|
3380
|
+
* Refresh a signed URL before expiration
|
|
3381
|
+
*
|
|
3382
|
+
* @example
|
|
3383
|
+
* ```ts
|
|
3384
|
+
* const refreshed = await client.screenshots.refresh({
|
|
3385
|
+
* path: "job_abc123/1234567890_nanoid_example.png",
|
|
3386
|
+
* ttl: 604800 // 7 days
|
|
3387
|
+
* });
|
|
3388
|
+
* console.log(refreshed.url); // New signed URL
|
|
3389
|
+
* console.log(refreshed.expiresAt); // "2026-01-25T12:00:00Z"
|
|
3390
|
+
* console.log(refreshed.tierLimits); // { min: 3600, max: 604800, default: 604800 }
|
|
3391
|
+
* ```
|
|
3392
|
+
*/
|
|
3393
|
+
refresh(options: RefreshScreenshotOptions): Promise<ScreenshotRefreshResponse>;
|
|
3394
|
+
/**
|
|
3395
|
+
* Get TTL limits for the current tier
|
|
3396
|
+
*
|
|
3397
|
+
* @example
|
|
3398
|
+
* ```ts
|
|
3399
|
+
* const limits = await client.screenshots.getLimits();
|
|
3400
|
+
* console.log(limits.tier); // "pro"
|
|
3401
|
+
* console.log(limits.limits.max); // 604800 (7 days in seconds)
|
|
3402
|
+
* console.log(limits.formattedLimits.max); // "7 days"
|
|
3403
|
+
* ```
|
|
3404
|
+
*/
|
|
3405
|
+
getLimits(): Promise<ScreenshotLimitsResponse>;
|
|
3406
|
+
}
|
|
3407
|
+
|
|
2723
3408
|
/**
|
|
2724
3409
|
* Search Resource
|
|
2725
3410
|
* Handles web search operations with AI optimization
|
|
@@ -2739,7 +3424,7 @@ declare class SearchResource {
|
|
|
2739
3424
|
* ```ts
|
|
2740
3425
|
* const result = await client.search.create({
|
|
2741
3426
|
* query: "laptop deals black friday",
|
|
2742
|
-
*
|
|
3427
|
+
* limit: 20,
|
|
2743
3428
|
* useDealScoring: true
|
|
2744
3429
|
* });
|
|
2745
3430
|
* ```
|
|
@@ -2775,12 +3460,12 @@ declare class SearchResource {
|
|
|
2775
3460
|
* @example
|
|
2776
3461
|
* ```ts
|
|
2777
3462
|
* const result = await client.search.andScrape("promo codes", {
|
|
2778
|
-
*
|
|
3463
|
+
* maxScrapeResults: 5
|
|
2779
3464
|
* });
|
|
2780
3465
|
* console.log(result.data.scrapedJobIds);
|
|
2781
3466
|
* ```
|
|
2782
3467
|
*/
|
|
2783
|
-
andScrape(query: string, options?: Omit<SearchOptions, "query" | "
|
|
3468
|
+
andScrape(query: string, options?: Omit<SearchOptions, "query" | "scrapeResults">): Promise<SearchJobResponse>;
|
|
2784
3469
|
/**
|
|
2785
3470
|
* Check search API status
|
|
2786
3471
|
* Returns availability and features info
|
|
@@ -3056,6 +3741,9 @@ declare class WebhooksResource {
|
|
|
3056
3741
|
* await client.extract.create({ url: "...", schema: {...} });
|
|
3057
3742
|
* await client.dork.create({ query: "..." });
|
|
3058
3743
|
*
|
|
3744
|
+
* // Utilities
|
|
3745
|
+
* await client.convert.htmlToMarkdown({ html: "..." });
|
|
3746
|
+
*
|
|
3059
3747
|
* // Status & polling
|
|
3060
3748
|
* await client.status.get(jobId);
|
|
3061
3749
|
* await client.waitForResult(jobId);
|
|
@@ -3147,6 +3835,20 @@ declare class DealCrawl {
|
|
|
3147
3835
|
* ```
|
|
3148
3836
|
*/
|
|
3149
3837
|
readonly dork: DorkResource;
|
|
3838
|
+
/**
|
|
3839
|
+
* Convert resource - HTML to Markdown conversion
|
|
3840
|
+
*
|
|
3841
|
+
* @example
|
|
3842
|
+
* ```ts
|
|
3843
|
+
* const result = await client.convert.htmlToMarkdown({
|
|
3844
|
+
* html: "<h1>Title</h1><p>Content</p>",
|
|
3845
|
+
* baseUrl: "https://example.com",
|
|
3846
|
+
* options: { removeNoise: true }
|
|
3847
|
+
* });
|
|
3848
|
+
* console.log(result.data.markdown);
|
|
3849
|
+
* ```
|
|
3850
|
+
*/
|
|
3851
|
+
readonly convert: ConvertResource;
|
|
3150
3852
|
/**
|
|
3151
3853
|
* Agent resource - AI-powered autonomous web navigation
|
|
3152
3854
|
*
|
|
@@ -3228,6 +3930,80 @@ declare class DealCrawl {
|
|
|
3228
3930
|
* ```
|
|
3229
3931
|
*/
|
|
3230
3932
|
readonly account: AccountResource;
|
|
3933
|
+
/**
|
|
3934
|
+
* Screenshots resource - Screenshot signed URL management
|
|
3935
|
+
*
|
|
3936
|
+
* @example
|
|
3937
|
+
* ```ts
|
|
3938
|
+
* // Refresh a signed URL before expiration
|
|
3939
|
+
* const refreshed = await client.screenshots.refresh({
|
|
3940
|
+
* path: "job_abc123/1234567890_nanoid_example.png",
|
|
3941
|
+
* ttl: 604800 // 7 days
|
|
3942
|
+
* });
|
|
3943
|
+
*
|
|
3944
|
+
* // Get tier-specific TTL limits
|
|
3945
|
+
* const limits = await client.screenshots.getLimits();
|
|
3946
|
+
* console.log(limits.formattedLimits.max); // "7 days"
|
|
3947
|
+
* ```
|
|
3948
|
+
*/
|
|
3949
|
+
readonly screenshots: ScreenshotsResource;
|
|
3950
|
+
/**
|
|
3951
|
+
* Auth resource - SSE (Server-Sent Events) authentication
|
|
3952
|
+
*
|
|
3953
|
+
* @example
|
|
3954
|
+
* ```ts
|
|
3955
|
+
* // Generate SSE token for browser EventSource
|
|
3956
|
+
* const { token, expiresAt } = await client.auth.generateSSEToken();
|
|
3957
|
+
*
|
|
3958
|
+
* // Use in browser
|
|
3959
|
+
* const eventSource = new EventSource(`/v1/events?token=${token}`);
|
|
3960
|
+
*
|
|
3961
|
+
* // Generate token for specific job
|
|
3962
|
+
* const jobToken = await client.auth.generateSSEToken({ jobId: "job_123" });
|
|
3963
|
+
*
|
|
3964
|
+
* // Check connection limits
|
|
3965
|
+
* const limits = await client.auth.getLimits();
|
|
3966
|
+
* console.log(`Available connections: ${limits.sse.available}`);
|
|
3967
|
+
* ```
|
|
3968
|
+
*/
|
|
3969
|
+
readonly auth: AuthResource;
|
|
3970
|
+
/**
|
|
3971
|
+
* Events resource - Real-time SSE event streaming (Browser only)
|
|
3972
|
+
*
|
|
3973
|
+
* IMPORTANT: This resource only works in browsers. For Node.js, use polling via client.status.get()
|
|
3974
|
+
*
|
|
3975
|
+
* @example Browser Usage
|
|
3976
|
+
* ```ts
|
|
3977
|
+
* // 1. Generate SSE token
|
|
3978
|
+
* const { token } = await client.auth.generateSSEToken();
|
|
3979
|
+
*
|
|
3980
|
+
* // 2. Subscribe to all events
|
|
3981
|
+
* const eventSource = client.events.subscribe(token, {
|
|
3982
|
+
* onEvent: (event) => {
|
|
3983
|
+
* console.log('Event:', event.type, JSON.parse(event.data));
|
|
3984
|
+
* }
|
|
3985
|
+
* });
|
|
3986
|
+
*
|
|
3987
|
+
* // 3. Or subscribe to specific job
|
|
3988
|
+
* const jobEvents = client.events.subscribeToJob('job_123', token, {
|
|
3989
|
+
* onEvent: (event) => {
|
|
3990
|
+
* const data = JSON.parse(event.data);
|
|
3991
|
+
* console.log(`Progress: ${data.progress}%`);
|
|
3992
|
+
* }
|
|
3993
|
+
* });
|
|
3994
|
+
*
|
|
3995
|
+
* // 4. Listen for specific events
|
|
3996
|
+
* eventSource.addEventListener('job.completed', (event) => {
|
|
3997
|
+
* const data = JSON.parse(event.data);
|
|
3998
|
+
* console.log('Job completed!', data.summary);
|
|
3999
|
+
* eventSource.close();
|
|
4000
|
+
* });
|
|
4001
|
+
*
|
|
4002
|
+
* // 5. Clean up
|
|
4003
|
+
* eventSource.close();
|
|
4004
|
+
* ```
|
|
4005
|
+
*/
|
|
4006
|
+
readonly events: EventsResource;
|
|
3231
4007
|
/**
|
|
3232
4008
|
* Create a new DealCrawl client
|
|
3233
4009
|
*
|
|
@@ -3635,4 +4411,352 @@ declare const ERROR_MESSAGES: Record<ErrorCode, string>;
|
|
|
3635
4411
|
*/
|
|
3636
4412
|
declare function getErrorMessage(code: ErrorCode): string;
|
|
3637
4413
|
|
|
3638
|
-
|
|
4414
|
+
/**
|
|
4415
|
+
* Advanced Job Status System
|
|
4416
|
+
*
|
|
4417
|
+
* Production-grade status model with:
|
|
4418
|
+
* - Primary status (lifecycle stage)
|
|
4419
|
+
* - Sub-status (detailed state within stage)
|
|
4420
|
+
* - Reason codes (why this state was reached)
|
|
4421
|
+
* - State machine transitions (valid state changes)
|
|
4422
|
+
*
|
|
4423
|
+
* @module job-status
|
|
4424
|
+
*/
|
|
4425
|
+
/**
|
|
4426
|
+
* Primary job status - represents the main lifecycle stage
|
|
4427
|
+
*/
|
|
4428
|
+
type JobStatus = "queued" | "scheduled" | "priority_queued" | "initializing" | "active" | "processing" | "extracting" | "uploading" | "finalizing" | "retrying" | "backing_off" | "paused" | "pausing" | "resuming" | "completed" | "partial" | "failed" | "timeout" | "cancelled" | "rejected" | "stale" | "zombie" | "stuck";
|
|
4429
|
+
/**
|
|
4430
|
+
* Sub-status provides granular detail within a primary status
|
|
4431
|
+
*/
|
|
4432
|
+
type JobSubStatus = "waiting_for_worker" | "waiting_for_rate_limit" | "waiting_for_quota" | "waiting_for_retry" | "waiting_for_dependency" | "launching_browser" | "establishing_connection" | "loading_config" | "validating_input" | "fetching_page" | "executing_javascript" | "waiting_for_render" | "detecting_signals" | "crawling_links" | "capturing_screenshot" | "calling_llm" | "parsing_response" | "validating_extraction" | "uploading_screenshot" | "uploading_artifacts" | "generating_signed_url" | "saving_to_database" | "cleaning_resources" | "sending_webhook" | "syncing_to_dealup" | "exponential_backoff" | "rate_limit_backoff" | "circuit_breaker_open" | "network_error" | "timeout_error" | "bot_detected" | "captcha_detected" | "auth_required" | "page_not_found" | "server_error" | "parse_error" | "extraction_failed" | "quota_exceeded" | "invalid_input" | "resource_unavailable" | "no_heartbeat" | "progress_stalled" | "worker_lost";
|
|
4433
|
+
/**
|
|
4434
|
+
* Reason codes - why a status/substatus was reached
|
|
4435
|
+
*/
|
|
4436
|
+
type StatusReason = "job_completed_successfully" | "partial_results_available" | "added_to_queue" | "priority_elevated" | "rate_limit_applied" | "quota_check_pending" | "transient_network_error" | "rate_limit_exceeded" | "timeout_exceeded" | "worker_crashed" | "max_retries_not_reached" | "max_retries_exceeded" | "permanent_error" | "blocked_by_site" | "captcha_required" | "invalid_url_format" | "ssrf_detected" | "quota_exhausted" | "tier_restriction" | "resource_not_available" | "cancelled_by_user" | "cancelled_by_system" | "duplicate_detected" | "policy_violation" | "heartbeat_timeout" | "progress_timeout" | "worker_unresponsive";
|
|
4437
|
+
|
|
4438
|
+
/**
|
|
4439
|
+
* Professional structured logger for DealCrawl
|
|
4440
|
+
* - Environment-aware (verbose in dev, minimal in prod)
|
|
4441
|
+
* - Structured JSON output in production
|
|
4442
|
+
* - Colored console output in development
|
|
4443
|
+
* - Correlation ID support for request tracing
|
|
4444
|
+
*/
|
|
4445
|
+
type LogLevel = "debug" | "info" | "warn" | "error";
|
|
4446
|
+
|
|
4447
|
+
/**
|
|
4448
|
+
* SSE (Server-Sent Events) Event Types
|
|
4449
|
+
*
|
|
4450
|
+
* Defines all events that can be pushed to clients via SSE.
|
|
4451
|
+
*
|
|
4452
|
+
* @module sse-events
|
|
4453
|
+
*/
|
|
4454
|
+
|
|
4455
|
+
/**
|
|
4456
|
+
* SSE event types - used as the "event:" field in SSE messages
|
|
4457
|
+
*/
|
|
4458
|
+
type SSEEventType = "job.created" | "job.queued" | "job.started" | "job.status" | "job.progress" | "job.completed" | "job.failed" | "job.cancelled" | "job.paused" | "job.resumed" | "job.log" | "job.metric" | "job.alert" | "job.checkpoint" | "deal.found" | "deal.validated" | "ping" | "connection.open" | "connection.close" | "error";
|
|
4459
|
+
/**
|
|
4460
|
+
* Base SSE event structure - all events extend this
|
|
4461
|
+
*/
|
|
4462
|
+
interface BaseSSEEvent {
|
|
4463
|
+
/** Event type (used in SSE "event:" field) */
|
|
4464
|
+
type: SSEEventType;
|
|
4465
|
+
/** Unique event ID (for Last-Event-ID replay) */
|
|
4466
|
+
id: string;
|
|
4467
|
+
/** When this event was generated */
|
|
4468
|
+
timestamp: string;
|
|
4469
|
+
/** Job ID this event relates to (null for system events) */
|
|
4470
|
+
jobId: string | null;
|
|
4471
|
+
/** Client ID (for server-side filtering) */
|
|
4472
|
+
clientId: string;
|
|
4473
|
+
}
|
|
4474
|
+
/**
|
|
4475
|
+
* Job created event - sent when job is first created
|
|
4476
|
+
*/
|
|
4477
|
+
interface JobCreatedEvent extends BaseSSEEvent {
|
|
4478
|
+
type: "job.created";
|
|
4479
|
+
jobId: string;
|
|
4480
|
+
data: {
|
|
4481
|
+
jobType: "scrape" | "crawl" | "dork" | "extract" | "agent";
|
|
4482
|
+
priority?: "high" | "medium" | "low";
|
|
4483
|
+
template?: string;
|
|
4484
|
+
};
|
|
4485
|
+
}
|
|
4486
|
+
/**
|
|
4487
|
+
* Job queued event - sent when job enters queue
|
|
4488
|
+
*/
|
|
4489
|
+
interface JobQueuedEvent extends BaseSSEEvent {
|
|
4490
|
+
type: "job.queued";
|
|
4491
|
+
jobId: string;
|
|
4492
|
+
data: {
|
|
4493
|
+
queue: string;
|
|
4494
|
+
position?: number;
|
|
4495
|
+
estimatedWaitMs?: number;
|
|
4496
|
+
};
|
|
4497
|
+
}
|
|
4498
|
+
/**
|
|
4499
|
+
* Job started event - sent when worker picks up job
|
|
4500
|
+
*/
|
|
4501
|
+
interface JobStartedEvent extends BaseSSEEvent {
|
|
4502
|
+
type: "job.started";
|
|
4503
|
+
jobId: string;
|
|
4504
|
+
data: {
|
|
4505
|
+
workerId?: string;
|
|
4506
|
+
startedAt: string;
|
|
4507
|
+
};
|
|
4508
|
+
}
|
|
4509
|
+
/**
|
|
4510
|
+
* Job status change event - sent on every status transition
|
|
4511
|
+
*/
|
|
4512
|
+
interface JobStatusEvent extends BaseSSEEvent {
|
|
4513
|
+
type: "job.status";
|
|
4514
|
+
jobId: string;
|
|
4515
|
+
data: {
|
|
4516
|
+
status: JobStatus;
|
|
4517
|
+
subStatus?: JobSubStatus;
|
|
4518
|
+
reason?: StatusReason;
|
|
4519
|
+
message?: string;
|
|
4520
|
+
previousStatus?: JobStatus;
|
|
4521
|
+
progress: number;
|
|
4522
|
+
};
|
|
4523
|
+
}
|
|
4524
|
+
/**
|
|
4525
|
+
* Job progress event - sent periodically during processing
|
|
4526
|
+
*/
|
|
4527
|
+
interface JobProgressEvent extends BaseSSEEvent {
|
|
4528
|
+
type: "job.progress";
|
|
4529
|
+
jobId: string;
|
|
4530
|
+
data: {
|
|
4531
|
+
progress: number;
|
|
4532
|
+
status: JobStatus;
|
|
4533
|
+
subStatus?: JobSubStatus;
|
|
4534
|
+
message?: string;
|
|
4535
|
+
eta?: {
|
|
4536
|
+
remainingMs: number;
|
|
4537
|
+
remainingFormatted: string;
|
|
4538
|
+
};
|
|
4539
|
+
stats?: {
|
|
4540
|
+
pagesProcessed?: number;
|
|
4541
|
+
dealsFound?: number;
|
|
4542
|
+
signalsDetected?: number;
|
|
4543
|
+
};
|
|
4544
|
+
};
|
|
4545
|
+
}
|
|
4546
|
+
/**
|
|
4547
|
+
* Job completed event - sent when job finishes successfully
|
|
4548
|
+
*/
|
|
4549
|
+
interface JobCompletedEvent extends BaseSSEEvent {
|
|
4550
|
+
type: "job.completed";
|
|
4551
|
+
jobId: string;
|
|
4552
|
+
data: {
|
|
4553
|
+
statusUrl: string;
|
|
4554
|
+
completedAt: string;
|
|
4555
|
+
durationMs: number;
|
|
4556
|
+
summary: {
|
|
4557
|
+
success: boolean;
|
|
4558
|
+
pagesProcessed?: number;
|
|
4559
|
+
dealsFound?: number;
|
|
4560
|
+
signalsDetected?: number;
|
|
4561
|
+
hasScreenshot?: boolean;
|
|
4562
|
+
};
|
|
4563
|
+
result?: {
|
|
4564
|
+
dealScore?: number;
|
|
4565
|
+
topSignals?: string[];
|
|
4566
|
+
dealTitle?: string;
|
|
4567
|
+
};
|
|
4568
|
+
};
|
|
4569
|
+
}
|
|
4570
|
+
/**
|
|
4571
|
+
* Job failed event - sent when job fails
|
|
4572
|
+
*/
|
|
4573
|
+
interface JobFailedEvent extends BaseSSEEvent {
|
|
4574
|
+
type: "job.failed";
|
|
4575
|
+
jobId: string;
|
|
4576
|
+
data: {
|
|
4577
|
+
statusUrl: string;
|
|
4578
|
+
failedAt: string;
|
|
4579
|
+
durationMs: number;
|
|
4580
|
+
error: {
|
|
4581
|
+
code: string;
|
|
4582
|
+
message: string;
|
|
4583
|
+
recoverable: boolean;
|
|
4584
|
+
};
|
|
4585
|
+
retry?: {
|
|
4586
|
+
attempt: number;
|
|
4587
|
+
maxAttempts: number;
|
|
4588
|
+
willRetry: boolean;
|
|
4589
|
+
nextRetryAt?: string;
|
|
4590
|
+
};
|
|
4591
|
+
};
|
|
4592
|
+
}
|
|
4593
|
+
/**
|
|
4594
|
+
* Job cancelled event - sent when job is cancelled
|
|
4595
|
+
*/
|
|
4596
|
+
interface JobCancelledEvent extends BaseSSEEvent {
|
|
4597
|
+
type: "job.cancelled";
|
|
4598
|
+
jobId: string;
|
|
4599
|
+
data: {
|
|
4600
|
+
cancelledAt: string;
|
|
4601
|
+
cancelledBy: "user" | "system";
|
|
4602
|
+
reason?: string;
|
|
4603
|
+
};
|
|
4604
|
+
}
|
|
4605
|
+
/**
|
|
4606
|
+
* Job paused event
|
|
4607
|
+
*/
|
|
4608
|
+
interface JobPausedEvent extends BaseSSEEvent {
|
|
4609
|
+
type: "job.paused";
|
|
4610
|
+
jobId: string;
|
|
4611
|
+
data: {
|
|
4612
|
+
pausedAt: string;
|
|
4613
|
+
checkpoint?: {
|
|
4614
|
+
lastProcessedUrl: string;
|
|
4615
|
+
urlsProcessed: number;
|
|
4616
|
+
urlsRemaining: number;
|
|
4617
|
+
};
|
|
4618
|
+
};
|
|
4619
|
+
}
|
|
4620
|
+
/**
|
|
4621
|
+
* Job resumed event
|
|
4622
|
+
*/
|
|
4623
|
+
interface JobResumedEvent extends BaseSSEEvent {
|
|
4624
|
+
type: "job.resumed";
|
|
4625
|
+
jobId: string;
|
|
4626
|
+
data: {
|
|
4627
|
+
resumedAt: string;
|
|
4628
|
+
fromCheckpoint: boolean;
|
|
4629
|
+
};
|
|
4630
|
+
}
|
|
4631
|
+
/**
|
|
4632
|
+
* Job log event - sent for important log messages
|
|
4633
|
+
*/
|
|
4634
|
+
interface JobLogEvent extends BaseSSEEvent {
|
|
4635
|
+
type: "job.log";
|
|
4636
|
+
jobId: string;
|
|
4637
|
+
data: {
|
|
4638
|
+
level: LogLevel;
|
|
4639
|
+
message: string;
|
|
4640
|
+
metadata?: Record<string, unknown>;
|
|
4641
|
+
};
|
|
4642
|
+
}
|
|
4643
|
+
/**
|
|
4644
|
+
* Job metric event - sent for performance/business metrics
|
|
4645
|
+
*/
|
|
4646
|
+
interface JobMetricEvent extends BaseSSEEvent {
|
|
4647
|
+
type: "job.metric";
|
|
4648
|
+
jobId: string;
|
|
4649
|
+
data: {
|
|
4650
|
+
metric: string;
|
|
4651
|
+
value: number;
|
|
4652
|
+
unit?: string;
|
|
4653
|
+
};
|
|
4654
|
+
}
|
|
4655
|
+
/**
|
|
4656
|
+
* Job alert event - sent for important alerts (quota warnings, etc.)
|
|
4657
|
+
*/
|
|
4658
|
+
interface JobAlertEvent extends BaseSSEEvent {
|
|
4659
|
+
type: "job.alert";
|
|
4660
|
+
jobId: string;
|
|
4661
|
+
data: {
|
|
4662
|
+
severity: "info" | "warning" | "error" | "critical";
|
|
4663
|
+
title: string;
|
|
4664
|
+
message: string;
|
|
4665
|
+
actionRequired?: boolean;
|
|
4666
|
+
};
|
|
4667
|
+
}
|
|
4668
|
+
/**
|
|
4669
|
+
* Job checkpoint event - sent when checkpoint is saved
|
|
4670
|
+
*/
|
|
4671
|
+
interface JobCheckpointEvent extends BaseSSEEvent {
|
|
4672
|
+
type: "job.checkpoint";
|
|
4673
|
+
jobId: string;
|
|
4674
|
+
data: {
|
|
4675
|
+
savedAt: string;
|
|
4676
|
+
lastProcessedUrl: string;
|
|
4677
|
+
urlsProcessed: number;
|
|
4678
|
+
urlsRemaining: number;
|
|
4679
|
+
canResume: boolean;
|
|
4680
|
+
};
|
|
4681
|
+
}
|
|
4682
|
+
/**
|
|
4683
|
+
* Deal found event - sent when a deal is detected
|
|
4684
|
+
*/
|
|
4685
|
+
interface DealFoundEvent extends BaseSSEEvent {
|
|
4686
|
+
type: "deal.found";
|
|
4687
|
+
jobId: string;
|
|
4688
|
+
data: {
|
|
4689
|
+
dealId?: string;
|
|
4690
|
+
url: string;
|
|
4691
|
+
title: string;
|
|
4692
|
+
score: number;
|
|
4693
|
+
price?: number;
|
|
4694
|
+
discount?: number;
|
|
4695
|
+
category?: string;
|
|
4696
|
+
signals: string[];
|
|
4697
|
+
};
|
|
4698
|
+
}
|
|
4699
|
+
/**
|
|
4700
|
+
* Deal validated event - sent when deal is validated/scored
|
|
4701
|
+
*/
|
|
4702
|
+
interface DealValidatedEvent extends BaseSSEEvent {
|
|
4703
|
+
type: "deal.validated";
|
|
4704
|
+
jobId: string;
|
|
4705
|
+
data: {
|
|
4706
|
+
dealId: string;
|
|
4707
|
+
score: number;
|
|
4708
|
+
confidence: number;
|
|
4709
|
+
accepted: boolean;
|
|
4710
|
+
rejectionReason?: string;
|
|
4711
|
+
};
|
|
4712
|
+
}
|
|
4713
|
+
/**
|
|
4714
|
+
* Ping event - keepalive to prevent connection timeout
|
|
4715
|
+
*/
|
|
4716
|
+
interface PingEvent extends BaseSSEEvent {
|
|
4717
|
+
type: "ping";
|
|
4718
|
+
jobId: null;
|
|
4719
|
+
data: {
|
|
4720
|
+
serverTime: string;
|
|
4721
|
+
};
|
|
4722
|
+
}
|
|
4723
|
+
/**
|
|
4724
|
+
* Connection open event - sent when SSE connection established
|
|
4725
|
+
*/
|
|
4726
|
+
interface ConnectionOpenEvent extends BaseSSEEvent {
|
|
4727
|
+
type: "connection.open";
|
|
4728
|
+
jobId: null;
|
|
4729
|
+
data: {
|
|
4730
|
+
clientId: string;
|
|
4731
|
+
reconnected: boolean;
|
|
4732
|
+
lastEventId?: string;
|
|
4733
|
+
};
|
|
4734
|
+
}
|
|
4735
|
+
/**
|
|
4736
|
+
* Connection close event - sent before connection closes
|
|
4737
|
+
*/
|
|
4738
|
+
interface ConnectionCloseEvent extends BaseSSEEvent {
|
|
4739
|
+
type: "connection.close";
|
|
4740
|
+
jobId: null;
|
|
4741
|
+
data: {
|
|
4742
|
+
reason: string;
|
|
4743
|
+
};
|
|
4744
|
+
}
|
|
4745
|
+
/**
|
|
4746
|
+
* Error event - sent when an error occurs
|
|
4747
|
+
*/
|
|
4748
|
+
interface ErrorEvent extends BaseSSEEvent {
|
|
4749
|
+
type: "error";
|
|
4750
|
+
jobId: string | null;
|
|
4751
|
+
data: {
|
|
4752
|
+
code: string;
|
|
4753
|
+
message: string;
|
|
4754
|
+
recoverable: boolean;
|
|
4755
|
+
};
|
|
4756
|
+
}
|
|
4757
|
+
/**
|
|
4758
|
+
* All possible SSE events (discriminated union)
|
|
4759
|
+
*/
|
|
4760
|
+
type SSEEvent = JobCreatedEvent | JobQueuedEvent | JobStartedEvent | JobStatusEvent | JobProgressEvent | JobCompletedEvent | JobFailedEvent | JobCancelledEvent | JobPausedEvent | JobResumedEvent | JobLogEvent | JobMetricEvent | JobAlertEvent | JobCheckpointEvent | DealFoundEvent | DealValidatedEvent | PingEvent | ConnectionOpenEvent | ConnectionCloseEvent | ErrorEvent;
|
|
4761
|
+
|
|
4762
|
+
export { ALL_API_KEY_SCOPES, type AccountInfoResponse, type AccountMetricsResponse, AccountResource, type ActionBaseOptions, type ActionInput, type AgentAction, type AgentActionType, type AgentCompletionReason, type AgentJobResponse, type AgentModel, type AgentOptions, AgentResource, type AgentResultResponse, type AgentStatusResponse, type AgentStepResponse, type ApiError, type ApiKeyInfo, type ApiKeyScope, type ApiResponse, AuthResource, type BaseSSEEvent, type BatchScrapeDefaults, type BatchScrapeItem, type BatchScrapeOptions, type BatchScrapeResponse, type BatchScrapeResultItem, type BatchStatusResponse, type CancelJobResponse, type CheckpointInfo, type ClickAction, type ClientPreferences, type ClientStatsResponse, type ConnectionCloseEvent, type ConnectionOpenEvent, type ConversionMetadata, type ConvertOptions, ConvertResource, type ConvertResponse, type CrawlAnalysisResponse, type CrawlError, type CrawlJobResponse, type CrawlMode, type CrawlOptions, type CrawlRecommendation, CrawlResource, type CrawlResult, type CrawlStats, type CrawlTemplate, type CrawlTemplateId, type CreateApiKeyOptions, type CreateKeyResponse, type CreateWebhookOptions, type CreateWebhookResponse, type CreatedApiKey, DEFAULT_API_KEY_SCOPES, DEFAULT_CONFIG, DataResource, DealCrawl, type DealCrawlConfig, DealCrawlError, type DealDetails, type DealFoundEvent, type DealItem, type DealMetrics, type DealScoreSummary, type DealSummary, type DealUpMetrics, type DealUpMetricsResponse, type DealValidatedEvent, type DeleteKeyResponse, type DeleteWebhookResponse, type DiscountSignal, type DorkJobResponse, type DorkOptions, DorkResource, type DorkResult, ERROR_CODES, ERROR_MESSAGES, type EngineSelection, type EngineType, type ErrorCode, type ErrorEvent, EventsResource, type ExportDealsOptions, type ExportFormat, type ExportJobsOptions, type ExtractJobResponse, type ExtractModel, type ExtractOptions, ExtractResource, type ExtractedDeal, type ExtractionErrorDetails, type FallbackConfig, type FallbackMetadata, type FallbackResult, type FallbackSource, type GenerateSSETokenOptions, type GetApiKeyStatsOptions, type GetDealsOptions, type HoverAction, type JobAlertEvent, type JobCancelledEvent, type JobCheckpointEvent, type JobCompletedEvent, type JobCreatedEvent, type JobDealsResponse, type JobFailedEvent, type JobLogEvent, type JobMetricEvent, type JobMetricsResponse, type JobPausedEvent, type JobProgressEvent, type JobQueuedEvent, type JobResponse, type JobResumedEvent, type JobStartedEvent, type JobStatus$1 as JobStatus, type JobStatusEvent, type JobStatusFilter, type JobStatusResponse, type JobSummary, type JobTypeFilter, type KeyStatsResponse, KeysResource, type ListApiKeysOptions, type ListDealsOptions, type ListDealsResponse, type ListJobsOptions, type ListJobsResponse, type ListKeysResponse, type ListWebhooksResponse, type PaginatedResponse, type PaginationInfo, type ParsedPage, type PingEvent, type PreferencesResponse, type PressAction, type PriceSignal, type PricingInfo, type ProductCategory, type ProductInfo, type RateLimitInfo, type RecommendationsResponse, type RefreshScreenshotOptions, type RequestContext, type ResumeJobResponse, type RevokeApiKeyOptions, type RotateApiKeyOptions, type RotateKeyResponse, type SSEConnectionLimitsResponse, type SSEEvent, type SSEEventType, type SSESubscribeOptions, type SSETokenResponse, type ScrapeJobResponse, type ScrapeOptions, ScrapeResource, type ScrapeResult, type ScreenshotAction, type ScreenshotAgentAction, type ScreenshotLimitsResponse, type ScreenshotOptions, type ScreenshotRefreshResponse, type ScreenshotResult, ScreenshotsResource, type ScrollAction, type SearchAiModel, type SearchAiProvider, type SearchData, type SearchDateRange, type SearchFilters, type SearchJobResponse, type SearchOptions, SearchResource, type SearchResultItem, type SearchStatusResponse, type SelectAction, type ClickAction$1 as SharedClickAction, type HoverAction$1 as SharedHoverAction, type PressAction$1 as SharedPressAction, type ScrollAction$1 as SharedScrollAction, type SelectAction$1 as SharedSelectAction, type WaitAction$1 as SharedWaitAction, type WriteAction$1 as SharedWriteAction, type Signal, type SortOrder, StatusResource, type TestWebhookResponse, type UpdatePreferencesOptions, type UpdatePreferencesResponse, type UpdateWebhookOptions, type UpdateWebhookResponse, type UrgencyLevel, type UsageStats, type ValidationError, type ValidationResult, type WaitAction, type WaitOptions, type WaitResult, type WebhookEvent, type WebhookItem, WebhooksResource, type WriteAction, DealCrawl as default, getErrorMessage, pollUntil, waitForAll, waitForAny, waitForResult };
|