@dealcrawl/sdk 2.10.0 → 2.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -19,7 +19,7 @@ interface ResponseMeta {
19
19
  timestamp: string;
20
20
  duration?: number;
21
21
  }
22
- type JobStatus = "pending" | "active" | "completed" | "failed" | "delayed" | "paused";
22
+ type JobStatus$1 = "pending" | "active" | "completed" | "failed" | "delayed" | "paused";
23
23
  interface PaginatedResponse<T> {
24
24
  data: T[];
25
25
  pagination: {
@@ -130,10 +130,19 @@ interface DealScoreSummary {
130
130
  };
131
131
  }
132
132
  interface ScreenshotResult {
133
+ /** Screenshot URL - private signed URL (default) or public URL (Enterprise opt-in) */
133
134
  url: string;
135
+ /** Whether this is a public URL (false = private signed URL with TTL) */
136
+ isPublic: boolean;
137
+ /** Expiration timestamp for signed URLs (ISO 8601, only for private URLs) */
138
+ expiresAt?: string;
139
+ /** Screenshot width in pixels */
134
140
  width: number;
141
+ /** Screenshot height in pixels */
135
142
  height: number;
143
+ /** Image format */
136
144
  format: "png" | "jpeg" | "webp";
145
+ /** File size in bytes */
137
146
  sizeBytes: number;
138
147
  }
139
148
  interface AIExtraction {
@@ -438,7 +447,7 @@ interface JobResponse {
438
447
  /** Unique job identifier */
439
448
  jobId: string;
440
449
  /** Current job status */
441
- status: JobStatus;
450
+ status: JobStatus$1;
442
451
  /** URL to check job status */
443
452
  statusUrl: string;
444
453
  /** Estimated time to completion */
@@ -468,6 +477,15 @@ interface BatchScrapeResultItem {
468
477
  /** Error message if job creation failed */
469
478
  error?: string;
470
479
  }
480
+ /** Invalid URL entry (returned when ignoreInvalidURLs=true) */
481
+ interface InvalidURLEntry {
482
+ /** The invalid URL */
483
+ url: string;
484
+ /** Reference ID if provided */
485
+ ref?: string;
486
+ /** Reason for rejection */
487
+ reason: string;
488
+ }
471
489
  /** Batch scrape response */
472
490
  interface BatchScrapeResponse {
473
491
  /** Batch ID for tracking */
@@ -482,6 +500,11 @@ interface BatchScrapeResponse {
482
500
  results: BatchScrapeResultItem[];
483
501
  /** Timestamp when batch was created */
484
502
  createdAt: string;
503
+ /**
504
+ * Invalid URLs that were skipped (only present when ignoreInvalidURLs=true)
505
+ * Firecrawl-compatible field
506
+ */
507
+ invalidURLs?: InvalidURLEntry[];
485
508
  }
486
509
  /** Batch status response */
487
510
  interface BatchStatusResponse {
@@ -502,7 +525,7 @@ interface BatchStatusResponse {
502
525
  /** Individual job statuses */
503
526
  jobs: Array<{
504
527
  id: string;
505
- status: JobStatus;
528
+ status: JobStatus$1;
506
529
  result?: unknown;
507
530
  error?: string;
508
531
  }>;
@@ -612,7 +635,7 @@ interface CheckpointInfo {
612
635
  /** Job status response */
613
636
  interface JobStatusResponse {
614
637
  jobId: string;
615
- status: JobStatus;
638
+ status: JobStatus$1;
616
639
  progress?: number;
617
640
  createdAt: string;
618
641
  completedAt?: string;
@@ -652,7 +675,7 @@ interface ResumeJobResponse {
652
675
  /** Job metrics response */
653
676
  interface JobMetricsResponse {
654
677
  jobId: string;
655
- status: JobStatus;
678
+ status: JobStatus$1;
656
679
  metrics: {
657
680
  totalUrls: number;
658
681
  successfulUrls: number;
@@ -671,7 +694,7 @@ interface CancelJobResponse {
671
694
  interface JobSummary {
672
695
  id: string;
673
696
  type: "scrape" | "crawl" | "dork" | "extract";
674
- status: JobStatus;
697
+ status: JobStatus$1;
675
698
  input: Record<string, unknown>;
676
699
  dealsFound?: number;
677
700
  avgDealScore?: number;
@@ -769,10 +792,17 @@ interface CreateWebhookResponse {
769
792
  minDealScore: number;
770
793
  active: boolean;
771
794
  }
772
- /** List webhooks response */
795
+ /** List webhooks response (standardized list format) */
773
796
  interface ListWebhooksResponse {
774
- webhooks: WebhookItem[];
775
- count: number;
797
+ data: WebhookItem[];
798
+ pagination: {
799
+ page: number;
800
+ limit: number;
801
+ total: number;
802
+ totalPages: number;
803
+ hasMore: boolean;
804
+ };
805
+ meta?: Record<string, unknown>;
776
806
  }
777
807
  /** Update webhook response */
778
808
  interface UpdateWebhookResponse {
@@ -964,7 +994,7 @@ interface AgentStepResponse {
964
994
  currentUrl?: string;
965
995
  }
966
996
  /** Agent completion reason */
967
- type AgentCompletionReason = "goal_achieved" | "max_steps_reached" | "stuck" | "error" | "cancelled";
997
+ type AgentCompletionReason = "goal_achieved" | "max_steps_reached" | "stuck" | "error";
968
998
  /** Agent result (from completed job) */
969
999
  interface AgentResultResponse {
970
1000
  /** Extracted/collected data (matches provided schema if given) */
@@ -1013,6 +1043,113 @@ interface SchemaGenerationResponse {
1013
1043
  /** Confidence score (0-1) in the generated schema */
1014
1044
  confidence: number;
1015
1045
  }
1046
+ /** SSE token response */
1047
+ interface SSETokenResponse {
1048
+ /** Short-lived SSE authentication token */
1049
+ token: string;
1050
+ /** Expiration timestamp (ISO 8601) */
1051
+ expiresAt: string;
1052
+ /** TTL in seconds (default: 300 = 5 minutes) */
1053
+ expiresIn: number;
1054
+ /** Token type identifier */
1055
+ type: "sse_token";
1056
+ /** Usage instructions for the token */
1057
+ usage: string;
1058
+ }
1059
+ /** SSE connection limits response */
1060
+ interface SSEConnectionLimitsResponse {
1061
+ /** Client tier */
1062
+ tier: "free" | "pro" | "enterprise";
1063
+ /** SSE connection limits */
1064
+ sse: {
1065
+ /** Maximum concurrent connections allowed */
1066
+ maxConnections: number;
1067
+ /** Current active connections */
1068
+ currentConnections: number;
1069
+ /** Available connection slots */
1070
+ available: number;
1071
+ };
1072
+ }
1073
+ /** Tier-specific TTL limits */
1074
+ interface TtlLimits {
1075
+ /** Minimum TTL in seconds */
1076
+ min: number;
1077
+ /** Maximum TTL in seconds */
1078
+ max: number;
1079
+ /** Default TTL in seconds */
1080
+ default: number;
1081
+ }
1082
+ /** Formatted TTL limits (human-readable) */
1083
+ interface FormattedTtlLimits {
1084
+ /** Minimum TTL formatted (e.g., "1 hour") */
1085
+ min: string;
1086
+ /** Maximum TTL formatted (e.g., "30 days") */
1087
+ max: string;
1088
+ /** Default TTL formatted (e.g., "7 days") */
1089
+ default: string;
1090
+ }
1091
+ /** Screenshot signed URL refresh response */
1092
+ interface ScreenshotRefreshResponse {
1093
+ /** Success status */
1094
+ success: boolean;
1095
+ /** New signed URL */
1096
+ url: string;
1097
+ /** Expiration timestamp (ISO 8601) */
1098
+ expiresAt: string;
1099
+ /** TTL in seconds */
1100
+ ttl: number;
1101
+ /** Tier-specific limits for reference */
1102
+ tierLimits: TtlLimits;
1103
+ /** Additional metadata */
1104
+ metadata: {
1105
+ /** File path in storage */
1106
+ path: string;
1107
+ /** Bucket name */
1108
+ bucket: string;
1109
+ /** Timestamp when URL was refreshed */
1110
+ refreshedAt: string;
1111
+ };
1112
+ }
1113
+ /** Screenshot TTL limits response */
1114
+ interface ScreenshotLimitsResponse {
1115
+ /** Client tier */
1116
+ tier: "free" | "pro" | "enterprise";
1117
+ /** TTL limits in seconds */
1118
+ limits: TtlLimits;
1119
+ /** Human-readable formatted limits */
1120
+ formattedLimits: FormattedTtlLimits;
1121
+ }
1122
+ /**
1123
+ * Metadata extracted during HTML to Markdown conversion
1124
+ */
1125
+ interface ConversionMetadata {
1126
+ /** Page title if found */
1127
+ title?: string;
1128
+ /** Word count in resulting markdown */
1129
+ wordCount: number;
1130
+ /** Number of links in markdown */
1131
+ linkCount: number;
1132
+ /** Number of images in markdown */
1133
+ imageCount: number;
1134
+ /** Conversion duration in milliseconds */
1135
+ conversionTimeMs: number;
1136
+ }
1137
+ /**
1138
+ * HTML to Markdown conversion response
1139
+ */
1140
+ interface ConvertResponse {
1141
+ /** Success status */
1142
+ success: boolean;
1143
+ /** Conversion result data */
1144
+ data: {
1145
+ /** Generated markdown content */
1146
+ markdown: string;
1147
+ /** Extraction metadata */
1148
+ metadata: ConversionMetadata;
1149
+ /** Warnings (truncation, excluded elements, etc.) */
1150
+ warnings?: string[];
1151
+ };
1152
+ }
1016
1153
 
1017
1154
  /**
1018
1155
  * Polling Utilities
@@ -1092,10 +1229,19 @@ interface ScreenshotOptions {
1092
1229
  /**
1093
1230
  * Request public URL (Enterprise only, default: false)
1094
1231
  * SECURITY WARNING: Public URLs can expose personal data, copyrighted content, and tokens
1095
- * By default, screenshots use private signed URLs with 7-day expiration
1232
+ * By default, screenshots use private signed URLs with tier-specific expiration
1096
1233
  * Requires Enterprise tier
1097
1234
  */
1098
1235
  publicUrl?: boolean;
1236
+ /**
1237
+ * TTL for signed URLs in seconds (only for private screenshots)
1238
+ * Tier limits:
1239
+ * - Free: 3600 (1h) - 86400 (24h), default: 86400
1240
+ * - Pro: 3600 (1h) - 604800 (7d), default: 604800
1241
+ * - Enterprise: 3600 (1h) - 2592000 (30d), default: 604800
1242
+ * If not specified, uses tier-specific default
1243
+ */
1244
+ signedUrlTtl?: number;
1099
1245
  }
1100
1246
  /** Options for scraping a single page */
1101
1247
  interface ScrapeOptions {
@@ -1158,8 +1304,6 @@ interface BatchScrapeItem {
1158
1304
  outputMarkdown?: boolean;
1159
1305
  /** Override markdownBaseUrl for this URL */
1160
1306
  markdownBaseUrl?: string;
1161
- /** Override actions for this URL */
1162
- actions?: ActionInput[];
1163
1307
  }
1164
1308
  /** Default options applied to all URLs in a batch */
1165
1309
  interface BatchScrapeDefaults {
@@ -1195,8 +1339,6 @@ interface BatchScrapeDefaults {
1195
1339
  outputMarkdown?: boolean;
1196
1340
  /** Base URL for resolving relative URLs in markdown */
1197
1341
  markdownBaseUrl?: string;
1198
- /** Browser actions to execute before scraping */
1199
- actions?: ActionInput[];
1200
1342
  }
1201
1343
  /** Options for batch scraping multiple URLs */
1202
1344
  interface BatchScrapeOptions {
@@ -1208,8 +1350,13 @@ interface BatchScrapeOptions {
1208
1350
  webhookUrl?: string;
1209
1351
  /** Priority for batch jobs (1-10, higher = faster) */
1210
1352
  priority?: number;
1211
- /** Delay between job submissions in ms (0-5000) */
1212
- delay?: number;
1353
+ /** Delay between job submissions in ms (0-5000, default: 0) */
1354
+ delayMs?: number;
1355
+ /**
1356
+ * Continue processing even if some URLs are invalid (default: false)
1357
+ * Invalid URLs will be returned in the response with status: "failed"
1358
+ */
1359
+ ignoreInvalidURLs?: boolean;
1213
1360
  }
1214
1361
  /** AI provider for search optimization */
1215
1362
  type SearchAiProvider = "openai" | "anthropic";
@@ -1237,11 +1384,11 @@ interface SearchOptions {
1237
1384
  /** Search query (required) */
1238
1385
  query: string;
1239
1386
  /** Maximum number of results (1-100, default: 10) */
1240
- maxResults?: number;
1241
- /** Auto-scrape top results */
1242
- autoScrape?: boolean;
1243
- /** Number of results to auto-scrape (1-10, default: 5) */
1244
- autoScrapeLimit?: number;
1387
+ limit?: number;
1388
+ /** Auto-scrape search results (default: false) */
1389
+ scrapeResults?: boolean;
1390
+ /** Maximum results to auto-scrape when scrapeResults=true (1-10, default: 5) */
1391
+ maxScrapeResults?: number;
1245
1392
  /** Use AI to optimize the search query */
1246
1393
  useAiOptimization?: boolean;
1247
1394
  /** AI provider for query optimization */
@@ -1275,7 +1422,7 @@ interface PriceRange {
1275
1422
  interface CrawlOptions {
1276
1423
  /** Starting URL for the crawl (required) */
1277
1424
  url: string;
1278
- /** Alias for url - Starting URL for the crawl */
1425
+ /** @deprecated Use 'url' instead. Kept for backward compatibility. */
1279
1426
  startUrl?: string;
1280
1427
  /** Prompt to guide the crawl (optional) */
1281
1428
  prompt?: string;
@@ -1352,7 +1499,7 @@ interface CrawlTemplate {
1352
1499
  defaultOptions: Partial<CrawlOptions>;
1353
1500
  }
1354
1501
  /** LLM model options for extraction */
1355
- type ExtractModel = "gpt-4o-mini" | "gpt-4o" | "claude-3-haiku" | "claude-3-sonnet";
1502
+ type ExtractModel = "gpt-4o-mini" | "gpt-4o" | "gpt-4o-2024-11-20" | "o1-mini" | "claude-3-5-haiku-20241022" | "claude-3-5-sonnet-20241022" | "claude-3-opus-20240229";
1356
1503
  /** Options for structured data extraction */
1357
1504
  interface ExtractOptions {
1358
1505
  /** URL to extract data from (required) */
@@ -1395,8 +1542,6 @@ interface DorkOptions {
1395
1542
  inTitle?: string;
1396
1543
  /** Maximum results to return (default: 10, max: 100) */
1397
1544
  maxResults?: number;
1398
- /** Region code (2 letter ISO code) */
1399
- region?: string;
1400
1545
  }
1401
1546
  /** Options for getting job deals */
1402
1547
  interface GetDealsOptions {
@@ -1472,19 +1617,25 @@ interface ExportDealsOptions {
1472
1617
  format?: ExportFormat;
1473
1618
  /** Minimum deal score */
1474
1619
  minScore?: number;
1475
- /** Maximum price */
1476
- maxPrice?: number;
1477
1620
  /** Filter by category */
1478
1621
  category?: string;
1479
- /** Include raw signals in export */
1480
- includeRawSignals?: boolean;
1622
+ /** Filter by sync status to DealUp */
1623
+ synced?: boolean;
1624
+ /** Filter from date (ISO string) */
1625
+ fromDate?: string;
1626
+ /** Filter to date (ISO string) */
1627
+ toDate?: string;
1628
+ /** Maximum number of deals to export (default: 500, max: 1000) */
1629
+ limit?: number;
1481
1630
  }
1482
1631
  /** Webhook event types */
1483
1632
  type WebhookEvent = "deal.found" | "deal.synced" | "crawl.completed" | "crawl.failed";
1484
1633
  /** Options for creating a webhook */
1485
1634
  interface CreateWebhookOptions {
1486
- /** Event type to subscribe to */
1487
- event: WebhookEvent;
1635
+ /** Event types to subscribe to (array) */
1636
+ events?: WebhookEvent[];
1637
+ /** @deprecated Use 'events' array instead */
1638
+ event?: WebhookEvent;
1488
1639
  /** URL to receive webhook notifications */
1489
1640
  url: string;
1490
1641
  /** Secret for signature verification */
@@ -1513,7 +1664,7 @@ interface UpdateWebhookOptions {
1513
1664
  * API key scope - Must match @dealcrawl/shared/src/types/api-key.types.ts
1514
1665
  * These are the actual scopes enforced by the backend via requireScope() middleware
1515
1666
  */
1516
- type ApiKeyScope = "scrape" | "scrape:batch" | "search" | "crawl" | "dork" | "extract" | "agent" | "status" | "data:read" | "data:export" | "keys:manage" | "webhooks:manage";
1667
+ type ApiKeyScope = "scrape" | "crawl" | "dork" | "extract" | "agent" | "status" | "data:read" | "data:export" | "keys:manage" | "webhooks:manage";
1517
1668
  /**
1518
1669
  * All available scopes (for reference and validation)
1519
1670
  */
@@ -1691,6 +1842,71 @@ interface SchemaGenerationOptions {
1691
1842
  /** LLM provider for generation (default: openai) */
1692
1843
  model?: AgentModel;
1693
1844
  }
1845
+ /**
1846
+ * Options for subscribing to SSE events
1847
+ */
1848
+ interface SSESubscribeOptions {
1849
+ /**
1850
+ * Last event ID for reconnection/replay
1851
+ * If provided, server will replay missed events since this ID
1852
+ */
1853
+ lastEventId?: string;
1854
+ /**
1855
+ * Callback for received events
1856
+ * Called for each event received from the server
1857
+ */
1858
+ onEvent?: (event: MessageEvent) => void;
1859
+ /**
1860
+ * Callback for errors
1861
+ * Called when connection error occurs
1862
+ */
1863
+ onError?: (error: Error) => void;
1864
+ /**
1865
+ * Callback for connection open
1866
+ * Called when SSE connection is established
1867
+ */
1868
+ onOpen?: (event: Event) => void;
1869
+ /**
1870
+ * Auto-reconnect on connection loss (default: true)
1871
+ * If true, EventSource will automatically reconnect
1872
+ */
1873
+ autoReconnect?: boolean;
1874
+ /**
1875
+ * Reconnection delay in milliseconds (default: 3000)
1876
+ * How long to wait before attempting reconnection
1877
+ */
1878
+ reconnectDelay?: number;
1879
+ }
1880
+ /**
1881
+ * Options for HTML to Markdown conversion
1882
+ */
1883
+ interface ConvertOptions {
1884
+ /** HTML content to convert (required, max 10MB) */
1885
+ html: string;
1886
+ /** Base URL for resolving relative URLs */
1887
+ baseUrl?: string;
1888
+ /** Conversion options */
1889
+ options?: {
1890
+ /** Enable GFM tables (default: true) */
1891
+ gfmTables?: boolean;
1892
+ /** Enable GFM strikethrough (default: true) */
1893
+ gfmStrikethrough?: boolean;
1894
+ /** Enable GFM task lists (default: true) */
1895
+ gfmTaskLists?: boolean;
1896
+ /** Remove noise elements like nav, footer, ads (default: true) */
1897
+ removeNoise?: boolean;
1898
+ /** CSS selectors to exclude from conversion */
1899
+ excludeSelectors?: string[];
1900
+ /** Resolve relative URLs to absolute (default: true) */
1901
+ absoluteUrls?: boolean;
1902
+ /** Maximum output length in characters (0 = unlimited, max: 1,000,000) */
1903
+ maxLength?: number;
1904
+ /** Include images in output (default: true) */
1905
+ includeImages?: boolean;
1906
+ /** Include links in output (default: true) */
1907
+ includeLinks?: boolean;
1908
+ };
1909
+ }
1694
1910
 
1695
1911
  /**
1696
1912
  * Account Resource
@@ -2035,6 +2251,257 @@ declare class AgentResource {
2035
2251
  generateSchema(options: SchemaGenerationOptions): Promise<SchemaGenerationResponse>;
2036
2252
  }
2037
2253
 
2254
+ /**
2255
+ * Auth Resource
2256
+ * Handles SSE (Server-Sent Events) authentication
2257
+ */
2258
+
2259
+ /**
2260
+ * Options for generating an SSE token
2261
+ */
2262
+ interface GenerateSSETokenOptions {
2263
+ /**
2264
+ * Optional job ID to restrict token to a specific job
2265
+ * If provided, token can only be used for /v1/events/:jobId
2266
+ * If omitted, token can be used for /v1/events (all events)
2267
+ */
2268
+ jobId?: string;
2269
+ }
2270
+ /**
2271
+ * Auth resource class
2272
+ * Provides methods for SSE authentication
2273
+ *
2274
+ * @example
2275
+ * ```ts
2276
+ * // Generate SSE token for browser EventSource
2277
+ * const { token, expiresAt } = await client.auth.generateSSEToken();
2278
+ *
2279
+ * // Use in browser
2280
+ * const eventSource = new EventSource(`/v1/events?token=${token}`);
2281
+ *
2282
+ * // Token for specific job
2283
+ * const jobToken = await client.auth.generateSSEToken({ jobId: "job_123" });
2284
+ * const jobEvents = new EventSource(`/v1/events/job_123?token=${jobToken.token}`);
2285
+ * ```
2286
+ */
2287
+ declare class AuthResource {
2288
+ private ctx;
2289
+ constructor(ctx: RequestContext);
2290
+ /**
2291
+ * Generate SSE authentication token
2292
+ *
2293
+ * Required for browser-based SSE connections because EventSource API
2294
+ * doesn't support custom headers. Token is short-lived (5 minutes).
2295
+ *
2296
+ * Security:
2297
+ * - Requires valid API key (Bearer token)
2298
+ * - Token expires in 5 minutes
2299
+ * - Token can be restricted to specific job
2300
+ * - Token stored in Redis (revocable)
2301
+ *
2302
+ * @example
2303
+ * ```ts
2304
+ * // 1. Generate token
2305
+ * const { token, expiresAt } = await client.auth.generateSSEToken();
2306
+ * console.log(`Token expires at: ${expiresAt}`);
2307
+ *
2308
+ * // 2. Use in browser EventSource
2309
+ * const eventSource = new EventSource(`/v1/events?token=${token}`);
2310
+ *
2311
+ * eventSource.addEventListener('job.completed', (event) => {
2312
+ * const data = JSON.parse(event.data);
2313
+ * console.log('Job completed:', data);
2314
+ * });
2315
+ *
2316
+ * // 3. For specific job only
2317
+ * const jobToken = await client.auth.generateSSEToken({ jobId: "job_abc123" });
2318
+ * const jobEvents = new EventSource(`/v1/events/job_abc123?token=${jobToken.token}`);
2319
+ * ```
2320
+ */
2321
+ generateSSEToken(options?: GenerateSSETokenOptions): Promise<SSETokenResponse>;
2322
+ /**
2323
+ * Get SSE connection limits for current tier
2324
+ *
2325
+ * Shows how many concurrent SSE connections are allowed
2326
+ * and how many are currently active.
2327
+ *
2328
+ * Tier limits:
2329
+ * - Free: 10 concurrent connections
2330
+ * - Pro: 50 concurrent connections
2331
+ * - Enterprise: 200 concurrent connections
2332
+ *
2333
+ * @example
2334
+ * ```ts
2335
+ * const limits = await client.auth.getLimits();
2336
+ *
2337
+ * console.log(`Tier: ${limits.tier}`);
2338
+ * console.log(`Max connections: ${limits.sse.maxConnections}`);
2339
+ * console.log(`Current connections: ${limits.sse.currentConnections}`);
2340
+ * console.log(`Available: ${limits.sse.available}`);
2341
+ *
2342
+ * // Check before opening new connection
2343
+ * if (limits.sse.available > 0) {
2344
+ * const token = await client.auth.generateSSEToken();
2345
+ * const eventSource = new EventSource(`/v1/events?token=${token.token}`);
2346
+ * } else {
2347
+ * console.error('No available SSE connection slots');
2348
+ * }
2349
+ * ```
2350
+ */
2351
+ getLimits(): Promise<SSEConnectionLimitsResponse>;
2352
+ }
2353
+
2354
+ /**
2355
+ * Convert Resource
2356
+ * Handles HTML to Markdown conversion
2357
+ */
2358
+
2359
+ /**
2360
+ * Convert resource class
2361
+ * Provides methods for HTML to Markdown conversion
2362
+ *
2363
+ * @example
2364
+ * ```ts
2365
+ * // Convert HTML to markdown
2366
+ * const result = await client.convert.htmlToMarkdown({
2367
+ * html: "<h1>Title</h1><p>Content</p>",
2368
+ * baseUrl: "https://example.com",
2369
+ * options: {
2370
+ * gfmTables: true,
2371
+ * removeNoise: true
2372
+ * }
2373
+ * });
2374
+ *
2375
+ * console.log(result.data.markdown);
2376
+ * console.log(result.data.metadata.wordCount);
2377
+ * ```
2378
+ */
2379
+ declare class ConvertResource {
2380
+ private ctx;
2381
+ constructor(ctx: RequestContext);
2382
+ /**
2383
+ * Convert HTML to Markdown
2384
+ *
2385
+ * Transforms raw HTML content into clean, readable Markdown using GitHub Flavored Markdown (GFM).
2386
+ * Useful for:
2387
+ * - Converting scraped HTML to markdown for LLM processing
2388
+ * - Cleaning up messy HTML from web pages
2389
+ * - Extracting main content while removing noise (ads, nav, footer)
2390
+ * - Creating documentation from HTML sources
2391
+ *
2392
+ * Features:
2393
+ * - GFM table, strikethrough, and task list support
2394
+ * - Automatic noise removal (scripts, ads, navigation)
2395
+ * - Relative URL resolution
2396
+ * - Custom element exclusion via CSS selectors
2397
+ * - Output length limiting
2398
+ *
2399
+ * @param options - Conversion options
2400
+ * @returns Conversion result with markdown, metadata, and warnings
2401
+ *
2402
+ * @example Basic usage
2403
+ * ```ts
2404
+ * const result = await client.convert.htmlToMarkdown({
2405
+ * html: "<h1>Product</h1><p>Price: $99</p>"
2406
+ * });
2407
+ * console.log(result.data.markdown);
2408
+ * ```
2409
+ *
2410
+ * @example With all options
2411
+ * ```ts
2412
+ * const result = await client.convert.htmlToMarkdown({
2413
+ * html: htmlContent,
2414
+ * baseUrl: "https://shop.example.com",
2415
+ * options: {
2416
+ * gfmTables: true,
2417
+ * removeNoise: true,
2418
+ * excludeSelectors: [".advertisement", "#sidebar"],
2419
+ * absoluteUrls: true,
2420
+ * maxLength: 100000,
2421
+ * includeImages: true,
2422
+ * includeLinks: true
2423
+ * }
2424
+ * });
2425
+ *
2426
+ * // Check metadata
2427
+ * console.log(`Words: ${result.data.metadata.wordCount}`);
2428
+ * console.log(`Links: ${result.data.metadata.linkCount}`);
2429
+ * console.log(`Images: ${result.data.metadata.imageCount}`);
2430
+ * console.log(`Conversion time: ${result.data.metadata.conversionTimeMs}ms`);
2431
+ *
2432
+ * // Check for warnings
2433
+ * if (result.data.warnings?.length) {
2434
+ * console.warn("Conversion warnings:", result.data.warnings);
2435
+ * }
2436
+ * ```
2437
+ *
2438
+ * @example Converting scraped HTML
2439
+ * ```ts
2440
+ * // First scrape a page
2441
+ * const scrapeJob = await client.scrape.create({
2442
+ * url: "https://example.com/article"
2443
+ * });
2444
+ * const scrapeResult = await client.waitForResult(scrapeJob.jobId);
2445
+ *
2446
+ * // Then convert HTML to markdown
2447
+ * const markdown = await client.convert.htmlToMarkdown({
2448
+ * html: scrapeResult.data.html,
2449
+ * baseUrl: scrapeResult.data.url,
2450
+ * options: {
2451
+ * removeNoise: true,
2452
+ * onlyMainContent: true
2453
+ * }
2454
+ * });
2455
+ * ```
2456
+ */
2457
+ htmlToMarkdown(options: ConvertOptions): Promise<ConvertResponse>;
2458
+ /**
2459
+ * Alias for htmlToMarkdown() for convenience
2460
+ *
2461
+ * @example
2462
+ * ```ts
2463
+ * const result = await client.convert.toMarkdown({
2464
+ * html: "<h1>Hello</h1>"
2465
+ * });
2466
+ * ```
2467
+ */
2468
+ toMarkdown(options: ConvertOptions): Promise<ConvertResponse>;
2469
+ /**
2470
+ * Convert HTML with minimal options (just the HTML content)
2471
+ * Uses all default settings
2472
+ *
2473
+ * @param html - HTML content to convert
2474
+ * @param baseUrl - Optional base URL for resolving relative links
2475
+ * @returns Conversion result
2476
+ *
2477
+ * @example
2478
+ * ```ts
2479
+ * const result = await client.convert.quick(
2480
+ * "<h1>Title</h1><p>Content</p>",
2481
+ * "https://example.com"
2482
+ * );
2483
+ * console.log(result.data.markdown);
2484
+ * ```
2485
+ */
2486
+ quick(html: string, baseUrl?: string): Promise<ConvertResponse>;
2487
+ /**
2488
+ * Convert HTML with noise removal enabled
2489
+ * Removes navigation, footer, ads, scripts, and other clutter
2490
+ *
2491
+ * @param html - HTML content to convert
2492
+ * @param baseUrl - Optional base URL
2493
+ * @returns Conversion result with clean markdown
2494
+ *
2495
+ * @example
2496
+ * ```ts
2497
+ * // Extract just the main content from a messy page
2498
+ * const result = await client.convert.clean(messyHtml, "https://example.com");
2499
+ * console.log(result.data.markdown); // Clean, readable markdown
2500
+ * ```
2501
+ */
2502
+ clean(html: string, baseUrl?: string): Promise<ConvertResponse>;
2503
+ }
2504
+
2038
2505
  /**
2039
2506
  * Crawl Resource
2040
2507
  * Handles website crawling operations
@@ -2220,17 +2687,9 @@ declare class DataResource {
2220
2687
  * @example
2221
2688
  * ```ts
2222
2689
  * const electronicsDeals = await client.data.getDealsByCategory("electronics");
2223
- async getDealsByCategory(
2224
- category: string,
2225
- options?: Omit<ListDealsOptions, "category">
2226
- ): Promise<ListDealsResponse> {
2227
- if (!category || !category.trim()) {
2228
- throw new Error("category is required and cannot be empty");
2229
- }
2230
- return this.listDeals({ category, ...options });
2231
- } return this.listDeals({ category, ...options });
2232
- }
2233
-
2690
+ * ```
2691
+ */
2692
+ getDealsByCategory(category: string, options?: Omit<ListDealsOptions, "category">): Promise<ListDealsResponse>;
2234
2693
  /**
2235
2694
  * Get unsynced deals (not yet sent to DealUp)
2236
2695
  *
@@ -2385,6 +2844,177 @@ declare class DorkResource {
2385
2844
  buildQuery(options: DorkOptions): string;
2386
2845
  }
2387
2846
 
2847
+ /**
2848
+ * Events Resource
2849
+ * Handles real-time SSE (Server-Sent Events) streaming
2850
+ */
2851
+
2852
+ /**
2853
+ * Events resource class
2854
+ * Provides methods for subscribing to real-time job events via SSE
2855
+ *
2856
+ * IMPORTANT: Browser Support Only
2857
+ * This resource uses the EventSource API which is only available in browsers.
2858
+ * For Node.js, use polling via client.status.get() instead.
2859
+ *
2860
+ * @example Browser Usage
2861
+ * ```ts
2862
+ * // 1. Generate SSE token (required for EventSource)
2863
+ * const { token } = await client.auth.generateSSEToken();
2864
+ *
2865
+ * // 2. Subscribe to all events
2866
+ * const eventSource = client.events.subscribe({
2867
+ * token,
2868
+ * onEvent: (event) => {
2869
+ * console.log('Event:', event.type, event.data);
2870
+ * }
2871
+ * });
2872
+ *
2873
+ * // 3. Or subscribe to specific job
2874
+ * const jobEvents = client.events.subscribeToJob('job_123', {
2875
+ * token,
2876
+ * onEvent: (event) => {
2877
+ * const data = JSON.parse(event.data);
2878
+ * console.log('Job event:', data);
2879
+ * }
2880
+ * });
2881
+ *
2882
+ * // 4. Clean up when done
2883
+ * eventSource.close();
2884
+ * ```
2885
+ */
2886
+ declare class EventsResource {
2887
+ private ctx;
2888
+ constructor(ctx: RequestContext);
2889
+ /**
2890
+ * Subscribe to all events for authenticated client
2891
+ *
2892
+ * Opens an SSE connection to receive real-time events for all jobs.
2893
+ * Requires an SSE token obtained via client.auth.generateSSEToken().
2894
+ *
2895
+ * Event Types:
2896
+ * - Job lifecycle: job.created, job.queued, job.started, job.progress,
2897
+ * job.completed, job.failed, job.cancelled
2898
+ * - Job details: job.log, job.metric, job.alert, job.checkpoint
2899
+ * - Deals: deal.found, deal.validated
2900
+ * - System: ping, connection.open, connection.close, error
2901
+ *
2902
+ * Features:
2903
+ * - Automatic reconnection on disconnect
2904
+ * - Event replay via Last-Event-ID
2905
+ * - Keepalive pings every 15 seconds
2906
+ * - Max connection time: 1 hour
2907
+ *
2908
+ * @param token - SSE authentication token from client.auth.generateSSEToken()
2909
+ * @param options - Subscription options (callbacks, reconnection settings)
2910
+ *
2911
+ * @example
2912
+ * ```ts
2913
+ * // Generate token
2914
+ * const { token } = await client.auth.generateSSEToken();
2915
+ *
2916
+ * // Subscribe with event handlers
2917
+ * const eventSource = client.events.subscribe(token, {
2918
+ * onEvent: (event) => {
2919
+ * // Handle all events
2920
+ * console.log('Event:', event.type);
2921
+ * const data = JSON.parse(event.data);
2922
+ *
2923
+ * if (data.jobId) {
2924
+ * console.log(`Job ${data.jobId}:`, data);
2925
+ * }
2926
+ * },
2927
+ * onError: (error) => {
2928
+ * console.error('SSE error:', error);
2929
+ * },
2930
+ * onOpen: () => {
2931
+ * console.log('SSE connection opened');
2932
+ * }
2933
+ * });
2934
+ *
2935
+ * // Listen for specific event types
2936
+ * eventSource.addEventListener('job.completed', (event) => {
2937
+ * const data = JSON.parse(event.data);
2938
+ * console.log('Job completed:', data);
2939
+ * });
2940
+ *
2941
+ * // Clean up
2942
+ * eventSource.close();
2943
+ * ```
2944
+ */
2945
+ subscribe(token: string, options?: Omit<SSESubscribeOptions, "lastEventId">): EventSource;
2946
+ /**
2947
+ * Subscribe to events for a specific job
2948
+ *
2949
+ * Opens an SSE connection filtered to a single job.
2950
+ * More efficient than global subscription when tracking one job.
2951
+ *
2952
+ * @param jobId - Job ID to subscribe to
2953
+ * @param token - SSE authentication token
2954
+ * @param options - Subscription options
2955
+ *
2956
+ * @example
2957
+ * ```ts
2958
+ * // Start a scrape job
2959
+ * const job = await client.scrape.create({ url: "https://example.com" });
2960
+ *
2961
+ * // Generate SSE token for this job
2962
+ * const { token } = await client.auth.generateSSEToken({ jobId: job.jobId });
2963
+ *
2964
+ * // Subscribe to job events
2965
+ * const eventSource = client.events.subscribeToJob(job.jobId, token, {
2966
+ * onEvent: (event) => {
2967
+ * const data = JSON.parse(event.data);
2968
+ * console.log(`[${event.type}]`, data);
2969
+ * }
2970
+ * });
2971
+ *
2972
+ * // Listen for completion
2973
+ * eventSource.addEventListener('job.completed', (event) => {
2974
+ * const data = JSON.parse(event.data);
2975
+ * console.log('Scrape completed!', data.summary);
2976
+ * eventSource.close();
2977
+ * });
2978
+ *
2979
+ * // Listen for progress
2980
+ * eventSource.addEventListener('job.progress', (event) => {
2981
+ * const data = JSON.parse(event.data);
2982
+ * console.log(`Progress: ${data.progress}%`);
2983
+ * });
2984
+ *
2985
+ * // Listen for errors
2986
+ * eventSource.addEventListener('job.failed', (event) => {
2987
+ * const data = JSON.parse(event.data);
2988
+ * console.error('Job failed:', data.error);
2989
+ * eventSource.close();
2990
+ * });
2991
+ * ```
2992
+ */
2993
+ subscribeToJob(jobId: string, token: string, options?: Omit<SSESubscribeOptions, "lastEventId">): EventSource;
2994
+ /**
2995
+ * Helper: Wait for job completion via SSE
2996
+ *
2997
+ * Convenience method that subscribes to a job and resolves when complete.
2998
+ * Automatically handles token generation and cleanup.
2999
+ *
3000
+ * @param jobId - Job ID to wait for
3001
+ * @param onProgress - Optional progress callback
3002
+ *
3003
+ * @example
3004
+ * ```ts
3005
+ * const job = await client.scrape.create({ url: "https://example.com" });
3006
+ *
3007
+ * // Wait for completion with progress updates
3008
+ * const result = await client.events.waitForCompletion(job.jobId, (progress) => {
3009
+ * console.log(`Progress: ${progress}%`);
3010
+ * });
3011
+ *
3012
+ * console.log('Job completed:', result);
3013
+ * ```
3014
+ */
3015
+ waitForCompletion<T = unknown>(jobId: string, onProgress?: (progress: number) => void): Promise<T>;
3016
+ }
3017
+
2388
3018
  /**
2389
3019
  * Extract Resource
2390
3020
  * Handles LLM-based structured data extraction
@@ -2594,7 +3224,9 @@ declare class KeysResource {
2594
3224
  */
2595
3225
  revokeAll(): Promise<{
2596
3226
  success: boolean;
2597
- count: number;
3227
+ revokedCount: number;
3228
+ message: string;
3229
+ warning: string;
2598
3230
  }>;
2599
3231
  /**
2600
3232
  * Get active keys only
@@ -2689,7 +3321,8 @@ declare class ScrapeResource {
2689
3321
  * { url: "https://shop1.com/product1" },
2690
3322
  * { url: "https://shop2.com/deal", extractDeal: true }
2691
3323
  * ],
2692
- * defaults: { detectSignals: true }
3324
+ * defaults: { detectSignals: true },
3325
+ * ignoreInvalidURLs: true
2693
3326
  * });
2694
3327
  * console.log(batch.batchId, batch.results);
2695
3328
  * ```
@@ -2720,6 +3353,58 @@ declare class ScrapeResource {
2720
3353
  batchForDeals(urls: string[], options?: Omit<BatchScrapeOptions, "urls">): Promise<BatchScrapeResponse>;
2721
3354
  }
2722
3355
 
3356
+ /**
3357
+ * Screenshots Resource
3358
+ * Handles screenshot signed URL refresh and TTL limits
3359
+ */
3360
+
3361
+ /**
3362
+ * Options for refreshing a screenshot signed URL
3363
+ */
3364
+ interface RefreshScreenshotOptions {
3365
+ /** File path in storage (e.g., "job_abc123/timestamp_id_url.png") */
3366
+ path: string;
3367
+ /** Optional: New TTL in seconds (must be within tier limits) */
3368
+ ttl?: number;
3369
+ /** Optional: Bucket name (defaults to 'screenshots-private') */
3370
+ bucket?: string;
3371
+ }
3372
+ /**
3373
+ * Screenshots resource class
3374
+ * Provides methods for managing screenshot signed URLs
3375
+ */
3376
+ declare class ScreenshotsResource {
3377
+ private ctx;
3378
+ constructor(ctx: RequestContext);
3379
+ /**
3380
+ * Refresh a signed URL before expiration
3381
+ *
3382
+ * @example
3383
+ * ```ts
3384
+ * const refreshed = await client.screenshots.refresh({
3385
+ * path: "job_abc123/1234567890_nanoid_example.png",
3386
+ * ttl: 604800 // 7 days
3387
+ * });
3388
+ * console.log(refreshed.url); // New signed URL
3389
+ * console.log(refreshed.expiresAt); // "2026-01-25T12:00:00Z"
3390
+ * console.log(refreshed.tierLimits); // { min: 3600, max: 604800, default: 604800 }
3391
+ * ```
3392
+ */
3393
+ refresh(options: RefreshScreenshotOptions): Promise<ScreenshotRefreshResponse>;
3394
+ /**
3395
+ * Get TTL limits for the current tier
3396
+ *
3397
+ * @example
3398
+ * ```ts
3399
+ * const limits = await client.screenshots.getLimits();
3400
+ * console.log(limits.tier); // "pro"
3401
+ * console.log(limits.limits.max); // 604800 (7 days in seconds)
3402
+ * console.log(limits.formattedLimits.max); // "7 days"
3403
+ * ```
3404
+ */
3405
+ getLimits(): Promise<ScreenshotLimitsResponse>;
3406
+ }
3407
+
2723
3408
  /**
2724
3409
  * Search Resource
2725
3410
  * Handles web search operations with AI optimization
@@ -2739,7 +3424,7 @@ declare class SearchResource {
2739
3424
  * ```ts
2740
3425
  * const result = await client.search.create({
2741
3426
  * query: "laptop deals black friday",
2742
- * maxResults: 20,
3427
+ * limit: 20,
2743
3428
  * useDealScoring: true
2744
3429
  * });
2745
3430
  * ```
@@ -2775,12 +3460,12 @@ declare class SearchResource {
2775
3460
  * @example
2776
3461
  * ```ts
2777
3462
  * const result = await client.search.andScrape("promo codes", {
2778
- * autoScrapeLimit: 5
3463
+ * maxScrapeResults: 5
2779
3464
  * });
2780
3465
  * console.log(result.data.scrapedJobIds);
2781
3466
  * ```
2782
3467
  */
2783
- andScrape(query: string, options?: Omit<SearchOptions, "query" | "autoScrape">): Promise<SearchJobResponse>;
3468
+ andScrape(query: string, options?: Omit<SearchOptions, "query" | "scrapeResults">): Promise<SearchJobResponse>;
2784
3469
  /**
2785
3470
  * Check search API status
2786
3471
  * Returns availability and features info
@@ -3056,6 +3741,9 @@ declare class WebhooksResource {
3056
3741
  * await client.extract.create({ url: "...", schema: {...} });
3057
3742
  * await client.dork.create({ query: "..." });
3058
3743
  *
3744
+ * // Utilities
3745
+ * await client.convert.htmlToMarkdown({ html: "..." });
3746
+ *
3059
3747
  * // Status & polling
3060
3748
  * await client.status.get(jobId);
3061
3749
  * await client.waitForResult(jobId);
@@ -3147,6 +3835,20 @@ declare class DealCrawl {
3147
3835
  * ```
3148
3836
  */
3149
3837
  readonly dork: DorkResource;
3838
+ /**
3839
+ * Convert resource - HTML to Markdown conversion
3840
+ *
3841
+ * @example
3842
+ * ```ts
3843
+ * const result = await client.convert.htmlToMarkdown({
3844
+ * html: "<h1>Title</h1><p>Content</p>",
3845
+ * baseUrl: "https://example.com",
3846
+ * options: { removeNoise: true }
3847
+ * });
3848
+ * console.log(result.data.markdown);
3849
+ * ```
3850
+ */
3851
+ readonly convert: ConvertResource;
3150
3852
  /**
3151
3853
  * Agent resource - AI-powered autonomous web navigation
3152
3854
  *
@@ -3228,6 +3930,80 @@ declare class DealCrawl {
3228
3930
  * ```
3229
3931
  */
3230
3932
  readonly account: AccountResource;
3933
+ /**
3934
+ * Screenshots resource - Screenshot signed URL management
3935
+ *
3936
+ * @example
3937
+ * ```ts
3938
+ * // Refresh a signed URL before expiration
3939
+ * const refreshed = await client.screenshots.refresh({
3940
+ * path: "job_abc123/1234567890_nanoid_example.png",
3941
+ * ttl: 604800 // 7 days
3942
+ * });
3943
+ *
3944
+ * // Get tier-specific TTL limits
3945
+ * const limits = await client.screenshots.getLimits();
3946
+ * console.log(limits.formattedLimits.max); // "7 days"
3947
+ * ```
3948
+ */
3949
+ readonly screenshots: ScreenshotsResource;
3950
+ /**
3951
+ * Auth resource - SSE (Server-Sent Events) authentication
3952
+ *
3953
+ * @example
3954
+ * ```ts
3955
+ * // Generate SSE token for browser EventSource
3956
+ * const { token, expiresAt } = await client.auth.generateSSEToken();
3957
+ *
3958
+ * // Use in browser
3959
+ * const eventSource = new EventSource(`/v1/events?token=${token}`);
3960
+ *
3961
+ * // Generate token for specific job
3962
+ * const jobToken = await client.auth.generateSSEToken({ jobId: "job_123" });
3963
+ *
3964
+ * // Check connection limits
3965
+ * const limits = await client.auth.getLimits();
3966
+ * console.log(`Available connections: ${limits.sse.available}`);
3967
+ * ```
3968
+ */
3969
+ readonly auth: AuthResource;
3970
+ /**
3971
+ * Events resource - Real-time SSE event streaming (Browser only)
3972
+ *
3973
+ * IMPORTANT: This resource only works in browsers. For Node.js, use polling via client.status.get()
3974
+ *
3975
+ * @example Browser Usage
3976
+ * ```ts
3977
+ * // 1. Generate SSE token
3978
+ * const { token } = await client.auth.generateSSEToken();
3979
+ *
3980
+ * // 2. Subscribe to all events
3981
+ * const eventSource = client.events.subscribe(token, {
3982
+ * onEvent: (event) => {
3983
+ * console.log('Event:', event.type, JSON.parse(event.data));
3984
+ * }
3985
+ * });
3986
+ *
3987
+ * // 3. Or subscribe to specific job
3988
+ * const jobEvents = client.events.subscribeToJob('job_123', token, {
3989
+ * onEvent: (event) => {
3990
+ * const data = JSON.parse(event.data);
3991
+ * console.log(`Progress: ${data.progress}%`);
3992
+ * }
3993
+ * });
3994
+ *
3995
+ * // 4. Listen for specific events
3996
+ * eventSource.addEventListener('job.completed', (event) => {
3997
+ * const data = JSON.parse(event.data);
3998
+ * console.log('Job completed!', data.summary);
3999
+ * eventSource.close();
4000
+ * });
4001
+ *
4002
+ * // 5. Clean up
4003
+ * eventSource.close();
4004
+ * ```
4005
+ */
4006
+ readonly events: EventsResource;
3231
4007
  /**
3232
4008
  * Create a new DealCrawl client
3233
4009
  *
@@ -3635,4 +4411,352 @@ declare const ERROR_MESSAGES: Record<ErrorCode, string>;
3635
4411
  */
3636
4412
  declare function getErrorMessage(code: ErrorCode): string;
3637
4413
 
3638
- export { ALL_API_KEY_SCOPES, type AccountInfoResponse, type AccountMetricsResponse, AccountResource, type ActionBaseOptions, type ActionInput, type AgentAction, type AgentActionType, type AgentCompletionReason, type AgentJobResponse, type AgentModel, type AgentOptions, AgentResource, type AgentResultResponse, type AgentStatusResponse, type AgentStepResponse, type ApiError, type ApiKeyInfo, type ApiKeyScope, type ApiResponse, type BatchScrapeDefaults, type BatchScrapeItem, type BatchScrapeOptions, type BatchScrapeResponse, type BatchScrapeResultItem, type BatchStatusResponse, type CancelJobResponse, type CheckpointInfo, type ClickAction, type ClientPreferences, type ClientStatsResponse, type CrawlAnalysisResponse, type CrawlError, type CrawlJobResponse, type CrawlMode, type CrawlOptions, type CrawlRecommendation, CrawlResource, type CrawlResult, type CrawlStats, type CrawlTemplate, type CrawlTemplateId, type CreateApiKeyOptions, type CreateKeyResponse, type CreateWebhookOptions, type CreateWebhookResponse, type CreatedApiKey, DEFAULT_API_KEY_SCOPES, DEFAULT_CONFIG, DataResource, DealCrawl, type DealCrawlConfig, DealCrawlError, type DealDetails, type DealItem, type DealMetrics, type DealScoreSummary, type DealSummary, type DealUpMetrics, type DealUpMetricsResponse, type DeleteKeyResponse, type DeleteWebhookResponse, type DiscountSignal, type DorkJobResponse, type DorkOptions, DorkResource, type DorkResult, ERROR_CODES, ERROR_MESSAGES, type EngineSelection, type EngineType, type ErrorCode, type ExportDealsOptions, type ExportFormat, type ExportJobsOptions, type ExtractJobResponse, type ExtractModel, type ExtractOptions, ExtractResource, type ExtractedDeal, type ExtractionErrorDetails, type FallbackConfig, type FallbackMetadata, type FallbackResult, type FallbackSource, type GetApiKeyStatsOptions, type GetDealsOptions, type HoverAction, type JobDealsResponse, type JobMetricsResponse, type JobResponse, type JobStatus, type JobStatusFilter, type JobStatusResponse, type JobSummary, type JobTypeFilter, type KeyStatsResponse, KeysResource, type ListApiKeysOptions, type ListDealsOptions, type ListDealsResponse, type ListJobsOptions, type ListJobsResponse, type ListKeysResponse, type ListWebhooksResponse, type PaginatedResponse, type PaginationInfo, type ParsedPage, type PreferencesResponse, type PressAction, type PriceSignal, type PricingInfo, type ProductCategory, type ProductInfo, type RateLimitInfo, type RecommendationsResponse, type RequestContext, type ResumeJobResponse, type RevokeApiKeyOptions, type RotateApiKeyOptions, type RotateKeyResponse, type ScrapeJobResponse, type ScrapeOptions, ScrapeResource, type ScrapeResult, type ScreenshotAction, type ScreenshotAgentAction, type ScreenshotOptions, type ScreenshotResult, type ScrollAction, type SearchAiModel, type SearchAiProvider, type SearchData, type SearchDateRange, type SearchFilters, type SearchJobResponse, type SearchOptions, SearchResource, type SearchResultItem, type SearchStatusResponse, type SelectAction, type ClickAction$1 as SharedClickAction, type HoverAction$1 as SharedHoverAction, type PressAction$1 as SharedPressAction, type ScrollAction$1 as SharedScrollAction, type SelectAction$1 as SharedSelectAction, type WaitAction$1 as SharedWaitAction, type WriteAction$1 as SharedWriteAction, type Signal, type SortOrder, StatusResource, type TestWebhookResponse, type UpdatePreferencesOptions, type UpdatePreferencesResponse, type UpdateWebhookOptions, type UpdateWebhookResponse, type UrgencyLevel, type UsageStats, type ValidationError, type ValidationResult, type WaitAction, type WaitOptions, type WaitResult, type WebhookEvent, type WebhookItem, WebhooksResource, type WriteAction, DealCrawl as default, getErrorMessage, pollUntil, waitForAll, waitForAny, waitForResult };
4414
+ /**
4415
+ * Advanced Job Status System
4416
+ *
4417
+ * Production-grade status model with:
4418
+ * - Primary status (lifecycle stage)
4419
+ * - Sub-status (detailed state within stage)
4420
+ * - Reason codes (why this state was reached)
4421
+ * - State machine transitions (valid state changes)
4422
+ *
4423
+ * @module job-status
4424
+ */
4425
+ /**
4426
+ * Primary job status - represents the main lifecycle stage
4427
+ */
4428
+ type JobStatus = "queued" | "scheduled" | "priority_queued" | "initializing" | "active" | "processing" | "extracting" | "uploading" | "finalizing" | "retrying" | "backing_off" | "paused" | "pausing" | "resuming" | "completed" | "partial" | "failed" | "timeout" | "cancelled" | "rejected" | "stale" | "zombie" | "stuck";
4429
+ /**
4430
+ * Sub-status provides granular detail within a primary status
4431
+ */
4432
+ type JobSubStatus = "waiting_for_worker" | "waiting_for_rate_limit" | "waiting_for_quota" | "waiting_for_retry" | "waiting_for_dependency" | "launching_browser" | "establishing_connection" | "loading_config" | "validating_input" | "fetching_page" | "executing_javascript" | "waiting_for_render" | "detecting_signals" | "crawling_links" | "capturing_screenshot" | "calling_llm" | "parsing_response" | "validating_extraction" | "uploading_screenshot" | "uploading_artifacts" | "generating_signed_url" | "saving_to_database" | "cleaning_resources" | "sending_webhook" | "syncing_to_dealup" | "exponential_backoff" | "rate_limit_backoff" | "circuit_breaker_open" | "network_error" | "timeout_error" | "bot_detected" | "captcha_detected" | "auth_required" | "page_not_found" | "server_error" | "parse_error" | "extraction_failed" | "quota_exceeded" | "invalid_input" | "resource_unavailable" | "no_heartbeat" | "progress_stalled" | "worker_lost";
4433
+ /**
4434
+ * Reason codes - why a status/substatus was reached
4435
+ */
4436
+ type StatusReason = "job_completed_successfully" | "partial_results_available" | "added_to_queue" | "priority_elevated" | "rate_limit_applied" | "quota_check_pending" | "transient_network_error" | "rate_limit_exceeded" | "timeout_exceeded" | "worker_crashed" | "max_retries_not_reached" | "max_retries_exceeded" | "permanent_error" | "blocked_by_site" | "captcha_required" | "invalid_url_format" | "ssrf_detected" | "quota_exhausted" | "tier_restriction" | "resource_not_available" | "cancelled_by_user" | "cancelled_by_system" | "duplicate_detected" | "policy_violation" | "heartbeat_timeout" | "progress_timeout" | "worker_unresponsive";
4437
+
4438
+ /**
4439
+ * Professional structured logger for DealCrawl
4440
+ * - Environment-aware (verbose in dev, minimal in prod)
4441
+ * - Structured JSON output in production
4442
+ * - Colored console output in development
4443
+ * - Correlation ID support for request tracing
4444
+ */
4445
+ type LogLevel = "debug" | "info" | "warn" | "error";
4446
+
4447
+ /**
4448
+ * SSE (Server-Sent Events) Event Types
4449
+ *
4450
+ * Defines all events that can be pushed to clients via SSE.
4451
+ *
4452
+ * @module sse-events
4453
+ */
4454
+
4455
+ /**
4456
+ * SSE event types - used as the "event:" field in SSE messages
4457
+ */
4458
+ type SSEEventType = "job.created" | "job.queued" | "job.started" | "job.status" | "job.progress" | "job.completed" | "job.failed" | "job.cancelled" | "job.paused" | "job.resumed" | "job.log" | "job.metric" | "job.alert" | "job.checkpoint" | "deal.found" | "deal.validated" | "ping" | "connection.open" | "connection.close" | "error";
4459
+ /**
4460
+ * Base SSE event structure - all events extend this
4461
+ */
4462
+ interface BaseSSEEvent {
4463
+ /** Event type (used in SSE "event:" field) */
4464
+ type: SSEEventType;
4465
+ /** Unique event ID (for Last-Event-ID replay) */
4466
+ id: string;
4467
+ /** When this event was generated */
4468
+ timestamp: string;
4469
+ /** Job ID this event relates to (null for system events) */
4470
+ jobId: string | null;
4471
+ /** Client ID (for server-side filtering) */
4472
+ clientId: string;
4473
+ }
4474
+ /**
4475
+ * Job created event - sent when job is first created
4476
+ */
4477
+ interface JobCreatedEvent extends BaseSSEEvent {
4478
+ type: "job.created";
4479
+ jobId: string;
4480
+ data: {
4481
+ jobType: "scrape" | "crawl" | "dork" | "extract" | "agent";
4482
+ priority?: "high" | "medium" | "low";
4483
+ template?: string;
4484
+ };
4485
+ }
4486
+ /**
4487
+ * Job queued event - sent when job enters queue
4488
+ */
4489
+ interface JobQueuedEvent extends BaseSSEEvent {
4490
+ type: "job.queued";
4491
+ jobId: string;
4492
+ data: {
4493
+ queue: string;
4494
+ position?: number;
4495
+ estimatedWaitMs?: number;
4496
+ };
4497
+ }
4498
+ /**
4499
+ * Job started event - sent when worker picks up job
4500
+ */
4501
+ interface JobStartedEvent extends BaseSSEEvent {
4502
+ type: "job.started";
4503
+ jobId: string;
4504
+ data: {
4505
+ workerId?: string;
4506
+ startedAt: string;
4507
+ };
4508
+ }
4509
+ /**
4510
+ * Job status change event - sent on every status transition
4511
+ */
4512
+ interface JobStatusEvent extends BaseSSEEvent {
4513
+ type: "job.status";
4514
+ jobId: string;
4515
+ data: {
4516
+ status: JobStatus;
4517
+ subStatus?: JobSubStatus;
4518
+ reason?: StatusReason;
4519
+ message?: string;
4520
+ previousStatus?: JobStatus;
4521
+ progress: number;
4522
+ };
4523
+ }
4524
+ /**
4525
+ * Job progress event - sent periodically during processing
4526
+ */
4527
+ interface JobProgressEvent extends BaseSSEEvent {
4528
+ type: "job.progress";
4529
+ jobId: string;
4530
+ data: {
4531
+ progress: number;
4532
+ status: JobStatus;
4533
+ subStatus?: JobSubStatus;
4534
+ message?: string;
4535
+ eta?: {
4536
+ remainingMs: number;
4537
+ remainingFormatted: string;
4538
+ };
4539
+ stats?: {
4540
+ pagesProcessed?: number;
4541
+ dealsFound?: number;
4542
+ signalsDetected?: number;
4543
+ };
4544
+ };
4545
+ }
4546
+ /**
4547
+ * Job completed event - sent when job finishes successfully
4548
+ */
4549
+ interface JobCompletedEvent extends BaseSSEEvent {
4550
+ type: "job.completed";
4551
+ jobId: string;
4552
+ data: {
4553
+ statusUrl: string;
4554
+ completedAt: string;
4555
+ durationMs: number;
4556
+ summary: {
4557
+ success: boolean;
4558
+ pagesProcessed?: number;
4559
+ dealsFound?: number;
4560
+ signalsDetected?: number;
4561
+ hasScreenshot?: boolean;
4562
+ };
4563
+ result?: {
4564
+ dealScore?: number;
4565
+ topSignals?: string[];
4566
+ dealTitle?: string;
4567
+ };
4568
+ };
4569
+ }
4570
+ /**
4571
+ * Job failed event - sent when job fails
4572
+ */
4573
+ interface JobFailedEvent extends BaseSSEEvent {
4574
+ type: "job.failed";
4575
+ jobId: string;
4576
+ data: {
4577
+ statusUrl: string;
4578
+ failedAt: string;
4579
+ durationMs: number;
4580
+ error: {
4581
+ code: string;
4582
+ message: string;
4583
+ recoverable: boolean;
4584
+ };
4585
+ retry?: {
4586
+ attempt: number;
4587
+ maxAttempts: number;
4588
+ willRetry: boolean;
4589
+ nextRetryAt?: string;
4590
+ };
4591
+ };
4592
+ }
4593
+ /**
4594
+ * Job cancelled event - sent when job is cancelled
4595
+ */
4596
+ interface JobCancelledEvent extends BaseSSEEvent {
4597
+ type: "job.cancelled";
4598
+ jobId: string;
4599
+ data: {
4600
+ cancelledAt: string;
4601
+ cancelledBy: "user" | "system";
4602
+ reason?: string;
4603
+ };
4604
+ }
4605
+ /**
4606
+ * Job paused event
4607
+ */
4608
+ interface JobPausedEvent extends BaseSSEEvent {
4609
+ type: "job.paused";
4610
+ jobId: string;
4611
+ data: {
4612
+ pausedAt: string;
4613
+ checkpoint?: {
4614
+ lastProcessedUrl: string;
4615
+ urlsProcessed: number;
4616
+ urlsRemaining: number;
4617
+ };
4618
+ };
4619
+ }
4620
+ /**
4621
+ * Job resumed event
4622
+ */
4623
+ interface JobResumedEvent extends BaseSSEEvent {
4624
+ type: "job.resumed";
4625
+ jobId: string;
4626
+ data: {
4627
+ resumedAt: string;
4628
+ fromCheckpoint: boolean;
4629
+ };
4630
+ }
4631
+ /**
4632
+ * Job log event - sent for important log messages
4633
+ */
4634
+ interface JobLogEvent extends BaseSSEEvent {
4635
+ type: "job.log";
4636
+ jobId: string;
4637
+ data: {
4638
+ level: LogLevel;
4639
+ message: string;
4640
+ metadata?: Record<string, unknown>;
4641
+ };
4642
+ }
4643
+ /**
4644
+ * Job metric event - sent for performance/business metrics
4645
+ */
4646
+ interface JobMetricEvent extends BaseSSEEvent {
4647
+ type: "job.metric";
4648
+ jobId: string;
4649
+ data: {
4650
+ metric: string;
4651
+ value: number;
4652
+ unit?: string;
4653
+ };
4654
+ }
4655
+ /**
4656
+ * Job alert event - sent for important alerts (quota warnings, etc.)
4657
+ */
4658
+ interface JobAlertEvent extends BaseSSEEvent {
4659
+ type: "job.alert";
4660
+ jobId: string;
4661
+ data: {
4662
+ severity: "info" | "warning" | "error" | "critical";
4663
+ title: string;
4664
+ message: string;
4665
+ actionRequired?: boolean;
4666
+ };
4667
+ }
4668
+ /**
4669
+ * Job checkpoint event - sent when checkpoint is saved
4670
+ */
4671
+ interface JobCheckpointEvent extends BaseSSEEvent {
4672
+ type: "job.checkpoint";
4673
+ jobId: string;
4674
+ data: {
4675
+ savedAt: string;
4676
+ lastProcessedUrl: string;
4677
+ urlsProcessed: number;
4678
+ urlsRemaining: number;
4679
+ canResume: boolean;
4680
+ };
4681
+ }
4682
+ /**
4683
+ * Deal found event - sent when a deal is detected
4684
+ */
4685
+ interface DealFoundEvent extends BaseSSEEvent {
4686
+ type: "deal.found";
4687
+ jobId: string;
4688
+ data: {
4689
+ dealId?: string;
4690
+ url: string;
4691
+ title: string;
4692
+ score: number;
4693
+ price?: number;
4694
+ discount?: number;
4695
+ category?: string;
4696
+ signals: string[];
4697
+ };
4698
+ }
4699
+ /**
4700
+ * Deal validated event - sent when deal is validated/scored
4701
+ */
4702
+ interface DealValidatedEvent extends BaseSSEEvent {
4703
+ type: "deal.validated";
4704
+ jobId: string;
4705
+ data: {
4706
+ dealId: string;
4707
+ score: number;
4708
+ confidence: number;
4709
+ accepted: boolean;
4710
+ rejectionReason?: string;
4711
+ };
4712
+ }
4713
+ /**
4714
+ * Ping event - keepalive to prevent connection timeout
4715
+ */
4716
+ interface PingEvent extends BaseSSEEvent {
4717
+ type: "ping";
4718
+ jobId: null;
4719
+ data: {
4720
+ serverTime: string;
4721
+ };
4722
+ }
4723
+ /**
4724
+ * Connection open event - sent when SSE connection established
4725
+ */
4726
+ interface ConnectionOpenEvent extends BaseSSEEvent {
4727
+ type: "connection.open";
4728
+ jobId: null;
4729
+ data: {
4730
+ clientId: string;
4731
+ reconnected: boolean;
4732
+ lastEventId?: string;
4733
+ };
4734
+ }
4735
+ /**
4736
+ * Connection close event - sent before connection closes
4737
+ */
4738
+ interface ConnectionCloseEvent extends BaseSSEEvent {
4739
+ type: "connection.close";
4740
+ jobId: null;
4741
+ data: {
4742
+ reason: string;
4743
+ };
4744
+ }
4745
+ /**
4746
+ * Error event - sent when an error occurs
4747
+ */
4748
+ interface ErrorEvent extends BaseSSEEvent {
4749
+ type: "error";
4750
+ jobId: string | null;
4751
+ data: {
4752
+ code: string;
4753
+ message: string;
4754
+ recoverable: boolean;
4755
+ };
4756
+ }
4757
+ /**
4758
+ * All possible SSE events (discriminated union)
4759
+ */
4760
+ type SSEEvent = JobCreatedEvent | JobQueuedEvent | JobStartedEvent | JobStatusEvent | JobProgressEvent | JobCompletedEvent | JobFailedEvent | JobCancelledEvent | JobPausedEvent | JobResumedEvent | JobLogEvent | JobMetricEvent | JobAlertEvent | JobCheckpointEvent | DealFoundEvent | DealValidatedEvent | PingEvent | ConnectionOpenEvent | ConnectionCloseEvent | ErrorEvent;
4761
+
4762
+ export { ALL_API_KEY_SCOPES, type AccountInfoResponse, type AccountMetricsResponse, AccountResource, type ActionBaseOptions, type ActionInput, type AgentAction, type AgentActionType, type AgentCompletionReason, type AgentJobResponse, type AgentModel, type AgentOptions, AgentResource, type AgentResultResponse, type AgentStatusResponse, type AgentStepResponse, type ApiError, type ApiKeyInfo, type ApiKeyScope, type ApiResponse, AuthResource, type BaseSSEEvent, type BatchScrapeDefaults, type BatchScrapeItem, type BatchScrapeOptions, type BatchScrapeResponse, type BatchScrapeResultItem, type BatchStatusResponse, type CancelJobResponse, type CheckpointInfo, type ClickAction, type ClientPreferences, type ClientStatsResponse, type ConnectionCloseEvent, type ConnectionOpenEvent, type ConversionMetadata, type ConvertOptions, ConvertResource, type ConvertResponse, type CrawlAnalysisResponse, type CrawlError, type CrawlJobResponse, type CrawlMode, type CrawlOptions, type CrawlRecommendation, CrawlResource, type CrawlResult, type CrawlStats, type CrawlTemplate, type CrawlTemplateId, type CreateApiKeyOptions, type CreateKeyResponse, type CreateWebhookOptions, type CreateWebhookResponse, type CreatedApiKey, DEFAULT_API_KEY_SCOPES, DEFAULT_CONFIG, DataResource, DealCrawl, type DealCrawlConfig, DealCrawlError, type DealDetails, type DealFoundEvent, type DealItem, type DealMetrics, type DealScoreSummary, type DealSummary, type DealUpMetrics, type DealUpMetricsResponse, type DealValidatedEvent, type DeleteKeyResponse, type DeleteWebhookResponse, type DiscountSignal, type DorkJobResponse, type DorkOptions, DorkResource, type DorkResult, ERROR_CODES, ERROR_MESSAGES, type EngineSelection, type EngineType, type ErrorCode, type ErrorEvent, EventsResource, type ExportDealsOptions, type ExportFormat, type ExportJobsOptions, type ExtractJobResponse, type ExtractModel, type ExtractOptions, ExtractResource, type ExtractedDeal, type ExtractionErrorDetails, type FallbackConfig, type FallbackMetadata, type FallbackResult, type FallbackSource, type GenerateSSETokenOptions, type GetApiKeyStatsOptions, type GetDealsOptions, type HoverAction, type JobAlertEvent, type JobCancelledEvent, type JobCheckpointEvent, type JobCompletedEvent, type JobCreatedEvent, type JobDealsResponse, type JobFailedEvent, type JobLogEvent, type JobMetricEvent, type JobMetricsResponse, type JobPausedEvent, type JobProgressEvent, type JobQueuedEvent, type JobResponse, type JobResumedEvent, type JobStartedEvent, type JobStatus$1 as JobStatus, type JobStatusEvent, type JobStatusFilter, type JobStatusResponse, type JobSummary, type JobTypeFilter, type KeyStatsResponse, KeysResource, type ListApiKeysOptions, type ListDealsOptions, type ListDealsResponse, type ListJobsOptions, type ListJobsResponse, type ListKeysResponse, type ListWebhooksResponse, type PaginatedResponse, type PaginationInfo, type ParsedPage, type PingEvent, type PreferencesResponse, type PressAction, type PriceSignal, type PricingInfo, type ProductCategory, type ProductInfo, type RateLimitInfo, type RecommendationsResponse, type RefreshScreenshotOptions, type RequestContext, type ResumeJobResponse, type RevokeApiKeyOptions, type RotateApiKeyOptions, type RotateKeyResponse, type SSEConnectionLimitsResponse, type SSEEvent, type SSEEventType, type SSESubscribeOptions, type SSETokenResponse, type ScrapeJobResponse, type ScrapeOptions, ScrapeResource, type ScrapeResult, type ScreenshotAction, type ScreenshotAgentAction, type ScreenshotLimitsResponse, type ScreenshotOptions, type ScreenshotRefreshResponse, type ScreenshotResult, ScreenshotsResource, type ScrollAction, type SearchAiModel, type SearchAiProvider, type SearchData, type SearchDateRange, type SearchFilters, type SearchJobResponse, type SearchOptions, SearchResource, type SearchResultItem, type SearchStatusResponse, type SelectAction, type ClickAction$1 as SharedClickAction, type HoverAction$1 as SharedHoverAction, type PressAction$1 as SharedPressAction, type ScrollAction$1 as SharedScrollAction, type SelectAction$1 as SharedSelectAction, type WaitAction$1 as SharedWaitAction, type WriteAction$1 as SharedWriteAction, type Signal, type SortOrder, StatusResource, type TestWebhookResponse, type UpdatePreferencesOptions, type UpdatePreferencesResponse, type UpdateWebhookOptions, type UpdateWebhookResponse, type UrgencyLevel, type UsageStats, type ValidationError, type ValidationResult, type WaitAction, type WaitOptions, type WaitResult, type WebhookEvent, type WebhookItem, WebhooksResource, type WriteAction, DealCrawl as default, getErrorMessage, pollUntil, waitForAll, waitForAny, waitForResult };