@dealcrawl/sdk 2.10.0 → 2.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -19,7 +19,7 @@ interface ResponseMeta {
19
19
  timestamp: string;
20
20
  duration?: number;
21
21
  }
22
- type JobStatus = "pending" | "active" | "completed" | "failed" | "delayed" | "paused";
22
+ type JobStatus$1 = "pending" | "active" | "completed" | "failed" | "delayed" | "paused";
23
23
  interface PaginatedResponse<T> {
24
24
  data: T[];
25
25
  pagination: {
@@ -130,10 +130,19 @@ interface DealScoreSummary {
130
130
  };
131
131
  }
132
132
  interface ScreenshotResult {
133
+ /** Screenshot URL - private signed URL (default) or public URL (Enterprise opt-in) */
133
134
  url: string;
135
+ /** Whether this is a public URL (false = private signed URL with TTL) */
136
+ isPublic: boolean;
137
+ /** Expiration timestamp for signed URLs (ISO 8601, only for private URLs) */
138
+ expiresAt?: string;
139
+ /** Screenshot width in pixels */
134
140
  width: number;
141
+ /** Screenshot height in pixels */
135
142
  height: number;
143
+ /** Image format */
136
144
  format: "png" | "jpeg" | "webp";
145
+ /** File size in bytes */
137
146
  sizeBytes: number;
138
147
  }
139
148
  interface AIExtraction {
@@ -438,7 +447,7 @@ interface JobResponse {
438
447
  /** Unique job identifier */
439
448
  jobId: string;
440
449
  /** Current job status */
441
- status: JobStatus;
450
+ status: JobStatus$1;
442
451
  /** URL to check job status */
443
452
  statusUrl: string;
444
453
  /** Estimated time to completion */
@@ -468,6 +477,15 @@ interface BatchScrapeResultItem {
468
477
  /** Error message if job creation failed */
469
478
  error?: string;
470
479
  }
480
+ /** Invalid URL entry (returned when ignoreInvalidURLs=true) */
481
+ interface InvalidURLEntry {
482
+ /** The invalid URL */
483
+ url: string;
484
+ /** Reference ID if provided */
485
+ ref?: string;
486
+ /** Reason for rejection */
487
+ reason: string;
488
+ }
471
489
  /** Batch scrape response */
472
490
  interface BatchScrapeResponse {
473
491
  /** Batch ID for tracking */
@@ -482,6 +500,11 @@ interface BatchScrapeResponse {
482
500
  results: BatchScrapeResultItem[];
483
501
  /** Timestamp when batch was created */
484
502
  createdAt: string;
503
+ /**
504
+ * Invalid URLs that were skipped (only present when ignoreInvalidURLs=true)
505
+ * Firecrawl-compatible field
506
+ */
507
+ invalidURLs?: InvalidURLEntry[];
485
508
  }
486
509
  /** Batch status response */
487
510
  interface BatchStatusResponse {
@@ -502,7 +525,7 @@ interface BatchStatusResponse {
502
525
  /** Individual job statuses */
503
526
  jobs: Array<{
504
527
  id: string;
505
- status: JobStatus;
528
+ status: JobStatus$1;
506
529
  result?: unknown;
507
530
  error?: string;
508
531
  }>;
@@ -612,7 +635,7 @@ interface CheckpointInfo {
612
635
  /** Job status response */
613
636
  interface JobStatusResponse {
614
637
  jobId: string;
615
- status: JobStatus;
638
+ status: JobStatus$1;
616
639
  progress?: number;
617
640
  createdAt: string;
618
641
  completedAt?: string;
@@ -652,7 +675,7 @@ interface ResumeJobResponse {
652
675
  /** Job metrics response */
653
676
  interface JobMetricsResponse {
654
677
  jobId: string;
655
- status: JobStatus;
678
+ status: JobStatus$1;
656
679
  metrics: {
657
680
  totalUrls: number;
658
681
  successfulUrls: number;
@@ -671,7 +694,7 @@ interface CancelJobResponse {
671
694
  interface JobSummary {
672
695
  id: string;
673
696
  type: "scrape" | "crawl" | "dork" | "extract";
674
- status: JobStatus;
697
+ status: JobStatus$1;
675
698
  input: Record<string, unknown>;
676
699
  dealsFound?: number;
677
700
  avgDealScore?: number;
@@ -964,7 +987,7 @@ interface AgentStepResponse {
964
987
  currentUrl?: string;
965
988
  }
966
989
  /** Agent completion reason */
967
- type AgentCompletionReason = "goal_achieved" | "max_steps_reached" | "stuck" | "error" | "cancelled";
990
+ type AgentCompletionReason = "goal_achieved" | "max_steps_reached" | "stuck" | "error";
968
991
  /** Agent result (from completed job) */
969
992
  interface AgentResultResponse {
970
993
  /** Extracted/collected data (matches provided schema if given) */
@@ -1013,6 +1036,113 @@ interface SchemaGenerationResponse {
1013
1036
  /** Confidence score (0-1) in the generated schema */
1014
1037
  confidence: number;
1015
1038
  }
1039
+ /** SSE token response */
1040
+ interface SSETokenResponse {
1041
+ /** Short-lived SSE authentication token */
1042
+ token: string;
1043
+ /** Expiration timestamp (ISO 8601) */
1044
+ expiresAt: string;
1045
+ /** TTL in seconds (default: 300 = 5 minutes) */
1046
+ expiresIn: number;
1047
+ /** Token type identifier */
1048
+ type: "sse_token";
1049
+ /** Usage instructions for the token */
1050
+ usage: string;
1051
+ }
1052
+ /** SSE connection limits response */
1053
+ interface SSEConnectionLimitsResponse {
1054
+ /** Client tier */
1055
+ tier: "free" | "pro" | "enterprise";
1056
+ /** SSE connection limits */
1057
+ sse: {
1058
+ /** Maximum concurrent connections allowed */
1059
+ maxConnections: number;
1060
+ /** Current active connections */
1061
+ currentConnections: number;
1062
+ /** Available connection slots */
1063
+ available: number;
1064
+ };
1065
+ }
1066
+ /** Tier-specific TTL limits */
1067
+ interface TtlLimits {
1068
+ /** Minimum TTL in seconds */
1069
+ min: number;
1070
+ /** Maximum TTL in seconds */
1071
+ max: number;
1072
+ /** Default TTL in seconds */
1073
+ default: number;
1074
+ }
1075
+ /** Formatted TTL limits (human-readable) */
1076
+ interface FormattedTtlLimits {
1077
+ /** Minimum TTL formatted (e.g., "1 hour") */
1078
+ min: string;
1079
+ /** Maximum TTL formatted (e.g., "30 days") */
1080
+ max: string;
1081
+ /** Default TTL formatted (e.g., "7 days") */
1082
+ default: string;
1083
+ }
1084
+ /** Screenshot signed URL refresh response */
1085
+ interface ScreenshotRefreshResponse {
1086
+ /** Success status */
1087
+ success: boolean;
1088
+ /** New signed URL */
1089
+ url: string;
1090
+ /** Expiration timestamp (ISO 8601) */
1091
+ expiresAt: string;
1092
+ /** TTL in seconds */
1093
+ ttl: number;
1094
+ /** Tier-specific limits for reference */
1095
+ tierLimits: TtlLimits;
1096
+ /** Additional metadata */
1097
+ metadata: {
1098
+ /** File path in storage */
1099
+ path: string;
1100
+ /** Bucket name */
1101
+ bucket: string;
1102
+ /** Timestamp when URL was refreshed */
1103
+ refreshedAt: string;
1104
+ };
1105
+ }
1106
+ /** Screenshot TTL limits response */
1107
+ interface ScreenshotLimitsResponse {
1108
+ /** Client tier */
1109
+ tier: "free" | "pro" | "enterprise";
1110
+ /** TTL limits in seconds */
1111
+ limits: TtlLimits;
1112
+ /** Human-readable formatted limits */
1113
+ formattedLimits: FormattedTtlLimits;
1114
+ }
1115
+ /**
1116
+ * Metadata extracted during HTML to Markdown conversion
1117
+ */
1118
+ interface ConversionMetadata {
1119
+ /** Page title if found */
1120
+ title?: string;
1121
+ /** Word count in resulting markdown */
1122
+ wordCount: number;
1123
+ /** Number of links in markdown */
1124
+ linkCount: number;
1125
+ /** Number of images in markdown */
1126
+ imageCount: number;
1127
+ /** Conversion duration in milliseconds */
1128
+ conversionTimeMs: number;
1129
+ }
1130
+ /**
1131
+ * HTML to Markdown conversion response
1132
+ */
1133
+ interface ConvertResponse {
1134
+ /** Success status */
1135
+ success: boolean;
1136
+ /** Conversion result data */
1137
+ data: {
1138
+ /** Generated markdown content */
1139
+ markdown: string;
1140
+ /** Extraction metadata */
1141
+ metadata: ConversionMetadata;
1142
+ /** Warnings (truncation, excluded elements, etc.) */
1143
+ warnings?: string[];
1144
+ };
1145
+ }
1016
1146
 
1017
1147
  /**
1018
1148
  * Polling Utilities
@@ -1092,10 +1222,19 @@ interface ScreenshotOptions {
1092
1222
  /**
1093
1223
  * Request public URL (Enterprise only, default: false)
1094
1224
  * SECURITY WARNING: Public URLs can expose personal data, copyrighted content, and tokens
1095
- * By default, screenshots use private signed URLs with 7-day expiration
1225
+ * By default, screenshots use private signed URLs with tier-specific expiration
1096
1226
  * Requires Enterprise tier
1097
1227
  */
1098
1228
  publicUrl?: boolean;
1229
+ /**
1230
+ * TTL for signed URLs in seconds (only for private screenshots)
1231
+ * Tier limits:
1232
+ * - Free: 3600 (1h) - 86400 (24h), default: 86400
1233
+ * - Pro: 3600 (1h) - 604800 (7d), default: 604800
1234
+ * - Enterprise: 3600 (1h) - 2592000 (30d), default: 604800
1235
+ * If not specified, uses tier-specific default
1236
+ */
1237
+ signedUrlTtl?: number;
1099
1238
  }
1100
1239
  /** Options for scraping a single page */
1101
1240
  interface ScrapeOptions {
@@ -1158,8 +1297,6 @@ interface BatchScrapeItem {
1158
1297
  outputMarkdown?: boolean;
1159
1298
  /** Override markdownBaseUrl for this URL */
1160
1299
  markdownBaseUrl?: string;
1161
- /** Override actions for this URL */
1162
- actions?: ActionInput[];
1163
1300
  }
1164
1301
  /** Default options applied to all URLs in a batch */
1165
1302
  interface BatchScrapeDefaults {
@@ -1195,8 +1332,6 @@ interface BatchScrapeDefaults {
1195
1332
  outputMarkdown?: boolean;
1196
1333
  /** Base URL for resolving relative URLs in markdown */
1197
1334
  markdownBaseUrl?: string;
1198
- /** Browser actions to execute before scraping */
1199
- actions?: ActionInput[];
1200
1335
  }
1201
1336
  /** Options for batch scraping multiple URLs */
1202
1337
  interface BatchScrapeOptions {
@@ -1208,8 +1343,13 @@ interface BatchScrapeOptions {
1208
1343
  webhookUrl?: string;
1209
1344
  /** Priority for batch jobs (1-10, higher = faster) */
1210
1345
  priority?: number;
1211
- /** Delay between job submissions in ms (0-5000) */
1212
- delay?: number;
1346
+ /** Delay between job submissions in ms (0-5000, default: 0) */
1347
+ delayMs?: number;
1348
+ /**
1349
+ * Continue processing even if some URLs are invalid (default: false)
1350
+ * Invalid URLs will be returned in the response with status: "failed"
1351
+ */
1352
+ ignoreInvalidURLs?: boolean;
1213
1353
  }
1214
1354
  /** AI provider for search optimization */
1215
1355
  type SearchAiProvider = "openai" | "anthropic";
@@ -1237,11 +1377,11 @@ interface SearchOptions {
1237
1377
  /** Search query (required) */
1238
1378
  query: string;
1239
1379
  /** Maximum number of results (1-100, default: 10) */
1240
- maxResults?: number;
1241
- /** Auto-scrape top results */
1242
- autoScrape?: boolean;
1243
- /** Number of results to auto-scrape (1-10, default: 5) */
1244
- autoScrapeLimit?: number;
1380
+ limit?: number;
1381
+ /** Auto-scrape search results (default: false) */
1382
+ scrapeResults?: boolean;
1383
+ /** Maximum results to auto-scrape when scrapeResults=true (1-10, default: 5) */
1384
+ maxScrapeResults?: number;
1245
1385
  /** Use AI to optimize the search query */
1246
1386
  useAiOptimization?: boolean;
1247
1387
  /** AI provider for query optimization */
@@ -1352,7 +1492,7 @@ interface CrawlTemplate {
1352
1492
  defaultOptions: Partial<CrawlOptions>;
1353
1493
  }
1354
1494
  /** LLM model options for extraction */
1355
- type ExtractModel = "gpt-4o-mini" | "gpt-4o" | "claude-3-haiku" | "claude-3-sonnet";
1495
+ type ExtractModel = "gpt-4o-mini" | "gpt-4o" | "gpt-4o-2024-11-20" | "o1-mini" | "claude-3-5-haiku-20241022" | "claude-3-5-sonnet-20241022" | "claude-3-opus-20240229";
1356
1496
  /** Options for structured data extraction */
1357
1497
  interface ExtractOptions {
1358
1498
  /** URL to extract data from (required) */
@@ -1513,7 +1653,7 @@ interface UpdateWebhookOptions {
1513
1653
  * API key scope - Must match @dealcrawl/shared/src/types/api-key.types.ts
1514
1654
  * These are the actual scopes enforced by the backend via requireScope() middleware
1515
1655
  */
1516
- type ApiKeyScope = "scrape" | "scrape:batch" | "search" | "crawl" | "dork" | "extract" | "agent" | "status" | "data:read" | "data:export" | "keys:manage" | "webhooks:manage";
1656
+ type ApiKeyScope = "scrape" | "crawl" | "dork" | "extract" | "agent" | "status" | "data:read" | "data:export" | "keys:manage" | "webhooks:manage";
1517
1657
  /**
1518
1658
  * All available scopes (for reference and validation)
1519
1659
  */
@@ -1691,6 +1831,71 @@ interface SchemaGenerationOptions {
1691
1831
  /** LLM provider for generation (default: openai) */
1692
1832
  model?: AgentModel;
1693
1833
  }
1834
+ /**
1835
+ * Options for subscribing to SSE events
1836
+ */
1837
+ interface SSESubscribeOptions {
1838
+ /**
1839
+ * Last event ID for reconnection/replay
1840
+ * If provided, server will replay missed events since this ID
1841
+ */
1842
+ lastEventId?: string;
1843
+ /**
1844
+ * Callback for received events
1845
+ * Called for each event received from the server
1846
+ */
1847
+ onEvent?: (event: MessageEvent) => void;
1848
+ /**
1849
+ * Callback for errors
1850
+ * Called when connection error occurs
1851
+ */
1852
+ onError?: (error: Error) => void;
1853
+ /**
1854
+ * Callback for connection open
1855
+ * Called when SSE connection is established
1856
+ */
1857
+ onOpen?: (event: Event) => void;
1858
+ /**
1859
+ * Auto-reconnect on connection loss (default: true)
1860
+ * If true, EventSource will automatically reconnect
1861
+ */
1862
+ autoReconnect?: boolean;
1863
+ /**
1864
+ * Reconnection delay in milliseconds (default: 3000)
1865
+ * How long to wait before attempting reconnection
1866
+ */
1867
+ reconnectDelay?: number;
1868
+ }
1869
+ /**
1870
+ * Options for HTML to Markdown conversion
1871
+ */
1872
+ interface ConvertOptions {
1873
+ /** HTML content to convert (required, max 10MB) */
1874
+ html: string;
1875
+ /** Base URL for resolving relative URLs */
1876
+ baseUrl?: string;
1877
+ /** Conversion options */
1878
+ options?: {
1879
+ /** Enable GFM tables (default: true) */
1880
+ gfmTables?: boolean;
1881
+ /** Enable GFM strikethrough (default: true) */
1882
+ gfmStrikethrough?: boolean;
1883
+ /** Enable GFM task lists (default: true) */
1884
+ gfmTaskLists?: boolean;
1885
+ /** Remove noise elements like nav, footer, ads (default: true) */
1886
+ removeNoise?: boolean;
1887
+ /** CSS selectors to exclude from conversion */
1888
+ excludeSelectors?: string[];
1889
+ /** Resolve relative URLs to absolute (default: true) */
1890
+ absoluteUrls?: boolean;
1891
+ /** Maximum output length in characters (0 = unlimited, max: 1,000,000) */
1892
+ maxLength?: number;
1893
+ /** Include images in output (default: true) */
1894
+ includeImages?: boolean;
1895
+ /** Include links in output (default: true) */
1896
+ includeLinks?: boolean;
1897
+ };
1898
+ }
1694
1899
 
1695
1900
  /**
1696
1901
  * Account Resource
@@ -2035,6 +2240,257 @@ declare class AgentResource {
2035
2240
  generateSchema(options: SchemaGenerationOptions): Promise<SchemaGenerationResponse>;
2036
2241
  }
2037
2242
 
2243
+ /**
2244
+ * Auth Resource
2245
+ * Handles SSE (Server-Sent Events) authentication
2246
+ */
2247
+
2248
+ /**
2249
+ * Options for generating an SSE token
2250
+ */
2251
+ interface GenerateSSETokenOptions {
2252
+ /**
2253
+ * Optional job ID to restrict token to a specific job
2254
+ * If provided, token can only be used for /v1/events/:jobId
2255
+ * If omitted, token can be used for /v1/events (all events)
2256
+ */
2257
+ jobId?: string;
2258
+ }
2259
+ /**
2260
+ * Auth resource class
2261
+ * Provides methods for SSE authentication
2262
+ *
2263
+ * @example
2264
+ * ```ts
2265
+ * // Generate SSE token for browser EventSource
2266
+ * const { token, expiresAt } = await client.auth.generateSSEToken();
2267
+ *
2268
+ * // Use in browser
2269
+ * const eventSource = new EventSource(`/v1/events?token=${token}`);
2270
+ *
2271
+ * // Token for specific job
2272
+ * const jobToken = await client.auth.generateSSEToken({ jobId: "job_123" });
2273
+ * const jobEvents = new EventSource(`/v1/events/job_123?token=${jobToken.token}`);
2274
+ * ```
2275
+ */
2276
+ declare class AuthResource {
2277
+ private ctx;
2278
+ constructor(ctx: RequestContext);
2279
+ /**
2280
+ * Generate SSE authentication token
2281
+ *
2282
+ * Required for browser-based SSE connections because EventSource API
2283
+ * doesn't support custom headers. Token is short-lived (5 minutes).
2284
+ *
2285
+ * Security:
2286
+ * - Requires valid API key (Bearer token)
2287
+ * - Token expires in 5 minutes
2288
+ * - Token can be restricted to specific job
2289
+ * - Token stored in Redis (revocable)
2290
+ *
2291
+ * @example
2292
+ * ```ts
2293
+ * // 1. Generate token
2294
+ * const { token, expiresAt } = await client.auth.generateSSEToken();
2295
+ * console.log(`Token expires at: ${expiresAt}`);
2296
+ *
2297
+ * // 2. Use in browser EventSource
2298
+ * const eventSource = new EventSource(`/v1/events?token=${token}`);
2299
+ *
2300
+ * eventSource.addEventListener('job.completed', (event) => {
2301
+ * const data = JSON.parse(event.data);
2302
+ * console.log('Job completed:', data);
2303
+ * });
2304
+ *
2305
+ * // 3. For specific job only
2306
+ * const jobToken = await client.auth.generateSSEToken({ jobId: "job_abc123" });
2307
+ * const jobEvents = new EventSource(`/v1/events/job_abc123?token=${jobToken.token}`);
2308
+ * ```
2309
+ */
2310
+ generateSSEToken(options?: GenerateSSETokenOptions): Promise<SSETokenResponse>;
2311
+ /**
2312
+ * Get SSE connection limits for current tier
2313
+ *
2314
+ * Shows how many concurrent SSE connections are allowed
2315
+ * and how many are currently active.
2316
+ *
2317
+ * Tier limits:
2318
+ * - Free: 2 concurrent connections
2319
+ * - Pro: 10 concurrent connections
2320
+ * - Enterprise: 50 concurrent connections
2321
+ *
2322
+ * @example
2323
+ * ```ts
2324
+ * const limits = await client.auth.getLimits();
2325
+ *
2326
+ * console.log(`Tier: ${limits.tier}`);
2327
+ * console.log(`Max connections: ${limits.sse.maxConnections}`);
2328
+ * console.log(`Current connections: ${limits.sse.currentConnections}`);
2329
+ * console.log(`Available: ${limits.sse.available}`);
2330
+ *
2331
+ * // Check before opening new connection
2332
+ * if (limits.sse.available > 0) {
2333
+ * const token = await client.auth.generateSSEToken();
2334
+ * const eventSource = new EventSource(`/v1/events?token=${token.token}`);
2335
+ * } else {
2336
+ * console.error('No available SSE connection slots');
2337
+ * }
2338
+ * ```
2339
+ */
2340
+ getLimits(): Promise<SSEConnectionLimitsResponse>;
2341
+ }
2342
+
2343
+ /**
2344
+ * Convert Resource
2345
+ * Handles HTML to Markdown conversion
2346
+ */
2347
+
2348
+ /**
2349
+ * Convert resource class
2350
+ * Provides methods for HTML to Markdown conversion
2351
+ *
2352
+ * @example
2353
+ * ```ts
2354
+ * // Convert HTML to markdown
2355
+ * const result = await client.convert.htmlToMarkdown({
2356
+ * html: "<h1>Title</h1><p>Content</p>",
2357
+ * baseUrl: "https://example.com",
2358
+ * options: {
2359
+ * gfmTables: true,
2360
+ * removeNoise: true
2361
+ * }
2362
+ * });
2363
+ *
2364
+ * console.log(result.data.markdown);
2365
+ * console.log(result.data.metadata.wordCount);
2366
+ * ```
2367
+ */
2368
+ declare class ConvertResource {
2369
+ private ctx;
2370
+ constructor(ctx: RequestContext);
2371
+ /**
2372
+ * Convert HTML to Markdown
2373
+ *
2374
+ * Transforms raw HTML content into clean, readable Markdown using GitHub Flavored Markdown (GFM).
2375
+ * Useful for:
2376
+ * - Converting scraped HTML to markdown for LLM processing
2377
+ * - Cleaning up messy HTML from web pages
2378
+ * - Extracting main content while removing noise (ads, nav, footer)
2379
+ * - Creating documentation from HTML sources
2380
+ *
2381
+ * Features:
2382
+ * - GFM table, strikethrough, and task list support
2383
+ * - Automatic noise removal (scripts, ads, navigation)
2384
+ * - Relative URL resolution
2385
+ * - Custom element exclusion via CSS selectors
2386
+ * - Output length limiting
2387
+ *
2388
+ * @param options - Conversion options
2389
+ * @returns Conversion result with markdown, metadata, and warnings
2390
+ *
2391
+ * @example Basic usage
2392
+ * ```ts
2393
+ * const result = await client.convert.htmlToMarkdown({
2394
+ * html: "<h1>Product</h1><p>Price: $99</p>"
2395
+ * });
2396
+ * console.log(result.data.markdown);
2397
+ * ```
2398
+ *
2399
+ * @example With all options
2400
+ * ```ts
2401
+ * const result = await client.convert.htmlToMarkdown({
2402
+ * html: htmlContent,
2403
+ * baseUrl: "https://shop.example.com",
2404
+ * options: {
2405
+ * gfmTables: true,
2406
+ * removeNoise: true,
2407
+ * excludeSelectors: [".advertisement", "#sidebar"],
2408
+ * absoluteUrls: true,
2409
+ * maxLength: 100000,
2410
+ * includeImages: true,
2411
+ * includeLinks: true
2412
+ * }
2413
+ * });
2414
+ *
2415
+ * // Check metadata
2416
+ * console.log(`Words: ${result.data.metadata.wordCount}`);
2417
+ * console.log(`Links: ${result.data.metadata.linkCount}`);
2418
+ * console.log(`Images: ${result.data.metadata.imageCount}`);
2419
+ * console.log(`Conversion time: ${result.data.metadata.conversionTimeMs}ms`);
2420
+ *
2421
+ * // Check for warnings
2422
+ * if (result.data.warnings?.length) {
2423
+ * console.warn("Conversion warnings:", result.data.warnings);
2424
+ * }
2425
+ * ```
2426
+ *
2427
+ * @example Converting scraped HTML
2428
+ * ```ts
2429
+ * // First scrape a page
2430
+ * const scrapeJob = await client.scrape.create({
2431
+ * url: "https://example.com/article"
2432
+ * });
2433
+ * const scrapeResult = await client.waitForResult(scrapeJob.jobId);
2434
+ *
2435
+ * // Then convert HTML to markdown
2436
+ * const markdown = await client.convert.htmlToMarkdown({
2437
+ * html: scrapeResult.data.html,
2438
+ * baseUrl: scrapeResult.data.url,
2439
+ * options: {
2440
+ * removeNoise: true,
2441
+ * onlyMainContent: true
2442
+ * }
2443
+ * });
2444
+ * ```
2445
+ */
2446
+ htmlToMarkdown(options: ConvertOptions): Promise<ConvertResponse>;
2447
+ /**
2448
+ * Alias for htmlToMarkdown() for convenience
2449
+ *
2450
+ * @example
2451
+ * ```ts
2452
+ * const result = await client.convert.toMarkdown({
2453
+ * html: "<h1>Hello</h1>"
2454
+ * });
2455
+ * ```
2456
+ */
2457
+ toMarkdown(options: ConvertOptions): Promise<ConvertResponse>;
2458
+ /**
2459
+ * Convert HTML with minimal options (just the HTML content)
2460
+ * Uses all default settings
2461
+ *
2462
+ * @param html - HTML content to convert
2463
+ * @param baseUrl - Optional base URL for resolving relative links
2464
+ * @returns Conversion result
2465
+ *
2466
+ * @example
2467
+ * ```ts
2468
+ * const result = await client.convert.quick(
2469
+ * "<h1>Title</h1><p>Content</p>",
2470
+ * "https://example.com"
2471
+ * );
2472
+ * console.log(result.data.markdown);
2473
+ * ```
2474
+ */
2475
+ quick(html: string, baseUrl?: string): Promise<ConvertResponse>;
2476
+ /**
2477
+ * Convert HTML with noise removal enabled
2478
+ * Removes navigation, footer, ads, scripts, and other clutter
2479
+ *
2480
+ * @param html - HTML content to convert
2481
+ * @param baseUrl - Optional base URL
2482
+ * @returns Conversion result with clean markdown
2483
+ *
2484
+ * @example
2485
+ * ```ts
2486
+ * // Extract just the main content from a messy page
2487
+ * const result = await client.convert.clean(messyHtml, "https://example.com");
2488
+ * console.log(result.data.markdown); // Clean, readable markdown
2489
+ * ```
2490
+ */
2491
+ clean(html: string, baseUrl?: string): Promise<ConvertResponse>;
2492
+ }
2493
+
2038
2494
  /**
2039
2495
  * Crawl Resource
2040
2496
  * Handles website crawling operations
@@ -2385,6 +2841,177 @@ declare class DorkResource {
2385
2841
  buildQuery(options: DorkOptions): string;
2386
2842
  }
2387
2843
 
2844
+ /**
2845
+ * Events Resource
2846
+ * Handles real-time SSE (Server-Sent Events) streaming
2847
+ */
2848
+
2849
+ /**
2850
+ * Events resource class
2851
+ * Provides methods for subscribing to real-time job events via SSE
2852
+ *
2853
+ * IMPORTANT: Browser Support Only
2854
+ * This resource uses the EventSource API which is only available in browsers.
2855
+ * For Node.js, use polling via client.status.get() instead.
2856
+ *
2857
+ * @example Browser Usage
2858
+ * ```ts
2859
+ * // 1. Generate SSE token (required for EventSource)
2860
+ * const { token } = await client.auth.generateSSEToken();
2861
+ *
2862
+ * // 2. Subscribe to all events
2863
+ * const eventSource = client.events.subscribe({
2864
+ * token,
2865
+ * onEvent: (event) => {
2866
+ * console.log('Event:', event.type, event.data);
2867
+ * }
2868
+ * });
2869
+ *
2870
+ * // 3. Or subscribe to specific job
2871
+ * const jobEvents = client.events.subscribeToJob('job_123', {
2872
+ * token,
2873
+ * onEvent: (event) => {
2874
+ * const data = JSON.parse(event.data);
2875
+ * console.log('Job event:', data);
2876
+ * }
2877
+ * });
2878
+ *
2879
+ * // 4. Clean up when done
2880
+ * eventSource.close();
2881
+ * ```
2882
+ */
2883
+ declare class EventsResource {
2884
+ private ctx;
2885
+ constructor(ctx: RequestContext);
2886
+ /**
2887
+ * Subscribe to all events for authenticated client
2888
+ *
2889
+ * Opens an SSE connection to receive real-time events for all jobs.
2890
+ * Requires an SSE token obtained via client.auth.generateSSEToken().
2891
+ *
2892
+ * Event Types:
2893
+ * - Job lifecycle: job.created, job.queued, job.started, job.progress,
2894
+ * job.completed, job.failed, job.cancelled
2895
+ * - Job details: job.log, job.metric, job.alert, job.checkpoint
2896
+ * - Deals: deal.found, deal.validated
2897
+ * - System: ping, connection.open, connection.close, error
2898
+ *
2899
+ * Features:
2900
+ * - Automatic reconnection on disconnect
2901
+ * - Event replay via Last-Event-ID
2902
+ * - Keepalive pings every 15 seconds
2903
+ * - Max connection time: 1 hour
2904
+ *
2905
+ * @param token - SSE authentication token from client.auth.generateSSEToken()
2906
+ * @param options - Subscription options (callbacks, reconnection settings)
2907
+ *
2908
+ * @example
2909
+ * ```ts
2910
+ * // Generate token
2911
+ * const { token } = await client.auth.generateSSEToken();
2912
+ *
2913
+ * // Subscribe with event handlers
2914
+ * const eventSource = client.events.subscribe(token, {
2915
+ * onEvent: (event) => {
2916
+ * // Handle all events
2917
+ * console.log('Event:', event.type);
2918
+ * const data = JSON.parse(event.data);
2919
+ *
2920
+ * if (data.jobId) {
2921
+ * console.log(`Job ${data.jobId}:`, data);
2922
+ * }
2923
+ * },
2924
+ * onError: (error) => {
2925
+ * console.error('SSE error:', error);
2926
+ * },
2927
+ * onOpen: () => {
2928
+ * console.log('SSE connection opened');
2929
+ * }
2930
+ * });
2931
+ *
2932
+ * // Listen for specific event types
2933
+ * eventSource.addEventListener('job.completed', (event) => {
2934
+ * const data = JSON.parse(event.data);
2935
+ * console.log('Job completed:', data);
2936
+ * });
2937
+ *
2938
+ * // Clean up
2939
+ * eventSource.close();
2940
+ * ```
2941
+ */
2942
+ subscribe(token: string, options?: Omit<SSESubscribeOptions, "lastEventId">): EventSource;
2943
+ /**
2944
+ * Subscribe to events for a specific job
2945
+ *
2946
+ * Opens an SSE connection filtered to a single job.
2947
+ * More efficient than global subscription when tracking one job.
2948
+ *
2949
+ * @param jobId - Job ID to subscribe to
2950
+ * @param token - SSE authentication token
2951
+ * @param options - Subscription options
2952
+ *
2953
+ * @example
2954
+ * ```ts
2955
+ * // Start a scrape job
2956
+ * const job = await client.scrape.create({ url: "https://example.com" });
2957
+ *
2958
+ * // Generate SSE token for this job
2959
+ * const { token } = await client.auth.generateSSEToken({ jobId: job.jobId });
2960
+ *
2961
+ * // Subscribe to job events
2962
+ * const eventSource = client.events.subscribeToJob(job.jobId, token, {
2963
+ * onEvent: (event) => {
2964
+ * const data = JSON.parse(event.data);
2965
+ * console.log(`[${event.type}]`, data);
2966
+ * }
2967
+ * });
2968
+ *
2969
+ * // Listen for completion
2970
+ * eventSource.addEventListener('job.completed', (event) => {
2971
+ * const data = JSON.parse(event.data);
2972
+ * console.log('Scrape completed!', data.summary);
2973
+ * eventSource.close();
2974
+ * });
2975
+ *
2976
+ * // Listen for progress
2977
+ * eventSource.addEventListener('job.progress', (event) => {
2978
+ * const data = JSON.parse(event.data);
2979
+ * console.log(`Progress: ${data.progress}%`);
2980
+ * });
2981
+ *
2982
+ * // Listen for errors
2983
+ * eventSource.addEventListener('job.failed', (event) => {
2984
+ * const data = JSON.parse(event.data);
2985
+ * console.error('Job failed:', data.error);
2986
+ * eventSource.close();
2987
+ * });
2988
+ * ```
2989
+ */
2990
+ subscribeToJob(jobId: string, token: string, options?: Omit<SSESubscribeOptions, "lastEventId">): EventSource;
2991
+ /**
2992
+ * Helper: Wait for job completion via SSE
2993
+ *
2994
+ * Convenience method that subscribes to a job and resolves when complete.
2995
+ * Automatically handles token generation and cleanup.
2996
+ *
2997
+ * @param jobId - Job ID to wait for
2998
+ * @param onProgress - Optional progress callback
2999
+ *
3000
+ * @example
3001
+ * ```ts
3002
+ * const job = await client.scrape.create({ url: "https://example.com" });
3003
+ *
3004
+ * // Wait for completion with progress updates
3005
+ * const result = await client.events.waitForCompletion(job.jobId, (progress) => {
3006
+ * console.log(`Progress: ${progress}%`);
3007
+ * });
3008
+ *
3009
+ * console.log('Job completed:', result);
3010
+ * ```
3011
+ */
3012
+ waitForCompletion<T = unknown>(jobId: string, onProgress?: (progress: number) => void): Promise<T>;
3013
+ }
3014
+
2388
3015
  /**
2389
3016
  * Extract Resource
2390
3017
  * Handles LLM-based structured data extraction
@@ -2689,7 +3316,8 @@ declare class ScrapeResource {
2689
3316
  * { url: "https://shop1.com/product1" },
2690
3317
  * { url: "https://shop2.com/deal", extractDeal: true }
2691
3318
  * ],
2692
- * defaults: { detectSignals: true }
3319
+ * defaults: { detectSignals: true },
3320
+ * ignoreInvalidURLs: true
2693
3321
  * });
2694
3322
  * console.log(batch.batchId, batch.results);
2695
3323
  * ```
@@ -2720,6 +3348,58 @@ declare class ScrapeResource {
2720
3348
  batchForDeals(urls: string[], options?: Omit<BatchScrapeOptions, "urls">): Promise<BatchScrapeResponse>;
2721
3349
  }
2722
3350
 
3351
+ /**
3352
+ * Screenshots Resource
3353
+ * Handles screenshot signed URL refresh and TTL limits
3354
+ */
3355
+
3356
+ /**
3357
+ * Options for refreshing a screenshot signed URL
3358
+ */
3359
+ interface RefreshScreenshotOptions {
3360
+ /** File path in storage (e.g., "job_abc123/timestamp_id_url.png") */
3361
+ path: string;
3362
+ /** Optional: New TTL in seconds (must be within tier limits) */
3363
+ ttl?: number;
3364
+ /** Optional: Bucket name (defaults to 'screenshots-private') */
3365
+ bucket?: string;
3366
+ }
3367
+ /**
3368
+ * Screenshots resource class
3369
+ * Provides methods for managing screenshot signed URLs
3370
+ */
3371
+ declare class ScreenshotsResource {
3372
+ private ctx;
3373
+ constructor(ctx: RequestContext);
3374
+ /**
3375
+ * Refresh a signed URL before expiration
3376
+ *
3377
+ * @example
3378
+ * ```ts
3379
+ * const refreshed = await client.screenshots.refresh({
3380
+ * path: "job_abc123/1234567890_nanoid_example.png",
3381
+ * ttl: 604800 // 7 days
3382
+ * });
3383
+ * console.log(refreshed.url); // New signed URL
3384
+ * console.log(refreshed.expiresAt); // "2026-01-25T12:00:00Z"
3385
+ * console.log(refreshed.tierLimits); // { min: 3600, max: 604800, default: 604800 }
3386
+ * ```
3387
+ */
3388
+ refresh(options: RefreshScreenshotOptions): Promise<ScreenshotRefreshResponse>;
3389
+ /**
3390
+ * Get TTL limits for the current tier
3391
+ *
3392
+ * @example
3393
+ * ```ts
3394
+ * const limits = await client.screenshots.getLimits();
3395
+ * console.log(limits.tier); // "pro"
3396
+ * console.log(limits.limits.max); // 604800 (7 days in seconds)
3397
+ * console.log(limits.formattedLimits.max); // "7 days"
3398
+ * ```
3399
+ */
3400
+ getLimits(): Promise<ScreenshotLimitsResponse>;
3401
+ }
3402
+
2723
3403
  /**
2724
3404
  * Search Resource
2725
3405
  * Handles web search operations with AI optimization
@@ -2739,7 +3419,7 @@ declare class SearchResource {
2739
3419
  * ```ts
2740
3420
  * const result = await client.search.create({
2741
3421
  * query: "laptop deals black friday",
2742
- * maxResults: 20,
3422
+ * limit: 20,
2743
3423
  * useDealScoring: true
2744
3424
  * });
2745
3425
  * ```
@@ -2775,12 +3455,12 @@ declare class SearchResource {
2775
3455
  * @example
2776
3456
  * ```ts
2777
3457
  * const result = await client.search.andScrape("promo codes", {
2778
- * autoScrapeLimit: 5
3458
+ * maxScrapeResults: 5
2779
3459
  * });
2780
3460
  * console.log(result.data.scrapedJobIds);
2781
3461
  * ```
2782
3462
  */
2783
- andScrape(query: string, options?: Omit<SearchOptions, "query" | "autoScrape">): Promise<SearchJobResponse>;
3463
+ andScrape(query: string, options?: Omit<SearchOptions, "query" | "scrapeResults">): Promise<SearchJobResponse>;
2784
3464
  /**
2785
3465
  * Check search API status
2786
3466
  * Returns availability and features info
@@ -3056,6 +3736,9 @@ declare class WebhooksResource {
3056
3736
  * await client.extract.create({ url: "...", schema: {...} });
3057
3737
  * await client.dork.create({ query: "..." });
3058
3738
  *
3739
+ * // Utilities
3740
+ * await client.convert.htmlToMarkdown({ html: "..." });
3741
+ *
3059
3742
  * // Status & polling
3060
3743
  * await client.status.get(jobId);
3061
3744
  * await client.waitForResult(jobId);
@@ -3147,6 +3830,20 @@ declare class DealCrawl {
3147
3830
  * ```
3148
3831
  */
3149
3832
  readonly dork: DorkResource;
3833
+ /**
3834
+ * Convert resource - HTML to Markdown conversion
3835
+ *
3836
+ * @example
3837
+ * ```ts
3838
+ * const result = await client.convert.htmlToMarkdown({
3839
+ * html: "<h1>Title</h1><p>Content</p>",
3840
+ * baseUrl: "https://example.com",
3841
+ * options: { removeNoise: true }
3842
+ * });
3843
+ * console.log(result.data.markdown);
3844
+ * ```
3845
+ */
3846
+ readonly convert: ConvertResource;
3150
3847
  /**
3151
3848
  * Agent resource - AI-powered autonomous web navigation
3152
3849
  *
@@ -3228,6 +3925,80 @@ declare class DealCrawl {
3228
3925
  * ```
3229
3926
  */
3230
3927
  readonly account: AccountResource;
3928
+ /**
3929
+ * Screenshots resource - Screenshot signed URL management
3930
+ *
3931
+ * @example
3932
+ * ```ts
3933
+ * // Refresh a signed URL before expiration
3934
+ * const refreshed = await client.screenshots.refresh({
3935
+ * path: "job_abc123/1234567890_nanoid_example.png",
3936
+ * ttl: 604800 // 7 days
3937
+ * });
3938
+ *
3939
+ * // Get tier-specific TTL limits
3940
+ * const limits = await client.screenshots.getLimits();
3941
+ * console.log(limits.formattedLimits.max); // "7 days"
3942
+ * ```
3943
+ */
3944
+ readonly screenshots: ScreenshotsResource;
3945
+ /**
3946
+ * Auth resource - SSE (Server-Sent Events) authentication
3947
+ *
3948
+ * @example
3949
+ * ```ts
3950
+ * // Generate SSE token for browser EventSource
3951
+ * const { token, expiresAt } = await client.auth.generateSSEToken();
3952
+ *
3953
+ * // Use in browser
3954
+ * const eventSource = new EventSource(`/v1/events?token=${token}`);
3955
+ *
3956
+ * // Generate token for specific job
3957
+ * const jobToken = await client.auth.generateSSEToken({ jobId: "job_123" });
3958
+ *
3959
+ * // Check connection limits
3960
+ * const limits = await client.auth.getLimits();
3961
+ * console.log(`Available connections: ${limits.sse.available}`);
3962
+ * ```
3963
+ */
3964
+ readonly auth: AuthResource;
3965
+ /**
3966
+ * Events resource - Real-time SSE event streaming (Browser only)
3967
+ *
3968
+ * IMPORTANT: This resource only works in browsers. For Node.js, use polling via client.status.get()
3969
+ *
3970
+ * @example Browser Usage
3971
+ * ```ts
3972
+ * // 1. Generate SSE token
3973
+ * const { token } = await client.auth.generateSSEToken();
3974
+ *
3975
+ * // 2. Subscribe to all events
3976
+ * const eventSource = client.events.subscribe(token, {
3977
+ * onEvent: (event) => {
3978
+ * console.log('Event:', event.type, JSON.parse(event.data));
3979
+ * }
3980
+ * });
3981
+ *
3982
+ * // 3. Or subscribe to specific job
3983
+ * const jobEvents = client.events.subscribeToJob('job_123', token, {
3984
+ * onEvent: (event) => {
3985
+ * const data = JSON.parse(event.data);
3986
+ * console.log(`Progress: ${data.progress}%`);
3987
+ * }
3988
+ * });
3989
+ *
3990
+ * // 4. Listen for specific events
3991
+ * eventSource.addEventListener('job.completed', (event) => {
3992
+ * const data = JSON.parse(event.data);
3993
+ * console.log('Job completed!', data.summary);
3994
+ * eventSource.close();
3995
+ * });
3996
+ *
3997
+ * // 5. Clean up
3998
+ * eventSource.close();
3999
+ * ```
4000
+ */
4001
+ readonly events: EventsResource;
3231
4002
  /**
3232
4003
  * Create a new DealCrawl client
3233
4004
  *
@@ -3635,4 +4406,352 @@ declare const ERROR_MESSAGES: Record<ErrorCode, string>;
3635
4406
  */
3636
4407
  declare function getErrorMessage(code: ErrorCode): string;
3637
4408
 
3638
- export { ALL_API_KEY_SCOPES, type AccountInfoResponse, type AccountMetricsResponse, AccountResource, type ActionBaseOptions, type ActionInput, type AgentAction, type AgentActionType, type AgentCompletionReason, type AgentJobResponse, type AgentModel, type AgentOptions, AgentResource, type AgentResultResponse, type AgentStatusResponse, type AgentStepResponse, type ApiError, type ApiKeyInfo, type ApiKeyScope, type ApiResponse, type BatchScrapeDefaults, type BatchScrapeItem, type BatchScrapeOptions, type BatchScrapeResponse, type BatchScrapeResultItem, type BatchStatusResponse, type CancelJobResponse, type CheckpointInfo, type ClickAction, type ClientPreferences, type ClientStatsResponse, type CrawlAnalysisResponse, type CrawlError, type CrawlJobResponse, type CrawlMode, type CrawlOptions, type CrawlRecommendation, CrawlResource, type CrawlResult, type CrawlStats, type CrawlTemplate, type CrawlTemplateId, type CreateApiKeyOptions, type CreateKeyResponse, type CreateWebhookOptions, type CreateWebhookResponse, type CreatedApiKey, DEFAULT_API_KEY_SCOPES, DEFAULT_CONFIG, DataResource, DealCrawl, type DealCrawlConfig, DealCrawlError, type DealDetails, type DealItem, type DealMetrics, type DealScoreSummary, type DealSummary, type DealUpMetrics, type DealUpMetricsResponse, type DeleteKeyResponse, type DeleteWebhookResponse, type DiscountSignal, type DorkJobResponse, type DorkOptions, DorkResource, type DorkResult, ERROR_CODES, ERROR_MESSAGES, type EngineSelection, type EngineType, type ErrorCode, type ExportDealsOptions, type ExportFormat, type ExportJobsOptions, type ExtractJobResponse, type ExtractModel, type ExtractOptions, ExtractResource, type ExtractedDeal, type ExtractionErrorDetails, type FallbackConfig, type FallbackMetadata, type FallbackResult, type FallbackSource, type GetApiKeyStatsOptions, type GetDealsOptions, type HoverAction, type JobDealsResponse, type JobMetricsResponse, type JobResponse, type JobStatus, type JobStatusFilter, type JobStatusResponse, type JobSummary, type JobTypeFilter, type KeyStatsResponse, KeysResource, type ListApiKeysOptions, type ListDealsOptions, type ListDealsResponse, type ListJobsOptions, type ListJobsResponse, type ListKeysResponse, type ListWebhooksResponse, type PaginatedResponse, type PaginationInfo, type ParsedPage, type PreferencesResponse, type PressAction, type PriceSignal, type PricingInfo, type ProductCategory, type ProductInfo, type RateLimitInfo, type RecommendationsResponse, type RequestContext, type ResumeJobResponse, type RevokeApiKeyOptions, type RotateApiKeyOptions, type RotateKeyResponse, type ScrapeJobResponse, type ScrapeOptions, ScrapeResource, type ScrapeResult, type ScreenshotAction, type ScreenshotAgentAction, type ScreenshotOptions, type ScreenshotResult, type ScrollAction, type SearchAiModel, type SearchAiProvider, type SearchData, type SearchDateRange, type SearchFilters, type SearchJobResponse, type SearchOptions, SearchResource, type SearchResultItem, type SearchStatusResponse, type SelectAction, type ClickAction$1 as SharedClickAction, type HoverAction$1 as SharedHoverAction, type PressAction$1 as SharedPressAction, type ScrollAction$1 as SharedScrollAction, type SelectAction$1 as SharedSelectAction, type WaitAction$1 as SharedWaitAction, type WriteAction$1 as SharedWriteAction, type Signal, type SortOrder, StatusResource, type TestWebhookResponse, type UpdatePreferencesOptions, type UpdatePreferencesResponse, type UpdateWebhookOptions, type UpdateWebhookResponse, type UrgencyLevel, type UsageStats, type ValidationError, type ValidationResult, type WaitAction, type WaitOptions, type WaitResult, type WebhookEvent, type WebhookItem, WebhooksResource, type WriteAction, DealCrawl as default, getErrorMessage, pollUntil, waitForAll, waitForAny, waitForResult };
4409
+ /**
4410
+ * Advanced Job Status System
4411
+ *
4412
+ * Production-grade status model with:
4413
+ * - Primary status (lifecycle stage)
4414
+ * - Sub-status (detailed state within stage)
4415
+ * - Reason codes (why this state was reached)
4416
+ * - State machine transitions (valid state changes)
4417
+ *
4418
+ * @module job-status
4419
+ */
4420
+ /**
4421
+ * Primary job status - represents the main lifecycle stage
4422
+ */
4423
+ type JobStatus = "queued" | "scheduled" | "priority_queued" | "initializing" | "active" | "processing" | "extracting" | "uploading" | "finalizing" | "retrying" | "backing_off" | "paused" | "pausing" | "resuming" | "completed" | "partial" | "failed" | "timeout" | "cancelled" | "rejected" | "stale" | "zombie" | "stuck";
4424
+ /**
4425
+ * Sub-status provides granular detail within a primary status
4426
+ */
4427
+ type JobSubStatus = "waiting_for_worker" | "waiting_for_rate_limit" | "waiting_for_quota" | "waiting_for_retry" | "waiting_for_dependency" | "launching_browser" | "establishing_connection" | "loading_config" | "validating_input" | "fetching_page" | "executing_javascript" | "waiting_for_render" | "detecting_signals" | "crawling_links" | "capturing_screenshot" | "calling_llm" | "parsing_response" | "validating_extraction" | "uploading_screenshot" | "uploading_artifacts" | "generating_signed_url" | "saving_to_database" | "cleaning_resources" | "sending_webhook" | "syncing_to_dealup" | "exponential_backoff" | "rate_limit_backoff" | "circuit_breaker_open" | "network_error" | "timeout_error" | "bot_detected" | "captcha_detected" | "auth_required" | "page_not_found" | "server_error" | "parse_error" | "extraction_failed" | "quota_exceeded" | "invalid_input" | "resource_unavailable" | "no_heartbeat" | "progress_stalled" | "worker_lost";
4428
+ /**
4429
+ * Reason codes - why a status/substatus was reached
4430
+ */
4431
+ type StatusReason = "job_completed_successfully" | "partial_results_available" | "added_to_queue" | "priority_elevated" | "rate_limit_applied" | "quota_check_pending" | "transient_network_error" | "rate_limit_exceeded" | "timeout_exceeded" | "worker_crashed" | "max_retries_not_reached" | "max_retries_exceeded" | "permanent_error" | "blocked_by_site" | "captcha_required" | "invalid_url_format" | "ssrf_detected" | "quota_exhausted" | "tier_restriction" | "resource_not_available" | "cancelled_by_user" | "cancelled_by_system" | "duplicate_detected" | "policy_violation" | "heartbeat_timeout" | "progress_timeout" | "worker_unresponsive";
4432
+
4433
+ /**
4434
+ * Professional structured logger for DealCrawl
4435
+ * - Environment-aware (verbose in dev, minimal in prod)
4436
+ * - Structured JSON output in production
4437
+ * - Colored console output in development
4438
+ * - Correlation ID support for request tracing
4439
+ */
4440
+ type LogLevel = "debug" | "info" | "warn" | "error";
4441
+
4442
+ /**
4443
+ * SSE (Server-Sent Events) Event Types
4444
+ *
4445
+ * Defines all events that can be pushed to clients via SSE.
4446
+ *
4447
+ * @module sse-events
4448
+ */
4449
+
4450
+ /**
4451
+ * SSE event types - used as the "event:" field in SSE messages
4452
+ */
4453
+ type SSEEventType = "job.created" | "job.queued" | "job.started" | "job.status" | "job.progress" | "job.completed" | "job.failed" | "job.cancelled" | "job.paused" | "job.resumed" | "job.log" | "job.metric" | "job.alert" | "job.checkpoint" | "deal.found" | "deal.validated" | "ping" | "connection.open" | "connection.close" | "error";
4454
+ /**
4455
+ * Base SSE event structure - all events extend this
4456
+ */
4457
+ interface BaseSSEEvent {
4458
+ /** Event type (used in SSE "event:" field) */
4459
+ type: SSEEventType;
4460
+ /** Unique event ID (for Last-Event-ID replay) */
4461
+ id: string;
4462
+ /** When this event was generated */
4463
+ timestamp: string;
4464
+ /** Job ID this event relates to (null for system events) */
4465
+ jobId: string | null;
4466
+ /** Client ID (for server-side filtering) */
4467
+ clientId: string;
4468
+ }
4469
+ /**
4470
+ * Job created event - sent when job is first created
4471
+ */
4472
+ interface JobCreatedEvent extends BaseSSEEvent {
4473
+ type: "job.created";
4474
+ jobId: string;
4475
+ data: {
4476
+ jobType: "scrape" | "crawl" | "dork" | "extract" | "agent";
4477
+ priority?: "high" | "medium" | "low";
4478
+ template?: string;
4479
+ };
4480
+ }
4481
+ /**
4482
+ * Job queued event - sent when job enters queue
4483
+ */
4484
+ interface JobQueuedEvent extends BaseSSEEvent {
4485
+ type: "job.queued";
4486
+ jobId: string;
4487
+ data: {
4488
+ queue: string;
4489
+ position?: number;
4490
+ estimatedWaitMs?: number;
4491
+ };
4492
+ }
4493
+ /**
4494
+ * Job started event - sent when worker picks up job
4495
+ */
4496
+ interface JobStartedEvent extends BaseSSEEvent {
4497
+ type: "job.started";
4498
+ jobId: string;
4499
+ data: {
4500
+ workerId?: string;
4501
+ startedAt: string;
4502
+ };
4503
+ }
4504
+ /**
4505
+ * Job status change event - sent on every status transition
4506
+ */
4507
+ interface JobStatusEvent extends BaseSSEEvent {
4508
+ type: "job.status";
4509
+ jobId: string;
4510
+ data: {
4511
+ status: JobStatus;
4512
+ subStatus?: JobSubStatus;
4513
+ reason?: StatusReason;
4514
+ message?: string;
4515
+ previousStatus?: JobStatus;
4516
+ progress: number;
4517
+ };
4518
+ }
4519
+ /**
4520
+ * Job progress event - sent periodically during processing
4521
+ */
4522
+ interface JobProgressEvent extends BaseSSEEvent {
4523
+ type: "job.progress";
4524
+ jobId: string;
4525
+ data: {
4526
+ progress: number;
4527
+ status: JobStatus;
4528
+ subStatus?: JobSubStatus;
4529
+ message?: string;
4530
+ eta?: {
4531
+ remainingMs: number;
4532
+ remainingFormatted: string;
4533
+ };
4534
+ stats?: {
4535
+ pagesProcessed?: number;
4536
+ dealsFound?: number;
4537
+ signalsDetected?: number;
4538
+ };
4539
+ };
4540
+ }
4541
+ /**
4542
+ * Job completed event - sent when job finishes successfully
4543
+ */
4544
+ interface JobCompletedEvent extends BaseSSEEvent {
4545
+ type: "job.completed";
4546
+ jobId: string;
4547
+ data: {
4548
+ statusUrl: string;
4549
+ completedAt: string;
4550
+ durationMs: number;
4551
+ summary: {
4552
+ success: boolean;
4553
+ pagesProcessed?: number;
4554
+ dealsFound?: number;
4555
+ signalsDetected?: number;
4556
+ hasScreenshot?: boolean;
4557
+ };
4558
+ result?: {
4559
+ dealScore?: number;
4560
+ topSignals?: string[];
4561
+ dealTitle?: string;
4562
+ };
4563
+ };
4564
+ }
4565
+ /**
4566
+ * Job failed event - sent when job fails
4567
+ */
4568
+ interface JobFailedEvent extends BaseSSEEvent {
4569
+ type: "job.failed";
4570
+ jobId: string;
4571
+ data: {
4572
+ statusUrl: string;
4573
+ failedAt: string;
4574
+ durationMs: number;
4575
+ error: {
4576
+ code: string;
4577
+ message: string;
4578
+ recoverable: boolean;
4579
+ };
4580
+ retry?: {
4581
+ attempt: number;
4582
+ maxAttempts: number;
4583
+ willRetry: boolean;
4584
+ nextRetryAt?: string;
4585
+ };
4586
+ };
4587
+ }
4588
+ /**
4589
+ * Job cancelled event - sent when job is cancelled
4590
+ */
4591
+ interface JobCancelledEvent extends BaseSSEEvent {
4592
+ type: "job.cancelled";
4593
+ jobId: string;
4594
+ data: {
4595
+ cancelledAt: string;
4596
+ cancelledBy: "user" | "system";
4597
+ reason?: string;
4598
+ };
4599
+ }
4600
+ /**
4601
+ * Job paused event
4602
+ */
4603
+ interface JobPausedEvent extends BaseSSEEvent {
4604
+ type: "job.paused";
4605
+ jobId: string;
4606
+ data: {
4607
+ pausedAt: string;
4608
+ checkpoint?: {
4609
+ lastProcessedUrl: string;
4610
+ urlsProcessed: number;
4611
+ urlsRemaining: number;
4612
+ };
4613
+ };
4614
+ }
4615
+ /**
4616
+ * Job resumed event
4617
+ */
4618
+ interface JobResumedEvent extends BaseSSEEvent {
4619
+ type: "job.resumed";
4620
+ jobId: string;
4621
+ data: {
4622
+ resumedAt: string;
4623
+ fromCheckpoint: boolean;
4624
+ };
4625
+ }
4626
+ /**
4627
+ * Job log event - sent for important log messages
4628
+ */
4629
+ interface JobLogEvent extends BaseSSEEvent {
4630
+ type: "job.log";
4631
+ jobId: string;
4632
+ data: {
4633
+ level: LogLevel;
4634
+ message: string;
4635
+ metadata?: Record<string, unknown>;
4636
+ };
4637
+ }
4638
+ /**
4639
+ * Job metric event - sent for performance/business metrics
4640
+ */
4641
+ interface JobMetricEvent extends BaseSSEEvent {
4642
+ type: "job.metric";
4643
+ jobId: string;
4644
+ data: {
4645
+ metric: string;
4646
+ value: number;
4647
+ unit?: string;
4648
+ };
4649
+ }
4650
+ /**
4651
+ * Job alert event - sent for important alerts (quota warnings, etc.)
4652
+ */
4653
+ interface JobAlertEvent extends BaseSSEEvent {
4654
+ type: "job.alert";
4655
+ jobId: string;
4656
+ data: {
4657
+ severity: "info" | "warning" | "error" | "critical";
4658
+ title: string;
4659
+ message: string;
4660
+ actionRequired?: boolean;
4661
+ };
4662
+ }
4663
+ /**
4664
+ * Job checkpoint event - sent when checkpoint is saved
4665
+ */
4666
+ interface JobCheckpointEvent extends BaseSSEEvent {
4667
+ type: "job.checkpoint";
4668
+ jobId: string;
4669
+ data: {
4670
+ savedAt: string;
4671
+ lastProcessedUrl: string;
4672
+ urlsProcessed: number;
4673
+ urlsRemaining: number;
4674
+ canResume: boolean;
4675
+ };
4676
+ }
4677
+ /**
4678
+ * Deal found event - sent when a deal is detected
4679
+ */
4680
+ interface DealFoundEvent extends BaseSSEEvent {
4681
+ type: "deal.found";
4682
+ jobId: string;
4683
+ data: {
4684
+ dealId?: string;
4685
+ url: string;
4686
+ title: string;
4687
+ score: number;
4688
+ price?: number;
4689
+ discount?: number;
4690
+ category?: string;
4691
+ signals: string[];
4692
+ };
4693
+ }
4694
+ /**
4695
+ * Deal validated event - sent when deal is validated/scored
4696
+ */
4697
+ interface DealValidatedEvent extends BaseSSEEvent {
4698
+ type: "deal.validated";
4699
+ jobId: string;
4700
+ data: {
4701
+ dealId: string;
4702
+ score: number;
4703
+ confidence: number;
4704
+ accepted: boolean;
4705
+ rejectionReason?: string;
4706
+ };
4707
+ }
4708
+ /**
4709
+ * Ping event - keepalive to prevent connection timeout
4710
+ */
4711
+ interface PingEvent extends BaseSSEEvent {
4712
+ type: "ping";
4713
+ jobId: null;
4714
+ data: {
4715
+ serverTime: string;
4716
+ };
4717
+ }
4718
+ /**
4719
+ * Connection open event - sent when SSE connection established
4720
+ */
4721
+ interface ConnectionOpenEvent extends BaseSSEEvent {
4722
+ type: "connection.open";
4723
+ jobId: null;
4724
+ data: {
4725
+ clientId: string;
4726
+ reconnected: boolean;
4727
+ lastEventId?: string;
4728
+ };
4729
+ }
4730
+ /**
4731
+ * Connection close event - sent before connection closes
4732
+ */
4733
+ interface ConnectionCloseEvent extends BaseSSEEvent {
4734
+ type: "connection.close";
4735
+ jobId: null;
4736
+ data: {
4737
+ reason: string;
4738
+ };
4739
+ }
4740
+ /**
4741
+ * Error event - sent when an error occurs
4742
+ */
4743
+ interface ErrorEvent extends BaseSSEEvent {
4744
+ type: "error";
4745
+ jobId: string | null;
4746
+ data: {
4747
+ code: string;
4748
+ message: string;
4749
+ recoverable: boolean;
4750
+ };
4751
+ }
4752
+ /**
4753
+ * All possible SSE events (discriminated union)
4754
+ */
4755
+ type SSEEvent = JobCreatedEvent | JobQueuedEvent | JobStartedEvent | JobStatusEvent | JobProgressEvent | JobCompletedEvent | JobFailedEvent | JobCancelledEvent | JobPausedEvent | JobResumedEvent | JobLogEvent | JobMetricEvent | JobAlertEvent | JobCheckpointEvent | DealFoundEvent | DealValidatedEvent | PingEvent | ConnectionOpenEvent | ConnectionCloseEvent | ErrorEvent;
4756
+
4757
+ export { ALL_API_KEY_SCOPES, type AccountInfoResponse, type AccountMetricsResponse, AccountResource, type ActionBaseOptions, type ActionInput, type AgentAction, type AgentActionType, type AgentCompletionReason, type AgentJobResponse, type AgentModel, type AgentOptions, AgentResource, type AgentResultResponse, type AgentStatusResponse, type AgentStepResponse, type ApiError, type ApiKeyInfo, type ApiKeyScope, type ApiResponse, AuthResource, type BaseSSEEvent, type BatchScrapeDefaults, type BatchScrapeItem, type BatchScrapeOptions, type BatchScrapeResponse, type BatchScrapeResultItem, type BatchStatusResponse, type CancelJobResponse, type CheckpointInfo, type ClickAction, type ClientPreferences, type ClientStatsResponse, type ConnectionCloseEvent, type ConnectionOpenEvent, type ConversionMetadata, type ConvertOptions, ConvertResource, type ConvertResponse, type CrawlAnalysisResponse, type CrawlError, type CrawlJobResponse, type CrawlMode, type CrawlOptions, type CrawlRecommendation, CrawlResource, type CrawlResult, type CrawlStats, type CrawlTemplate, type CrawlTemplateId, type CreateApiKeyOptions, type CreateKeyResponse, type CreateWebhookOptions, type CreateWebhookResponse, type CreatedApiKey, DEFAULT_API_KEY_SCOPES, DEFAULT_CONFIG, DataResource, DealCrawl, type DealCrawlConfig, DealCrawlError, type DealDetails, type DealFoundEvent, type DealItem, type DealMetrics, type DealScoreSummary, type DealSummary, type DealUpMetrics, type DealUpMetricsResponse, type DealValidatedEvent, type DeleteKeyResponse, type DeleteWebhookResponse, type DiscountSignal, type DorkJobResponse, type DorkOptions, DorkResource, type DorkResult, ERROR_CODES, ERROR_MESSAGES, type EngineSelection, type EngineType, type ErrorCode, type ErrorEvent, EventsResource, type ExportDealsOptions, type ExportFormat, type ExportJobsOptions, type ExtractJobResponse, type ExtractModel, type ExtractOptions, ExtractResource, type ExtractedDeal, type ExtractionErrorDetails, type FallbackConfig, type FallbackMetadata, type FallbackResult, type FallbackSource, type GenerateSSETokenOptions, type GetApiKeyStatsOptions, type GetDealsOptions, type HoverAction, type JobAlertEvent, type JobCancelledEvent, type JobCheckpointEvent, type JobCompletedEvent, type JobCreatedEvent, type JobDealsResponse, type JobFailedEvent, type JobLogEvent, type JobMetricEvent, type JobMetricsResponse, type JobPausedEvent, type JobProgressEvent, type JobQueuedEvent, type JobResponse, type JobResumedEvent, type JobStartedEvent, type JobStatus$1 as JobStatus, type JobStatusEvent, type JobStatusFilter, type JobStatusResponse, type JobSummary, type JobTypeFilter, type KeyStatsResponse, KeysResource, type ListApiKeysOptions, type ListDealsOptions, type ListDealsResponse, type ListJobsOptions, type ListJobsResponse, type ListKeysResponse, type ListWebhooksResponse, type PaginatedResponse, type PaginationInfo, type ParsedPage, type PingEvent, type PreferencesResponse, type PressAction, type PriceSignal, type PricingInfo, type ProductCategory, type ProductInfo, type RateLimitInfo, type RecommendationsResponse, type RefreshScreenshotOptions, type RequestContext, type ResumeJobResponse, type RevokeApiKeyOptions, type RotateApiKeyOptions, type RotateKeyResponse, type SSEConnectionLimitsResponse, type SSEEvent, type SSEEventType, type SSESubscribeOptions, type SSETokenResponse, type ScrapeJobResponse, type ScrapeOptions, ScrapeResource, type ScrapeResult, type ScreenshotAction, type ScreenshotAgentAction, type ScreenshotLimitsResponse, type ScreenshotOptions, type ScreenshotRefreshResponse, type ScreenshotResult, ScreenshotsResource, type ScrollAction, type SearchAiModel, type SearchAiProvider, type SearchData, type SearchDateRange, type SearchFilters, type SearchJobResponse, type SearchOptions, SearchResource, type SearchResultItem, type SearchStatusResponse, type SelectAction, type ClickAction$1 as SharedClickAction, type HoverAction$1 as SharedHoverAction, type PressAction$1 as SharedPressAction, type ScrollAction$1 as SharedScrollAction, type SelectAction$1 as SharedSelectAction, type WaitAction$1 as SharedWaitAction, type WriteAction$1 as SharedWriteAction, type Signal, type SortOrder, StatusResource, type TestWebhookResponse, type UpdatePreferencesOptions, type UpdatePreferencesResponse, type UpdateWebhookOptions, type UpdateWebhookResponse, type UrgencyLevel, type UsageStats, type ValidationError, type ValidationResult, type WaitAction, type WaitOptions, type WaitResult, type WebhookEvent, type WebhookItem, WebhooksResource, type WriteAction, DealCrawl as default, getErrorMessage, pollUntil, waitForAll, waitForAny, waitForResult };