@dealcrawl/sdk 2.6.0 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +291 -83
- package/dist/index.d.mts +124 -6
- package/dist/index.d.ts +124 -6
- package/dist/index.js +21 -7
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +21 -7
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.d.mts
CHANGED
|
@@ -26,6 +26,7 @@ interface PaginatedResponse<T> {
|
|
|
26
26
|
page: number;
|
|
27
27
|
limit: number;
|
|
28
28
|
total: number;
|
|
29
|
+
totalPages: number;
|
|
29
30
|
hasMore: boolean;
|
|
30
31
|
};
|
|
31
32
|
}
|
|
@@ -97,6 +98,8 @@ interface ParsedPage {
|
|
|
97
98
|
title?: string;
|
|
98
99
|
description?: string;
|
|
99
100
|
content?: string;
|
|
101
|
+
/** Markdown version of content (when outputMarkdown is enabled) */
|
|
102
|
+
markdown?: string;
|
|
100
103
|
language?: string;
|
|
101
104
|
/** @deprecated Use linksMetadata instead */
|
|
102
105
|
links?: string[];
|
|
@@ -175,6 +178,10 @@ interface ExtractedDeal {
|
|
|
175
178
|
category?: string;
|
|
176
179
|
description?: string;
|
|
177
180
|
imageUrl?: string;
|
|
181
|
+
/** Multiple product images (gallery) */
|
|
182
|
+
imageUrls?: string[];
|
|
183
|
+
/** Stored image URL in Supabase Storage */
|
|
184
|
+
imageStoredUrl?: string;
|
|
178
185
|
sku?: string;
|
|
179
186
|
};
|
|
180
187
|
pricing: {
|
|
@@ -299,13 +306,88 @@ interface ValidationError {
|
|
|
299
306
|
/** Received value */
|
|
300
307
|
received?: unknown;
|
|
301
308
|
}
|
|
309
|
+
/** Base action options for resilience */
|
|
310
|
+
interface ActionBaseOptions {
|
|
311
|
+
/** Don't fail the job if this action fails */
|
|
312
|
+
optional?: boolean;
|
|
313
|
+
/** Number of retries (1-5) */
|
|
314
|
+
retries?: number;
|
|
315
|
+
/** Delay before executing action (ms) */
|
|
316
|
+
delayBefore?: number;
|
|
317
|
+
}
|
|
318
|
+
/** Click action */
|
|
319
|
+
interface ClickAction$1 extends ActionBaseOptions {
|
|
320
|
+
type: "click";
|
|
321
|
+
selector: string;
|
|
322
|
+
waitAfter?: number;
|
|
323
|
+
button?: "left" | "right" | "middle";
|
|
324
|
+
clickCount?: number;
|
|
325
|
+
force?: boolean;
|
|
326
|
+
}
|
|
327
|
+
/** Scroll action */
|
|
328
|
+
interface ScrollAction$1 extends ActionBaseOptions {
|
|
329
|
+
type: "scroll";
|
|
330
|
+
direction: "up" | "down" | "left" | "right" | "to-element";
|
|
331
|
+
selector?: string;
|
|
332
|
+
amount?: number;
|
|
333
|
+
smooth?: boolean;
|
|
334
|
+
}
|
|
335
|
+
/** Write action */
|
|
336
|
+
interface WriteAction$1 extends ActionBaseOptions {
|
|
337
|
+
type: "write";
|
|
338
|
+
selector: string;
|
|
339
|
+
text: string;
|
|
340
|
+
clearFirst?: boolean;
|
|
341
|
+
pressEnter?: boolean;
|
|
342
|
+
typeDelay?: number;
|
|
343
|
+
}
|
|
344
|
+
/** Wait action */
|
|
345
|
+
interface WaitAction$1 extends ActionBaseOptions {
|
|
346
|
+
type: "wait";
|
|
347
|
+
milliseconds?: number;
|
|
348
|
+
selector?: string;
|
|
349
|
+
condition?: "visible" | "hidden" | "attached" | "detached";
|
|
350
|
+
timeout?: number;
|
|
351
|
+
}
|
|
352
|
+
/** Press action */
|
|
353
|
+
interface PressAction$1 extends ActionBaseOptions {
|
|
354
|
+
type: "press";
|
|
355
|
+
key: string;
|
|
356
|
+
modifiers?: ("Alt" | "Control" | "Meta" | "Shift")[];
|
|
357
|
+
}
|
|
358
|
+
/** Screenshot action */
|
|
359
|
+
interface ScreenshotAction extends ActionBaseOptions {
|
|
360
|
+
type: "screenshot";
|
|
361
|
+
fullPage?: boolean;
|
|
362
|
+
selector?: string;
|
|
363
|
+
format?: "png" | "jpeg";
|
|
364
|
+
quality?: number;
|
|
365
|
+
name?: string;
|
|
366
|
+
}
|
|
367
|
+
/** Hover action */
|
|
368
|
+
interface HoverAction$1 extends ActionBaseOptions {
|
|
369
|
+
type: "hover";
|
|
370
|
+
selector: string;
|
|
371
|
+
duration?: number;
|
|
372
|
+
}
|
|
373
|
+
/** Select action */
|
|
374
|
+
interface SelectAction$1 extends ActionBaseOptions {
|
|
375
|
+
type: "select";
|
|
376
|
+
selector: string;
|
|
377
|
+
value: string | string[];
|
|
378
|
+
byLabel?: boolean;
|
|
379
|
+
}
|
|
380
|
+
/** Union of all action types */
|
|
381
|
+
type ActionInput = ClickAction$1 | ScrollAction$1 | WriteAction$1 | WaitAction$1 | PressAction$1 | ScreenshotAction | HoverAction$1 | SelectAction$1;
|
|
302
382
|
|
|
303
383
|
/**
|
|
304
384
|
* SDK Configuration options
|
|
305
385
|
*/
|
|
306
386
|
interface DealCrawlConfig {
|
|
307
|
-
/** API key for authentication (required) */
|
|
308
|
-
apiKey
|
|
387
|
+
/** API key for authentication (required unless clientId is provided) */
|
|
388
|
+
apiKey?: string;
|
|
389
|
+
/** Client ID for internal authentication (alternative to apiKey) */
|
|
390
|
+
clientId?: string;
|
|
309
391
|
/** Base URL for the API (default: https://api.dealcrawl.dev) */
|
|
310
392
|
baseUrl?: string;
|
|
311
393
|
/** Default timeout for requests in milliseconds (default: 30000) */
|
|
@@ -323,8 +405,10 @@ interface DealCrawlConfig {
|
|
|
323
405
|
interface RequestContext {
|
|
324
406
|
/** Base URL for API requests */
|
|
325
407
|
baseUrl: string;
|
|
326
|
-
/** API key for authentication */
|
|
327
|
-
apiKey
|
|
408
|
+
/** API key for authentication (optional if clientId is provided) */
|
|
409
|
+
apiKey?: string;
|
|
410
|
+
/** Client ID for internal authentication (alternative to apiKey) */
|
|
411
|
+
clientId?: string;
|
|
328
412
|
/** Default timeout in milliseconds */
|
|
329
413
|
timeout: number;
|
|
330
414
|
/** Maximum number of retries */
|
|
@@ -990,6 +1074,7 @@ declare function pollUntil<T>(fetchFn: () => Promise<T>, conditionFn: (data: T)
|
|
|
990
1074
|
* SDK Request Options Types
|
|
991
1075
|
* These types define the options for SDK methods
|
|
992
1076
|
*/
|
|
1077
|
+
|
|
993
1078
|
/** Screenshot capture options */
|
|
994
1079
|
interface ScreenshotOptions {
|
|
995
1080
|
/** Enable screenshot capture */
|
|
@@ -1037,6 +1122,12 @@ interface ScrapeOptions {
|
|
|
1037
1122
|
headers?: Record<string, string>;
|
|
1038
1123
|
/** Timeout in milliseconds (default: 30000, max: 120000) */
|
|
1039
1124
|
timeout?: number;
|
|
1125
|
+
/** Convert content to Markdown (GFM) (default: false) */
|
|
1126
|
+
outputMarkdown?: boolean;
|
|
1127
|
+
/** Base URL for resolving relative URLs in markdown (defaults to scraped URL) */
|
|
1128
|
+
markdownBaseUrl?: string;
|
|
1129
|
+
/** Browser actions to execute before scraping (max: 20) */
|
|
1130
|
+
actions?: ActionInput[];
|
|
1040
1131
|
}
|
|
1041
1132
|
/** Individual URL item for batch scraping */
|
|
1042
1133
|
interface BatchScrapeItem {
|
|
@@ -1056,6 +1147,12 @@ interface BatchScrapeItem {
|
|
|
1056
1147
|
headers?: Record<string, string>;
|
|
1057
1148
|
/** Override timeout for this URL */
|
|
1058
1149
|
timeout?: number;
|
|
1150
|
+
/** Override outputMarkdown for this URL */
|
|
1151
|
+
outputMarkdown?: boolean;
|
|
1152
|
+
/** Override markdownBaseUrl for this URL */
|
|
1153
|
+
markdownBaseUrl?: string;
|
|
1154
|
+
/** Override actions for this URL */
|
|
1155
|
+
actions?: ActionInput[];
|
|
1059
1156
|
}
|
|
1060
1157
|
/** Default options applied to all URLs in a batch */
|
|
1061
1158
|
interface BatchScrapeDefaults {
|
|
@@ -1087,6 +1184,12 @@ interface BatchScrapeDefaults {
|
|
|
1087
1184
|
headers?: Record<string, string>;
|
|
1088
1185
|
/** Timeout in milliseconds */
|
|
1089
1186
|
timeout?: number;
|
|
1187
|
+
/** Convert content to Markdown (GFM) */
|
|
1188
|
+
outputMarkdown?: boolean;
|
|
1189
|
+
/** Base URL for resolving relative URLs in markdown */
|
|
1190
|
+
markdownBaseUrl?: string;
|
|
1191
|
+
/** Browser actions to execute before scraping */
|
|
1192
|
+
actions?: ActionInput[];
|
|
1090
1193
|
}
|
|
1091
1194
|
/** Options for batch scraping multiple URLs */
|
|
1092
1195
|
interface BatchScrapeOptions {
|
|
@@ -1211,6 +1314,14 @@ interface CrawlOptions {
|
|
|
1211
1314
|
webhookUrl?: string;
|
|
1212
1315
|
/** Auto-sync discovered deals to DealUp */
|
|
1213
1316
|
syncToDealup?: boolean;
|
|
1317
|
+
/** Don't save crawl results - Zero Data Retention (Pro/Enterprise only) */
|
|
1318
|
+
noStore?: boolean;
|
|
1319
|
+
/** Download and store product images to Supabase Storage (default: true) */
|
|
1320
|
+
downloadImages?: boolean;
|
|
1321
|
+
/** Maximum images per deal for gallery (default: 5, max: 10) */
|
|
1322
|
+
maxImagesPerDeal?: number;
|
|
1323
|
+
/** Image download timeout in ms (default: 10000) */
|
|
1324
|
+
imageDownloadTimeout?: number;
|
|
1214
1325
|
/** Site-specific config name from registry */
|
|
1215
1326
|
siteConfig?: string;
|
|
1216
1327
|
/** Job template to use (ecommerce, blog, docs, marketplace, custom) */
|
|
@@ -1219,6 +1330,10 @@ interface CrawlOptions {
|
|
|
1219
1330
|
useSmartRouting?: boolean;
|
|
1220
1331
|
/** Priority queue override (Enterprise only) */
|
|
1221
1332
|
priority?: CrawlPriority;
|
|
1333
|
+
/** Convert pages to Markdown (GFM) (default: false) */
|
|
1334
|
+
outputMarkdown?: boolean;
|
|
1335
|
+
/** Base URL for resolving relative URLs in markdown */
|
|
1336
|
+
markdownBaseUrl?: string;
|
|
1222
1337
|
}
|
|
1223
1338
|
/** Crawl template identifier */
|
|
1224
1339
|
type CrawlTemplateId = "ecommerce" | "blog" | "docs" | "marketplace" | "custom";
|
|
@@ -3113,9 +3228,12 @@ declare class DealCrawl {
|
|
|
3113
3228
|
*
|
|
3114
3229
|
* @example
|
|
3115
3230
|
* ```ts
|
|
3116
|
-
* //
|
|
3231
|
+
* // With API key
|
|
3117
3232
|
* const client = new DealCrawl({ apiKey: "sk_xxx" });
|
|
3118
3233
|
*
|
|
3234
|
+
* // With Client ID (for internal calls)
|
|
3235
|
+
* const client = new DealCrawl({ clientId: "uuid-xxx" });
|
|
3236
|
+
*
|
|
3119
3237
|
* // Full config
|
|
3120
3238
|
* const client = new DealCrawl({
|
|
3121
3239
|
* apiKey: "sk_xxx",
|
|
@@ -3510,4 +3628,4 @@ declare const ERROR_MESSAGES: Record<ErrorCode, string>;
|
|
|
3510
3628
|
*/
|
|
3511
3629
|
declare function getErrorMessage(code: ErrorCode): string;
|
|
3512
3630
|
|
|
3513
|
-
export { ALL_API_KEY_SCOPES, type AccountInfoResponse, type AccountMetricsResponse, AccountResource, type AgentAction, type AgentActionType, type AgentCompletionReason, type AgentJobResponse, type AgentModel, type AgentOptions, AgentResource, type AgentResultResponse, type AgentStatusResponse, type AgentStepResponse, type ApiError, type ApiKeyInfo, type ApiKeyScope, type ApiResponse, type BatchScrapeDefaults, type BatchScrapeItem, type BatchScrapeOptions, type BatchScrapeResponse, type BatchScrapeResultItem, type BatchStatusResponse, type CancelJobResponse, type CheckpointInfo, type ClickAction, type ClientPreferences, type ClientStatsResponse, type CrawlAnalysisResponse, type CrawlError, type CrawlJobResponse, type CrawlMode, type CrawlOptions, type CrawlRecommendation, CrawlResource, type CrawlResult, type CrawlStats, type CrawlTemplate, type CrawlTemplateId, type CreateApiKeyOptions, type CreateKeyResponse, type CreateWebhookOptions, type CreateWebhookResponse, type CreatedApiKey, DEFAULT_API_KEY_SCOPES, DEFAULT_CONFIG, DataResource, DealCrawl, type DealCrawlConfig, DealCrawlError, type DealDetails, type DealItem, type DealMetrics, type DealScoreSummary, type DealSummary, type DealUpMetrics, type DealUpMetricsResponse, type DeleteKeyResponse, type DeleteWebhookResponse, type DiscountSignal, type DorkJobResponse, type DorkOptions, DorkResource, type DorkResult, ERROR_CODES, ERROR_MESSAGES, type EngineSelection, type EngineType, type ErrorCode, type ExportDealsOptions, type ExportFormat, type ExportJobsOptions, type ExtractJobResponse, type ExtractModel, type ExtractOptions, ExtractResource, type ExtractedDeal, type ExtractionErrorDetails, type FallbackConfig, type FallbackMetadata, type FallbackResult, type FallbackSource, type GetApiKeyStatsOptions, type GetDealsOptions, type HoverAction, type JobDealsResponse, type JobMetricsResponse, type JobResponse, type JobStatus, type JobStatusFilter, type JobStatusResponse, type JobSummary, type JobTypeFilter, type KeyStatsResponse, KeysResource, type ListApiKeysOptions, type ListDealsOptions, type ListDealsResponse, type ListJobsOptions, type ListJobsResponse, type ListKeysResponse, type ListWebhooksResponse, type PaginatedResponse, type PaginationInfo, type ParsedPage, type PreferencesResponse, type PressAction, type PriceSignal, type PricingInfo, type ProductCategory, type ProductInfo, type RateLimitInfo, type RecommendationsResponse, type RequestContext, type ResumeJobResponse, type RevokeApiKeyOptions, type RotateApiKeyOptions, type RotateKeyResponse, type ScrapeJobResponse, type ScrapeOptions, ScrapeResource, type ScrapeResult, type ScreenshotAgentAction, type ScreenshotOptions, type ScreenshotResult, type ScrollAction, type SearchAiModel, type SearchAiProvider, type SearchData, type SearchDateRange, type SearchFilters, type SearchJobResponse, type SearchOptions, SearchResource, type SearchResultItem, type SearchStatusResponse, type SelectAction, type Signal, type SortOrder, StatusResource, type TestWebhookResponse, type UpdatePreferencesOptions, type UpdatePreferencesResponse, type UpdateWebhookOptions, type UpdateWebhookResponse, type UrgencyLevel, type UsageStats, type ValidationError, type ValidationResult, type WaitAction, type WaitOptions, type WaitResult, type WebhookEvent, type WebhookItem, WebhooksResource, type WriteAction, DealCrawl as default, getErrorMessage, pollUntil, waitForAll, waitForAny, waitForResult };
|
|
3631
|
+
export { ALL_API_KEY_SCOPES, type AccountInfoResponse, type AccountMetricsResponse, AccountResource, type ActionBaseOptions, type ActionInput, type AgentAction, type AgentActionType, type AgentCompletionReason, type AgentJobResponse, type AgentModel, type AgentOptions, AgentResource, type AgentResultResponse, type AgentStatusResponse, type AgentStepResponse, type ApiError, type ApiKeyInfo, type ApiKeyScope, type ApiResponse, type BatchScrapeDefaults, type BatchScrapeItem, type BatchScrapeOptions, type BatchScrapeResponse, type BatchScrapeResultItem, type BatchStatusResponse, type CancelJobResponse, type CheckpointInfo, type ClickAction, type ClientPreferences, type ClientStatsResponse, type CrawlAnalysisResponse, type CrawlError, type CrawlJobResponse, type CrawlMode, type CrawlOptions, type CrawlRecommendation, CrawlResource, type CrawlResult, type CrawlStats, type CrawlTemplate, type CrawlTemplateId, type CreateApiKeyOptions, type CreateKeyResponse, type CreateWebhookOptions, type CreateWebhookResponse, type CreatedApiKey, DEFAULT_API_KEY_SCOPES, DEFAULT_CONFIG, DataResource, DealCrawl, type DealCrawlConfig, DealCrawlError, type DealDetails, type DealItem, type DealMetrics, type DealScoreSummary, type DealSummary, type DealUpMetrics, type DealUpMetricsResponse, type DeleteKeyResponse, type DeleteWebhookResponse, type DiscountSignal, type DorkJobResponse, type DorkOptions, DorkResource, type DorkResult, ERROR_CODES, ERROR_MESSAGES, type EngineSelection, type EngineType, type ErrorCode, type ExportDealsOptions, type ExportFormat, type ExportJobsOptions, type ExtractJobResponse, type ExtractModel, type ExtractOptions, ExtractResource, type ExtractedDeal, type ExtractionErrorDetails, type FallbackConfig, type FallbackMetadata, type FallbackResult, type FallbackSource, type GetApiKeyStatsOptions, type GetDealsOptions, type HoverAction, type JobDealsResponse, type JobMetricsResponse, type JobResponse, type JobStatus, type JobStatusFilter, type JobStatusResponse, type JobSummary, type JobTypeFilter, type KeyStatsResponse, KeysResource, type ListApiKeysOptions, type ListDealsOptions, type ListDealsResponse, type ListJobsOptions, type ListJobsResponse, type ListKeysResponse, type ListWebhooksResponse, type PaginatedResponse, type PaginationInfo, type ParsedPage, type PreferencesResponse, type PressAction, type PriceSignal, type PricingInfo, type ProductCategory, type ProductInfo, type RateLimitInfo, type RecommendationsResponse, type RequestContext, type ResumeJobResponse, type RevokeApiKeyOptions, type RotateApiKeyOptions, type RotateKeyResponse, type ScrapeJobResponse, type ScrapeOptions, ScrapeResource, type ScrapeResult, type ScreenshotAction, type ScreenshotAgentAction, type ScreenshotOptions, type ScreenshotResult, type ScrollAction, type SearchAiModel, type SearchAiProvider, type SearchData, type SearchDateRange, type SearchFilters, type SearchJobResponse, type SearchOptions, SearchResource, type SearchResultItem, type SearchStatusResponse, type SelectAction, type ClickAction$1 as SharedClickAction, type HoverAction$1 as SharedHoverAction, type PressAction$1 as SharedPressAction, type ScrollAction$1 as SharedScrollAction, type SelectAction$1 as SharedSelectAction, type WaitAction$1 as SharedWaitAction, type WriteAction$1 as SharedWriteAction, type Signal, type SortOrder, StatusResource, type TestWebhookResponse, type UpdatePreferencesOptions, type UpdatePreferencesResponse, type UpdateWebhookOptions, type UpdateWebhookResponse, type UrgencyLevel, type UsageStats, type ValidationError, type ValidationResult, type WaitAction, type WaitOptions, type WaitResult, type WebhookEvent, type WebhookItem, WebhooksResource, type WriteAction, DealCrawl as default, getErrorMessage, pollUntil, waitForAll, waitForAny, waitForResult };
|
package/dist/index.d.ts
CHANGED
|
@@ -26,6 +26,7 @@ interface PaginatedResponse<T> {
|
|
|
26
26
|
page: number;
|
|
27
27
|
limit: number;
|
|
28
28
|
total: number;
|
|
29
|
+
totalPages: number;
|
|
29
30
|
hasMore: boolean;
|
|
30
31
|
};
|
|
31
32
|
}
|
|
@@ -97,6 +98,8 @@ interface ParsedPage {
|
|
|
97
98
|
title?: string;
|
|
98
99
|
description?: string;
|
|
99
100
|
content?: string;
|
|
101
|
+
/** Markdown version of content (when outputMarkdown is enabled) */
|
|
102
|
+
markdown?: string;
|
|
100
103
|
language?: string;
|
|
101
104
|
/** @deprecated Use linksMetadata instead */
|
|
102
105
|
links?: string[];
|
|
@@ -175,6 +178,10 @@ interface ExtractedDeal {
|
|
|
175
178
|
category?: string;
|
|
176
179
|
description?: string;
|
|
177
180
|
imageUrl?: string;
|
|
181
|
+
/** Multiple product images (gallery) */
|
|
182
|
+
imageUrls?: string[];
|
|
183
|
+
/** Stored image URL in Supabase Storage */
|
|
184
|
+
imageStoredUrl?: string;
|
|
178
185
|
sku?: string;
|
|
179
186
|
};
|
|
180
187
|
pricing: {
|
|
@@ -299,13 +306,88 @@ interface ValidationError {
|
|
|
299
306
|
/** Received value */
|
|
300
307
|
received?: unknown;
|
|
301
308
|
}
|
|
309
|
+
/** Base action options for resilience */
|
|
310
|
+
interface ActionBaseOptions {
|
|
311
|
+
/** Don't fail the job if this action fails */
|
|
312
|
+
optional?: boolean;
|
|
313
|
+
/** Number of retries (1-5) */
|
|
314
|
+
retries?: number;
|
|
315
|
+
/** Delay before executing action (ms) */
|
|
316
|
+
delayBefore?: number;
|
|
317
|
+
}
|
|
318
|
+
/** Click action */
|
|
319
|
+
interface ClickAction$1 extends ActionBaseOptions {
|
|
320
|
+
type: "click";
|
|
321
|
+
selector: string;
|
|
322
|
+
waitAfter?: number;
|
|
323
|
+
button?: "left" | "right" | "middle";
|
|
324
|
+
clickCount?: number;
|
|
325
|
+
force?: boolean;
|
|
326
|
+
}
|
|
327
|
+
/** Scroll action */
|
|
328
|
+
interface ScrollAction$1 extends ActionBaseOptions {
|
|
329
|
+
type: "scroll";
|
|
330
|
+
direction: "up" | "down" | "left" | "right" | "to-element";
|
|
331
|
+
selector?: string;
|
|
332
|
+
amount?: number;
|
|
333
|
+
smooth?: boolean;
|
|
334
|
+
}
|
|
335
|
+
/** Write action */
|
|
336
|
+
interface WriteAction$1 extends ActionBaseOptions {
|
|
337
|
+
type: "write";
|
|
338
|
+
selector: string;
|
|
339
|
+
text: string;
|
|
340
|
+
clearFirst?: boolean;
|
|
341
|
+
pressEnter?: boolean;
|
|
342
|
+
typeDelay?: number;
|
|
343
|
+
}
|
|
344
|
+
/** Wait action */
|
|
345
|
+
interface WaitAction$1 extends ActionBaseOptions {
|
|
346
|
+
type: "wait";
|
|
347
|
+
milliseconds?: number;
|
|
348
|
+
selector?: string;
|
|
349
|
+
condition?: "visible" | "hidden" | "attached" | "detached";
|
|
350
|
+
timeout?: number;
|
|
351
|
+
}
|
|
352
|
+
/** Press action */
|
|
353
|
+
interface PressAction$1 extends ActionBaseOptions {
|
|
354
|
+
type: "press";
|
|
355
|
+
key: string;
|
|
356
|
+
modifiers?: ("Alt" | "Control" | "Meta" | "Shift")[];
|
|
357
|
+
}
|
|
358
|
+
/** Screenshot action */
|
|
359
|
+
interface ScreenshotAction extends ActionBaseOptions {
|
|
360
|
+
type: "screenshot";
|
|
361
|
+
fullPage?: boolean;
|
|
362
|
+
selector?: string;
|
|
363
|
+
format?: "png" | "jpeg";
|
|
364
|
+
quality?: number;
|
|
365
|
+
name?: string;
|
|
366
|
+
}
|
|
367
|
+
/** Hover action */
|
|
368
|
+
interface HoverAction$1 extends ActionBaseOptions {
|
|
369
|
+
type: "hover";
|
|
370
|
+
selector: string;
|
|
371
|
+
duration?: number;
|
|
372
|
+
}
|
|
373
|
+
/** Select action */
|
|
374
|
+
interface SelectAction$1 extends ActionBaseOptions {
|
|
375
|
+
type: "select";
|
|
376
|
+
selector: string;
|
|
377
|
+
value: string | string[];
|
|
378
|
+
byLabel?: boolean;
|
|
379
|
+
}
|
|
380
|
+
/** Union of all action types */
|
|
381
|
+
type ActionInput = ClickAction$1 | ScrollAction$1 | WriteAction$1 | WaitAction$1 | PressAction$1 | ScreenshotAction | HoverAction$1 | SelectAction$1;
|
|
302
382
|
|
|
303
383
|
/**
|
|
304
384
|
* SDK Configuration options
|
|
305
385
|
*/
|
|
306
386
|
interface DealCrawlConfig {
|
|
307
|
-
/** API key for authentication (required) */
|
|
308
|
-
apiKey
|
|
387
|
+
/** API key for authentication (required unless clientId is provided) */
|
|
388
|
+
apiKey?: string;
|
|
389
|
+
/** Client ID for internal authentication (alternative to apiKey) */
|
|
390
|
+
clientId?: string;
|
|
309
391
|
/** Base URL for the API (default: https://api.dealcrawl.dev) */
|
|
310
392
|
baseUrl?: string;
|
|
311
393
|
/** Default timeout for requests in milliseconds (default: 30000) */
|
|
@@ -323,8 +405,10 @@ interface DealCrawlConfig {
|
|
|
323
405
|
interface RequestContext {
|
|
324
406
|
/** Base URL for API requests */
|
|
325
407
|
baseUrl: string;
|
|
326
|
-
/** API key for authentication */
|
|
327
|
-
apiKey
|
|
408
|
+
/** API key for authentication (optional if clientId is provided) */
|
|
409
|
+
apiKey?: string;
|
|
410
|
+
/** Client ID for internal authentication (alternative to apiKey) */
|
|
411
|
+
clientId?: string;
|
|
328
412
|
/** Default timeout in milliseconds */
|
|
329
413
|
timeout: number;
|
|
330
414
|
/** Maximum number of retries */
|
|
@@ -990,6 +1074,7 @@ declare function pollUntil<T>(fetchFn: () => Promise<T>, conditionFn: (data: T)
|
|
|
990
1074
|
* SDK Request Options Types
|
|
991
1075
|
* These types define the options for SDK methods
|
|
992
1076
|
*/
|
|
1077
|
+
|
|
993
1078
|
/** Screenshot capture options */
|
|
994
1079
|
interface ScreenshotOptions {
|
|
995
1080
|
/** Enable screenshot capture */
|
|
@@ -1037,6 +1122,12 @@ interface ScrapeOptions {
|
|
|
1037
1122
|
headers?: Record<string, string>;
|
|
1038
1123
|
/** Timeout in milliseconds (default: 30000, max: 120000) */
|
|
1039
1124
|
timeout?: number;
|
|
1125
|
+
/** Convert content to Markdown (GFM) (default: false) */
|
|
1126
|
+
outputMarkdown?: boolean;
|
|
1127
|
+
/** Base URL for resolving relative URLs in markdown (defaults to scraped URL) */
|
|
1128
|
+
markdownBaseUrl?: string;
|
|
1129
|
+
/** Browser actions to execute before scraping (max: 20) */
|
|
1130
|
+
actions?: ActionInput[];
|
|
1040
1131
|
}
|
|
1041
1132
|
/** Individual URL item for batch scraping */
|
|
1042
1133
|
interface BatchScrapeItem {
|
|
@@ -1056,6 +1147,12 @@ interface BatchScrapeItem {
|
|
|
1056
1147
|
headers?: Record<string, string>;
|
|
1057
1148
|
/** Override timeout for this URL */
|
|
1058
1149
|
timeout?: number;
|
|
1150
|
+
/** Override outputMarkdown for this URL */
|
|
1151
|
+
outputMarkdown?: boolean;
|
|
1152
|
+
/** Override markdownBaseUrl for this URL */
|
|
1153
|
+
markdownBaseUrl?: string;
|
|
1154
|
+
/** Override actions for this URL */
|
|
1155
|
+
actions?: ActionInput[];
|
|
1059
1156
|
}
|
|
1060
1157
|
/** Default options applied to all URLs in a batch */
|
|
1061
1158
|
interface BatchScrapeDefaults {
|
|
@@ -1087,6 +1184,12 @@ interface BatchScrapeDefaults {
|
|
|
1087
1184
|
headers?: Record<string, string>;
|
|
1088
1185
|
/** Timeout in milliseconds */
|
|
1089
1186
|
timeout?: number;
|
|
1187
|
+
/** Convert content to Markdown (GFM) */
|
|
1188
|
+
outputMarkdown?: boolean;
|
|
1189
|
+
/** Base URL for resolving relative URLs in markdown */
|
|
1190
|
+
markdownBaseUrl?: string;
|
|
1191
|
+
/** Browser actions to execute before scraping */
|
|
1192
|
+
actions?: ActionInput[];
|
|
1090
1193
|
}
|
|
1091
1194
|
/** Options for batch scraping multiple URLs */
|
|
1092
1195
|
interface BatchScrapeOptions {
|
|
@@ -1211,6 +1314,14 @@ interface CrawlOptions {
|
|
|
1211
1314
|
webhookUrl?: string;
|
|
1212
1315
|
/** Auto-sync discovered deals to DealUp */
|
|
1213
1316
|
syncToDealup?: boolean;
|
|
1317
|
+
/** Don't save crawl results - Zero Data Retention (Pro/Enterprise only) */
|
|
1318
|
+
noStore?: boolean;
|
|
1319
|
+
/** Download and store product images to Supabase Storage (default: true) */
|
|
1320
|
+
downloadImages?: boolean;
|
|
1321
|
+
/** Maximum images per deal for gallery (default: 5, max: 10) */
|
|
1322
|
+
maxImagesPerDeal?: number;
|
|
1323
|
+
/** Image download timeout in ms (default: 10000) */
|
|
1324
|
+
imageDownloadTimeout?: number;
|
|
1214
1325
|
/** Site-specific config name from registry */
|
|
1215
1326
|
siteConfig?: string;
|
|
1216
1327
|
/** Job template to use (ecommerce, blog, docs, marketplace, custom) */
|
|
@@ -1219,6 +1330,10 @@ interface CrawlOptions {
|
|
|
1219
1330
|
useSmartRouting?: boolean;
|
|
1220
1331
|
/** Priority queue override (Enterprise only) */
|
|
1221
1332
|
priority?: CrawlPriority;
|
|
1333
|
+
/** Convert pages to Markdown (GFM) (default: false) */
|
|
1334
|
+
outputMarkdown?: boolean;
|
|
1335
|
+
/** Base URL for resolving relative URLs in markdown */
|
|
1336
|
+
markdownBaseUrl?: string;
|
|
1222
1337
|
}
|
|
1223
1338
|
/** Crawl template identifier */
|
|
1224
1339
|
type CrawlTemplateId = "ecommerce" | "blog" | "docs" | "marketplace" | "custom";
|
|
@@ -3113,9 +3228,12 @@ declare class DealCrawl {
|
|
|
3113
3228
|
*
|
|
3114
3229
|
* @example
|
|
3115
3230
|
* ```ts
|
|
3116
|
-
* //
|
|
3231
|
+
* // With API key
|
|
3117
3232
|
* const client = new DealCrawl({ apiKey: "sk_xxx" });
|
|
3118
3233
|
*
|
|
3234
|
+
* // With Client ID (for internal calls)
|
|
3235
|
+
* const client = new DealCrawl({ clientId: "uuid-xxx" });
|
|
3236
|
+
*
|
|
3119
3237
|
* // Full config
|
|
3120
3238
|
* const client = new DealCrawl({
|
|
3121
3239
|
* apiKey: "sk_xxx",
|
|
@@ -3510,4 +3628,4 @@ declare const ERROR_MESSAGES: Record<ErrorCode, string>;
|
|
|
3510
3628
|
*/
|
|
3511
3629
|
declare function getErrorMessage(code: ErrorCode): string;
|
|
3512
3630
|
|
|
3513
|
-
export { ALL_API_KEY_SCOPES, type AccountInfoResponse, type AccountMetricsResponse, AccountResource, type AgentAction, type AgentActionType, type AgentCompletionReason, type AgentJobResponse, type AgentModel, type AgentOptions, AgentResource, type AgentResultResponse, type AgentStatusResponse, type AgentStepResponse, type ApiError, type ApiKeyInfo, type ApiKeyScope, type ApiResponse, type BatchScrapeDefaults, type BatchScrapeItem, type BatchScrapeOptions, type BatchScrapeResponse, type BatchScrapeResultItem, type BatchStatusResponse, type CancelJobResponse, type CheckpointInfo, type ClickAction, type ClientPreferences, type ClientStatsResponse, type CrawlAnalysisResponse, type CrawlError, type CrawlJobResponse, type CrawlMode, type CrawlOptions, type CrawlRecommendation, CrawlResource, type CrawlResult, type CrawlStats, type CrawlTemplate, type CrawlTemplateId, type CreateApiKeyOptions, type CreateKeyResponse, type CreateWebhookOptions, type CreateWebhookResponse, type CreatedApiKey, DEFAULT_API_KEY_SCOPES, DEFAULT_CONFIG, DataResource, DealCrawl, type DealCrawlConfig, DealCrawlError, type DealDetails, type DealItem, type DealMetrics, type DealScoreSummary, type DealSummary, type DealUpMetrics, type DealUpMetricsResponse, type DeleteKeyResponse, type DeleteWebhookResponse, type DiscountSignal, type DorkJobResponse, type DorkOptions, DorkResource, type DorkResult, ERROR_CODES, ERROR_MESSAGES, type EngineSelection, type EngineType, type ErrorCode, type ExportDealsOptions, type ExportFormat, type ExportJobsOptions, type ExtractJobResponse, type ExtractModel, type ExtractOptions, ExtractResource, type ExtractedDeal, type ExtractionErrorDetails, type FallbackConfig, type FallbackMetadata, type FallbackResult, type FallbackSource, type GetApiKeyStatsOptions, type GetDealsOptions, type HoverAction, type JobDealsResponse, type JobMetricsResponse, type JobResponse, type JobStatus, type JobStatusFilter, type JobStatusResponse, type JobSummary, type JobTypeFilter, type KeyStatsResponse, KeysResource, type ListApiKeysOptions, type ListDealsOptions, type ListDealsResponse, type ListJobsOptions, type ListJobsResponse, type ListKeysResponse, type ListWebhooksResponse, type PaginatedResponse, type PaginationInfo, type ParsedPage, type PreferencesResponse, type PressAction, type PriceSignal, type PricingInfo, type ProductCategory, type ProductInfo, type RateLimitInfo, type RecommendationsResponse, type RequestContext, type ResumeJobResponse, type RevokeApiKeyOptions, type RotateApiKeyOptions, type RotateKeyResponse, type ScrapeJobResponse, type ScrapeOptions, ScrapeResource, type ScrapeResult, type ScreenshotAgentAction, type ScreenshotOptions, type ScreenshotResult, type ScrollAction, type SearchAiModel, type SearchAiProvider, type SearchData, type SearchDateRange, type SearchFilters, type SearchJobResponse, type SearchOptions, SearchResource, type SearchResultItem, type SearchStatusResponse, type SelectAction, type Signal, type SortOrder, StatusResource, type TestWebhookResponse, type UpdatePreferencesOptions, type UpdatePreferencesResponse, type UpdateWebhookOptions, type UpdateWebhookResponse, type UrgencyLevel, type UsageStats, type ValidationError, type ValidationResult, type WaitAction, type WaitOptions, type WaitResult, type WebhookEvent, type WebhookItem, WebhooksResource, type WriteAction, DealCrawl as default, getErrorMessage, pollUntil, waitForAll, waitForAny, waitForResult };
|
|
3631
|
+
export { ALL_API_KEY_SCOPES, type AccountInfoResponse, type AccountMetricsResponse, AccountResource, type ActionBaseOptions, type ActionInput, type AgentAction, type AgentActionType, type AgentCompletionReason, type AgentJobResponse, type AgentModel, type AgentOptions, AgentResource, type AgentResultResponse, type AgentStatusResponse, type AgentStepResponse, type ApiError, type ApiKeyInfo, type ApiKeyScope, type ApiResponse, type BatchScrapeDefaults, type BatchScrapeItem, type BatchScrapeOptions, type BatchScrapeResponse, type BatchScrapeResultItem, type BatchStatusResponse, type CancelJobResponse, type CheckpointInfo, type ClickAction, type ClientPreferences, type ClientStatsResponse, type CrawlAnalysisResponse, type CrawlError, type CrawlJobResponse, type CrawlMode, type CrawlOptions, type CrawlRecommendation, CrawlResource, type CrawlResult, type CrawlStats, type CrawlTemplate, type CrawlTemplateId, type CreateApiKeyOptions, type CreateKeyResponse, type CreateWebhookOptions, type CreateWebhookResponse, type CreatedApiKey, DEFAULT_API_KEY_SCOPES, DEFAULT_CONFIG, DataResource, DealCrawl, type DealCrawlConfig, DealCrawlError, type DealDetails, type DealItem, type DealMetrics, type DealScoreSummary, type DealSummary, type DealUpMetrics, type DealUpMetricsResponse, type DeleteKeyResponse, type DeleteWebhookResponse, type DiscountSignal, type DorkJobResponse, type DorkOptions, DorkResource, type DorkResult, ERROR_CODES, ERROR_MESSAGES, type EngineSelection, type EngineType, type ErrorCode, type ExportDealsOptions, type ExportFormat, type ExportJobsOptions, type ExtractJobResponse, type ExtractModel, type ExtractOptions, ExtractResource, type ExtractedDeal, type ExtractionErrorDetails, type FallbackConfig, type FallbackMetadata, type FallbackResult, type FallbackSource, type GetApiKeyStatsOptions, type GetDealsOptions, type HoverAction, type JobDealsResponse, type JobMetricsResponse, type JobResponse, type JobStatus, type JobStatusFilter, type JobStatusResponse, type JobSummary, type JobTypeFilter, type KeyStatsResponse, KeysResource, type ListApiKeysOptions, type ListDealsOptions, type ListDealsResponse, type ListJobsOptions, type ListJobsResponse, type ListKeysResponse, type ListWebhooksResponse, type PaginatedResponse, type PaginationInfo, type ParsedPage, type PreferencesResponse, type PressAction, type PriceSignal, type PricingInfo, type ProductCategory, type ProductInfo, type RateLimitInfo, type RecommendationsResponse, type RequestContext, type ResumeJobResponse, type RevokeApiKeyOptions, type RotateApiKeyOptions, type RotateKeyResponse, type ScrapeJobResponse, type ScrapeOptions, ScrapeResource, type ScrapeResult, type ScreenshotAction, type ScreenshotAgentAction, type ScreenshotOptions, type ScreenshotResult, type ScrollAction, type SearchAiModel, type SearchAiProvider, type SearchData, type SearchDateRange, type SearchFilters, type SearchJobResponse, type SearchOptions, SearchResource, type SearchResultItem, type SearchStatusResponse, type SelectAction, type ClickAction$1 as SharedClickAction, type HoverAction$1 as SharedHoverAction, type PressAction$1 as SharedPressAction, type ScrollAction$1 as SharedScrollAction, type SelectAction$1 as SharedSelectAction, type WaitAction$1 as SharedWaitAction, type WriteAction$1 as SharedWriteAction, type Signal, type SortOrder, StatusResource, type TestWebhookResponse, type UpdatePreferencesOptions, type UpdatePreferencesResponse, type UpdateWebhookOptions, type UpdateWebhookResponse, type UrgencyLevel, type UsageStats, type ValidationError, type ValidationResult, type WaitAction, type WaitOptions, type WaitResult, type WebhookEvent, type WebhookItem, WebhooksResource, type WriteAction, DealCrawl as default, getErrorMessage, pollUntil, waitForAll, waitForAny, waitForResult };
|
package/dist/index.js
CHANGED
|
@@ -477,12 +477,17 @@ async function request(ctx, path, options = {}) {
|
|
|
477
477
|
currentController = null;
|
|
478
478
|
};
|
|
479
479
|
try {
|
|
480
|
+
const headers = {
|
|
481
|
+
"Content-Type": "application/json"
|
|
482
|
+
};
|
|
483
|
+
if (ctx.apiKey) {
|
|
484
|
+
headers["Authorization"] = `Bearer ${ctx.apiKey}`;
|
|
485
|
+
} else if (ctx.clientId) {
|
|
486
|
+
headers["X-Client-ID"] = ctx.clientId;
|
|
487
|
+
}
|
|
480
488
|
const response = await fetch(url, {
|
|
481
489
|
method,
|
|
482
|
-
headers
|
|
483
|
-
"Content-Type": "application/json",
|
|
484
|
-
Authorization: `Bearer ${ctx.apiKey}`
|
|
485
|
-
},
|
|
490
|
+
headers,
|
|
486
491
|
body: body ? JSON.stringify(body) : void 0,
|
|
487
492
|
signal: controller.signal
|
|
488
493
|
});
|
|
@@ -2897,9 +2902,12 @@ var DealCrawl = class {
|
|
|
2897
2902
|
*
|
|
2898
2903
|
* @example
|
|
2899
2904
|
* ```ts
|
|
2900
|
-
* //
|
|
2905
|
+
* // With API key
|
|
2901
2906
|
* const client = new DealCrawl({ apiKey: "sk_xxx" });
|
|
2902
2907
|
*
|
|
2908
|
+
* // With Client ID (for internal calls)
|
|
2909
|
+
* const client = new DealCrawl({ clientId: "uuid-xxx" });
|
|
2910
|
+
*
|
|
2903
2911
|
* // Full config
|
|
2904
2912
|
* const client = new DealCrawl({
|
|
2905
2913
|
* apiKey: "sk_xxx",
|
|
@@ -2912,11 +2920,17 @@ var DealCrawl = class {
|
|
|
2912
2920
|
* ```
|
|
2913
2921
|
*/
|
|
2914
2922
|
constructor(config) {
|
|
2915
|
-
|
|
2916
|
-
|
|
2923
|
+
const hasApiKey = config.apiKey && config.apiKey.trim();
|
|
2924
|
+
const hasClientId = config.clientId && config.clientId.trim();
|
|
2925
|
+
if (!hasApiKey && !hasClientId) {
|
|
2926
|
+
throw new Error("Either API key or Client ID is required");
|
|
2927
|
+
}
|
|
2928
|
+
if (hasApiKey && hasClientId) {
|
|
2929
|
+
throw new Error("Provide either API key OR Client ID, not both");
|
|
2917
2930
|
}
|
|
2918
2931
|
this.ctx = {
|
|
2919
2932
|
apiKey: config.apiKey,
|
|
2933
|
+
clientId: config.clientId,
|
|
2920
2934
|
baseUrl: config.baseUrl ?? DEFAULT_CONFIG.baseUrl,
|
|
2921
2935
|
timeout: config.timeout ?? DEFAULT_CONFIG.timeout,
|
|
2922
2936
|
maxRetries: config.maxRetries ?? DEFAULT_CONFIG.maxRetries,
|