@dealcrawl/sdk 2.10.0 → 2.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +246 -37
- package/dist/index.d.mts +1174 -50
- package/dist/index.d.ts +1174 -50
- package/dist/index.js +668 -52
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +665 -53
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -1118,6 +1118,245 @@ var AgentResource = class {
|
|
|
1118
1118
|
}
|
|
1119
1119
|
};
|
|
1120
1120
|
|
|
1121
|
+
// src/resources/auth.ts
|
|
1122
|
+
var AuthResource = class {
|
|
1123
|
+
constructor(ctx) {
|
|
1124
|
+
this.ctx = ctx;
|
|
1125
|
+
}
|
|
1126
|
+
/**
|
|
1127
|
+
* Generate SSE authentication token
|
|
1128
|
+
*
|
|
1129
|
+
* Required for browser-based SSE connections because EventSource API
|
|
1130
|
+
* doesn't support custom headers. Token is short-lived (5 minutes).
|
|
1131
|
+
*
|
|
1132
|
+
* Security:
|
|
1133
|
+
* - Requires valid API key (Bearer token)
|
|
1134
|
+
* - Token expires in 5 minutes
|
|
1135
|
+
* - Token can be restricted to specific job
|
|
1136
|
+
* - Token stored in Redis (revocable)
|
|
1137
|
+
*
|
|
1138
|
+
* @example
|
|
1139
|
+
* ```ts
|
|
1140
|
+
* // 1. Generate token
|
|
1141
|
+
* const { token, expiresAt } = await client.auth.generateSSEToken();
|
|
1142
|
+
* console.log(`Token expires at: ${expiresAt}`);
|
|
1143
|
+
*
|
|
1144
|
+
* // 2. Use in browser EventSource
|
|
1145
|
+
* const eventSource = new EventSource(`/v1/events?token=${token}`);
|
|
1146
|
+
*
|
|
1147
|
+
* eventSource.addEventListener('job.completed', (event) => {
|
|
1148
|
+
* const data = JSON.parse(event.data);
|
|
1149
|
+
* console.log('Job completed:', data);
|
|
1150
|
+
* });
|
|
1151
|
+
*
|
|
1152
|
+
* // 3. For specific job only
|
|
1153
|
+
* const jobToken = await client.auth.generateSSEToken({ jobId: "job_abc123" });
|
|
1154
|
+
* const jobEvents = new EventSource(`/v1/events/job_abc123?token=${jobToken.token}`);
|
|
1155
|
+
* ```
|
|
1156
|
+
*/
|
|
1157
|
+
async generateSSEToken(options) {
|
|
1158
|
+
const result = await post(
|
|
1159
|
+
this.ctx,
|
|
1160
|
+
"/v1/auth/sse-token",
|
|
1161
|
+
options ?? {}
|
|
1162
|
+
);
|
|
1163
|
+
return result.data;
|
|
1164
|
+
}
|
|
1165
|
+
/**
|
|
1166
|
+
* Get SSE connection limits for current tier
|
|
1167
|
+
*
|
|
1168
|
+
* Shows how many concurrent SSE connections are allowed
|
|
1169
|
+
* and how many are currently active.
|
|
1170
|
+
*
|
|
1171
|
+
* Tier limits:
|
|
1172
|
+
* - Free: 10 concurrent connections
|
|
1173
|
+
* - Pro: 50 concurrent connections
|
|
1174
|
+
* - Enterprise: 200 concurrent connections
|
|
1175
|
+
*
|
|
1176
|
+
* @example
|
|
1177
|
+
* ```ts
|
|
1178
|
+
* const limits = await client.auth.getLimits();
|
|
1179
|
+
*
|
|
1180
|
+
* console.log(`Tier: ${limits.tier}`);
|
|
1181
|
+
* console.log(`Max connections: ${limits.sse.maxConnections}`);
|
|
1182
|
+
* console.log(`Current connections: ${limits.sse.currentConnections}`);
|
|
1183
|
+
* console.log(`Available: ${limits.sse.available}`);
|
|
1184
|
+
*
|
|
1185
|
+
* // Check before opening new connection
|
|
1186
|
+
* if (limits.sse.available > 0) {
|
|
1187
|
+
* const token = await client.auth.generateSSEToken();
|
|
1188
|
+
* const eventSource = new EventSource(`/v1/events?token=${token.token}`);
|
|
1189
|
+
* } else {
|
|
1190
|
+
* console.error('No available SSE connection slots');
|
|
1191
|
+
* }
|
|
1192
|
+
* ```
|
|
1193
|
+
*/
|
|
1194
|
+
async getLimits() {
|
|
1195
|
+
const result = await get(
|
|
1196
|
+
this.ctx,
|
|
1197
|
+
"/v1/auth/limits"
|
|
1198
|
+
);
|
|
1199
|
+
return result.data;
|
|
1200
|
+
}
|
|
1201
|
+
};
|
|
1202
|
+
|
|
1203
|
+
// src/resources/convert.ts
|
|
1204
|
+
var ConvertResource = class {
|
|
1205
|
+
constructor(ctx) {
|
|
1206
|
+
this.ctx = ctx;
|
|
1207
|
+
}
|
|
1208
|
+
/**
|
|
1209
|
+
* Convert HTML to Markdown
|
|
1210
|
+
*
|
|
1211
|
+
* Transforms raw HTML content into clean, readable Markdown using GitHub Flavored Markdown (GFM).
|
|
1212
|
+
* Useful for:
|
|
1213
|
+
* - Converting scraped HTML to markdown for LLM processing
|
|
1214
|
+
* - Cleaning up messy HTML from web pages
|
|
1215
|
+
* - Extracting main content while removing noise (ads, nav, footer)
|
|
1216
|
+
* - Creating documentation from HTML sources
|
|
1217
|
+
*
|
|
1218
|
+
* Features:
|
|
1219
|
+
* - GFM table, strikethrough, and task list support
|
|
1220
|
+
* - Automatic noise removal (scripts, ads, navigation)
|
|
1221
|
+
* - Relative URL resolution
|
|
1222
|
+
* - Custom element exclusion via CSS selectors
|
|
1223
|
+
* - Output length limiting
|
|
1224
|
+
*
|
|
1225
|
+
* @param options - Conversion options
|
|
1226
|
+
* @returns Conversion result with markdown, metadata, and warnings
|
|
1227
|
+
*
|
|
1228
|
+
* @example Basic usage
|
|
1229
|
+
* ```ts
|
|
1230
|
+
* const result = await client.convert.htmlToMarkdown({
|
|
1231
|
+
* html: "<h1>Product</h1><p>Price: $99</p>"
|
|
1232
|
+
* });
|
|
1233
|
+
* console.log(result.data.markdown);
|
|
1234
|
+
* ```
|
|
1235
|
+
*
|
|
1236
|
+
* @example With all options
|
|
1237
|
+
* ```ts
|
|
1238
|
+
* const result = await client.convert.htmlToMarkdown({
|
|
1239
|
+
* html: htmlContent,
|
|
1240
|
+
* baseUrl: "https://shop.example.com",
|
|
1241
|
+
* options: {
|
|
1242
|
+
* gfmTables: true,
|
|
1243
|
+
* removeNoise: true,
|
|
1244
|
+
* excludeSelectors: [".advertisement", "#sidebar"],
|
|
1245
|
+
* absoluteUrls: true,
|
|
1246
|
+
* maxLength: 100000,
|
|
1247
|
+
* includeImages: true,
|
|
1248
|
+
* includeLinks: true
|
|
1249
|
+
* }
|
|
1250
|
+
* });
|
|
1251
|
+
*
|
|
1252
|
+
* // Check metadata
|
|
1253
|
+
* console.log(`Words: ${result.data.metadata.wordCount}`);
|
|
1254
|
+
* console.log(`Links: ${result.data.metadata.linkCount}`);
|
|
1255
|
+
* console.log(`Images: ${result.data.metadata.imageCount}`);
|
|
1256
|
+
* console.log(`Conversion time: ${result.data.metadata.conversionTimeMs}ms`);
|
|
1257
|
+
*
|
|
1258
|
+
* // Check for warnings
|
|
1259
|
+
* if (result.data.warnings?.length) {
|
|
1260
|
+
* console.warn("Conversion warnings:", result.data.warnings);
|
|
1261
|
+
* }
|
|
1262
|
+
* ```
|
|
1263
|
+
*
|
|
1264
|
+
* @example Converting scraped HTML
|
|
1265
|
+
* ```ts
|
|
1266
|
+
* // First scrape a page
|
|
1267
|
+
* const scrapeJob = await client.scrape.create({
|
|
1268
|
+
* url: "https://example.com/article"
|
|
1269
|
+
* });
|
|
1270
|
+
* const scrapeResult = await client.waitForResult(scrapeJob.jobId);
|
|
1271
|
+
*
|
|
1272
|
+
* // Then convert HTML to markdown
|
|
1273
|
+
* const markdown = await client.convert.htmlToMarkdown({
|
|
1274
|
+
* html: scrapeResult.data.html,
|
|
1275
|
+
* baseUrl: scrapeResult.data.url,
|
|
1276
|
+
* options: {
|
|
1277
|
+
* removeNoise: true,
|
|
1278
|
+
* onlyMainContent: true
|
|
1279
|
+
* }
|
|
1280
|
+
* });
|
|
1281
|
+
* ```
|
|
1282
|
+
*/
|
|
1283
|
+
async htmlToMarkdown(options) {
|
|
1284
|
+
const body = {
|
|
1285
|
+
html: options.html,
|
|
1286
|
+
baseUrl: options.baseUrl,
|
|
1287
|
+
options: options.options
|
|
1288
|
+
};
|
|
1289
|
+
const result = await post(this.ctx, "/v1/convert", body);
|
|
1290
|
+
return result.data;
|
|
1291
|
+
}
|
|
1292
|
+
/**
|
|
1293
|
+
* Alias for htmlToMarkdown() for convenience
|
|
1294
|
+
*
|
|
1295
|
+
* @example
|
|
1296
|
+
* ```ts
|
|
1297
|
+
* const result = await client.convert.toMarkdown({
|
|
1298
|
+
* html: "<h1>Hello</h1>"
|
|
1299
|
+
* });
|
|
1300
|
+
* ```
|
|
1301
|
+
*/
|
|
1302
|
+
async toMarkdown(options) {
|
|
1303
|
+
return this.htmlToMarkdown(options);
|
|
1304
|
+
}
|
|
1305
|
+
/**
|
|
1306
|
+
* Convert HTML with minimal options (just the HTML content)
|
|
1307
|
+
* Uses all default settings
|
|
1308
|
+
*
|
|
1309
|
+
* @param html - HTML content to convert
|
|
1310
|
+
* @param baseUrl - Optional base URL for resolving relative links
|
|
1311
|
+
* @returns Conversion result
|
|
1312
|
+
*
|
|
1313
|
+
* @example
|
|
1314
|
+
* ```ts
|
|
1315
|
+
* const result = await client.convert.quick(
|
|
1316
|
+
* "<h1>Title</h1><p>Content</p>",
|
|
1317
|
+
* "https://example.com"
|
|
1318
|
+
* );
|
|
1319
|
+
* console.log(result.data.markdown);
|
|
1320
|
+
* ```
|
|
1321
|
+
*/
|
|
1322
|
+
async quick(html, baseUrl) {
|
|
1323
|
+
return this.htmlToMarkdown({ html, baseUrl });
|
|
1324
|
+
}
|
|
1325
|
+
/**
|
|
1326
|
+
* Convert HTML with noise removal enabled
|
|
1327
|
+
* Removes navigation, footer, ads, scripts, and other clutter
|
|
1328
|
+
*
|
|
1329
|
+
* @param html - HTML content to convert
|
|
1330
|
+
* @param baseUrl - Optional base URL
|
|
1331
|
+
* @returns Conversion result with clean markdown
|
|
1332
|
+
*
|
|
1333
|
+
* @example
|
|
1334
|
+
* ```ts
|
|
1335
|
+
* // Extract just the main content from a messy page
|
|
1336
|
+
* const result = await client.convert.clean(messyHtml, "https://example.com");
|
|
1337
|
+
* console.log(result.data.markdown); // Clean, readable markdown
|
|
1338
|
+
* ```
|
|
1339
|
+
*/
|
|
1340
|
+
async clean(html, baseUrl) {
|
|
1341
|
+
return this.htmlToMarkdown({
|
|
1342
|
+
html,
|
|
1343
|
+
baseUrl,
|
|
1344
|
+
options: {
|
|
1345
|
+
removeNoise: true,
|
|
1346
|
+
excludeSelectors: [
|
|
1347
|
+
"nav",
|
|
1348
|
+
"footer",
|
|
1349
|
+
"aside",
|
|
1350
|
+
".advertisement",
|
|
1351
|
+
".ad",
|
|
1352
|
+
".sidebar",
|
|
1353
|
+
"#comments"
|
|
1354
|
+
]
|
|
1355
|
+
}
|
|
1356
|
+
});
|
|
1357
|
+
}
|
|
1358
|
+
};
|
|
1359
|
+
|
|
1121
1360
|
// src/resources/crawl.ts
|
|
1122
1361
|
var CRAWL_TEMPLATES = {
|
|
1123
1362
|
ecommerce: {
|
|
@@ -1290,12 +1529,10 @@ var CrawlResource = class {
|
|
|
1290
1529
|
* ```
|
|
1291
1530
|
*/
|
|
1292
1531
|
async analyze(url) {
|
|
1293
|
-
const result = await
|
|
1532
|
+
const result = await post(
|
|
1294
1533
|
this.ctx,
|
|
1295
1534
|
"/v1/crawl/analyze",
|
|
1296
|
-
{
|
|
1297
|
-
url
|
|
1298
|
-
}
|
|
1535
|
+
{ url }
|
|
1299
1536
|
);
|
|
1300
1537
|
return result.data;
|
|
1301
1538
|
}
|
|
@@ -1461,31 +1698,28 @@ var DataResource = class {
|
|
|
1461
1698
|
});
|
|
1462
1699
|
}
|
|
1463
1700
|
/**
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
)
|
|
1474
|
-
|
|
1475
|
-
throw new Error("category is required and cannot be empty");
|
|
1476
|
-
}
|
|
1477
|
-
return this.listDeals({ category, ...options });
|
|
1478
|
-
} return this.listDeals({ category, ...options });
|
|
1701
|
+
* Get deals by category
|
|
1702
|
+
* Convenience method for filtering by category
|
|
1703
|
+
*
|
|
1704
|
+
* @example
|
|
1705
|
+
* ```ts
|
|
1706
|
+
* const electronicsDeals = await client.data.getDealsByCategory("electronics");
|
|
1707
|
+
* ```
|
|
1708
|
+
*/
|
|
1709
|
+
async getDealsByCategory(category, options) {
|
|
1710
|
+
if (!category || !category.trim()) {
|
|
1711
|
+
throw new Error("category is required and cannot be empty");
|
|
1479
1712
|
}
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1713
|
+
return this.listDeals({ category, ...options });
|
|
1714
|
+
}
|
|
1715
|
+
/**
|
|
1716
|
+
* Get unsynced deals (not yet sent to DealUp)
|
|
1717
|
+
*
|
|
1718
|
+
* @example
|
|
1719
|
+
* ```ts
|
|
1720
|
+
* const unsyncedDeals = await client.data.getUnsyncedDeals();
|
|
1721
|
+
* ```
|
|
1722
|
+
*/
|
|
1489
1723
|
async getUnsyncedDeals(options) {
|
|
1490
1724
|
return this.listDeals({ synced: false, ...options });
|
|
1491
1725
|
}
|
|
@@ -1541,9 +1775,11 @@ var DataResource = class {
|
|
|
1541
1775
|
{
|
|
1542
1776
|
format: options.format || "json",
|
|
1543
1777
|
minScore: options.minScore,
|
|
1544
|
-
maxPrice: options.maxPrice,
|
|
1545
1778
|
category: options.category,
|
|
1546
|
-
|
|
1779
|
+
synced: options.synced?.toString(),
|
|
1780
|
+
fromDate: options.fromDate,
|
|
1781
|
+
toDate: options.toDate,
|
|
1782
|
+
limit: options.limit
|
|
1547
1783
|
}
|
|
1548
1784
|
);
|
|
1549
1785
|
return result.data;
|
|
@@ -1588,8 +1824,8 @@ var DorkResource = class {
|
|
|
1588
1824
|
async create(options) {
|
|
1589
1825
|
const body = {
|
|
1590
1826
|
query: this.buildQuery(options),
|
|
1591
|
-
|
|
1592
|
-
|
|
1827
|
+
site: options.site,
|
|
1828
|
+
maxResults: options.maxResults
|
|
1593
1829
|
};
|
|
1594
1830
|
const result = await post(this.ctx, "/v1/dork", body);
|
|
1595
1831
|
return result.data;
|
|
@@ -1704,9 +1940,6 @@ var DorkResource = class {
|
|
|
1704
1940
|
if (typeof options.query === "string" && options.query.trim() !== "") {
|
|
1705
1941
|
parts.push(options.query);
|
|
1706
1942
|
}
|
|
1707
|
-
if (options.site) {
|
|
1708
|
-
parts.push(`site:${options.site}`);
|
|
1709
|
-
}
|
|
1710
1943
|
if (options.fileType) {
|
|
1711
1944
|
parts.push(`filetype:${options.fileType}`);
|
|
1712
1945
|
}
|
|
@@ -1720,6 +1953,236 @@ var DorkResource = class {
|
|
|
1720
1953
|
}
|
|
1721
1954
|
};
|
|
1722
1955
|
|
|
1956
|
+
// src/resources/events.ts
|
|
1957
|
+
var EventsResource = class {
|
|
1958
|
+
constructor(ctx) {
|
|
1959
|
+
this.ctx = ctx;
|
|
1960
|
+
}
|
|
1961
|
+
/**
|
|
1962
|
+
* Subscribe to all events for authenticated client
|
|
1963
|
+
*
|
|
1964
|
+
* Opens an SSE connection to receive real-time events for all jobs.
|
|
1965
|
+
* Requires an SSE token obtained via client.auth.generateSSEToken().
|
|
1966
|
+
*
|
|
1967
|
+
* Event Types:
|
|
1968
|
+
* - Job lifecycle: job.created, job.queued, job.started, job.progress,
|
|
1969
|
+
* job.completed, job.failed, job.cancelled
|
|
1970
|
+
* - Job details: job.log, job.metric, job.alert, job.checkpoint
|
|
1971
|
+
* - Deals: deal.found, deal.validated
|
|
1972
|
+
* - System: ping, connection.open, connection.close, error
|
|
1973
|
+
*
|
|
1974
|
+
* Features:
|
|
1975
|
+
* - Automatic reconnection on disconnect
|
|
1976
|
+
* - Event replay via Last-Event-ID
|
|
1977
|
+
* - Keepalive pings every 15 seconds
|
|
1978
|
+
* - Max connection time: 1 hour
|
|
1979
|
+
*
|
|
1980
|
+
* @param token - SSE authentication token from client.auth.generateSSEToken()
|
|
1981
|
+
* @param options - Subscription options (callbacks, reconnection settings)
|
|
1982
|
+
*
|
|
1983
|
+
* @example
|
|
1984
|
+
* ```ts
|
|
1985
|
+
* // Generate token
|
|
1986
|
+
* const { token } = await client.auth.generateSSEToken();
|
|
1987
|
+
*
|
|
1988
|
+
* // Subscribe with event handlers
|
|
1989
|
+
* const eventSource = client.events.subscribe(token, {
|
|
1990
|
+
* onEvent: (event) => {
|
|
1991
|
+
* // Handle all events
|
|
1992
|
+
* console.log('Event:', event.type);
|
|
1993
|
+
* const data = JSON.parse(event.data);
|
|
1994
|
+
*
|
|
1995
|
+
* if (data.jobId) {
|
|
1996
|
+
* console.log(`Job ${data.jobId}:`, data);
|
|
1997
|
+
* }
|
|
1998
|
+
* },
|
|
1999
|
+
* onError: (error) => {
|
|
2000
|
+
* console.error('SSE error:', error);
|
|
2001
|
+
* },
|
|
2002
|
+
* onOpen: () => {
|
|
2003
|
+
* console.log('SSE connection opened');
|
|
2004
|
+
* }
|
|
2005
|
+
* });
|
|
2006
|
+
*
|
|
2007
|
+
* // Listen for specific event types
|
|
2008
|
+
* eventSource.addEventListener('job.completed', (event) => {
|
|
2009
|
+
* const data = JSON.parse(event.data);
|
|
2010
|
+
* console.log('Job completed:', data);
|
|
2011
|
+
* });
|
|
2012
|
+
*
|
|
2013
|
+
* // Clean up
|
|
2014
|
+
* eventSource.close();
|
|
2015
|
+
* ```
|
|
2016
|
+
*/
|
|
2017
|
+
subscribe(token, options) {
|
|
2018
|
+
if (typeof EventSource === "undefined") {
|
|
2019
|
+
throw new Error(
|
|
2020
|
+
"EventSource is not available. SSE subscriptions only work in browsers. For Node.js, use polling via client.status.get() instead."
|
|
2021
|
+
);
|
|
2022
|
+
}
|
|
2023
|
+
const url = new URL("/v1/events", this.ctx.baseUrl);
|
|
2024
|
+
url.searchParams.set("token", token);
|
|
2025
|
+
const eventSource = new EventSource(url.toString());
|
|
2026
|
+
if (options?.onEvent) {
|
|
2027
|
+
eventSource.onmessage = options.onEvent;
|
|
2028
|
+
}
|
|
2029
|
+
if (options?.onError) {
|
|
2030
|
+
eventSource.onerror = (event) => {
|
|
2031
|
+
options.onError(
|
|
2032
|
+
new Error("SSE connection error. Will auto-reconnect if enabled.")
|
|
2033
|
+
);
|
|
2034
|
+
};
|
|
2035
|
+
}
|
|
2036
|
+
if (options?.onOpen) {
|
|
2037
|
+
eventSource.onopen = options.onOpen;
|
|
2038
|
+
}
|
|
2039
|
+
return eventSource;
|
|
2040
|
+
}
|
|
2041
|
+
/**
|
|
2042
|
+
* Subscribe to events for a specific job
|
|
2043
|
+
*
|
|
2044
|
+
* Opens an SSE connection filtered to a single job.
|
|
2045
|
+
* More efficient than global subscription when tracking one job.
|
|
2046
|
+
*
|
|
2047
|
+
* @param jobId - Job ID to subscribe to
|
|
2048
|
+
* @param token - SSE authentication token
|
|
2049
|
+
* @param options - Subscription options
|
|
2050
|
+
*
|
|
2051
|
+
* @example
|
|
2052
|
+
* ```ts
|
|
2053
|
+
* // Start a scrape job
|
|
2054
|
+
* const job = await client.scrape.create({ url: "https://example.com" });
|
|
2055
|
+
*
|
|
2056
|
+
* // Generate SSE token for this job
|
|
2057
|
+
* const { token } = await client.auth.generateSSEToken({ jobId: job.jobId });
|
|
2058
|
+
*
|
|
2059
|
+
* // Subscribe to job events
|
|
2060
|
+
* const eventSource = client.events.subscribeToJob(job.jobId, token, {
|
|
2061
|
+
* onEvent: (event) => {
|
|
2062
|
+
* const data = JSON.parse(event.data);
|
|
2063
|
+
* console.log(`[${event.type}]`, data);
|
|
2064
|
+
* }
|
|
2065
|
+
* });
|
|
2066
|
+
*
|
|
2067
|
+
* // Listen for completion
|
|
2068
|
+
* eventSource.addEventListener('job.completed', (event) => {
|
|
2069
|
+
* const data = JSON.parse(event.data);
|
|
2070
|
+
* console.log('Scrape completed!', data.summary);
|
|
2071
|
+
* eventSource.close();
|
|
2072
|
+
* });
|
|
2073
|
+
*
|
|
2074
|
+
* // Listen for progress
|
|
2075
|
+
* eventSource.addEventListener('job.progress', (event) => {
|
|
2076
|
+
* const data = JSON.parse(event.data);
|
|
2077
|
+
* console.log(`Progress: ${data.progress}%`);
|
|
2078
|
+
* });
|
|
2079
|
+
*
|
|
2080
|
+
* // Listen for errors
|
|
2081
|
+
* eventSource.addEventListener('job.failed', (event) => {
|
|
2082
|
+
* const data = JSON.parse(event.data);
|
|
2083
|
+
* console.error('Job failed:', data.error);
|
|
2084
|
+
* eventSource.close();
|
|
2085
|
+
* });
|
|
2086
|
+
* ```
|
|
2087
|
+
*/
|
|
2088
|
+
subscribeToJob(jobId, token, options) {
|
|
2089
|
+
if (typeof EventSource === "undefined") {
|
|
2090
|
+
throw new Error(
|
|
2091
|
+
"EventSource is not available. SSE subscriptions only work in browsers. For Node.js, use polling via client.status.get() instead."
|
|
2092
|
+
);
|
|
2093
|
+
}
|
|
2094
|
+
const url = new URL(`/v1/events/${jobId}`, this.ctx.baseUrl);
|
|
2095
|
+
url.searchParams.set("token", token);
|
|
2096
|
+
const eventSource = new EventSource(url.toString());
|
|
2097
|
+
if (options?.onEvent) {
|
|
2098
|
+
eventSource.onmessage = options.onEvent;
|
|
2099
|
+
}
|
|
2100
|
+
if (options?.onError) {
|
|
2101
|
+
eventSource.onerror = (event) => {
|
|
2102
|
+
options.onError(
|
|
2103
|
+
new Error("SSE connection error. Will auto-reconnect if enabled.")
|
|
2104
|
+
);
|
|
2105
|
+
};
|
|
2106
|
+
}
|
|
2107
|
+
if (options?.onOpen) {
|
|
2108
|
+
eventSource.onopen = options.onOpen;
|
|
2109
|
+
}
|
|
2110
|
+
return eventSource;
|
|
2111
|
+
}
|
|
2112
|
+
/**
|
|
2113
|
+
* Helper: Wait for job completion via SSE
|
|
2114
|
+
*
|
|
2115
|
+
* Convenience method that subscribes to a job and resolves when complete.
|
|
2116
|
+
* Automatically handles token generation and cleanup.
|
|
2117
|
+
*
|
|
2118
|
+
* @param jobId - Job ID to wait for
|
|
2119
|
+
* @param onProgress - Optional progress callback
|
|
2120
|
+
*
|
|
2121
|
+
* @example
|
|
2122
|
+
* ```ts
|
|
2123
|
+
* const job = await client.scrape.create({ url: "https://example.com" });
|
|
2124
|
+
*
|
|
2125
|
+
* // Wait for completion with progress updates
|
|
2126
|
+
* const result = await client.events.waitForCompletion(job.jobId, (progress) => {
|
|
2127
|
+
* console.log(`Progress: ${progress}%`);
|
|
2128
|
+
* });
|
|
2129
|
+
*
|
|
2130
|
+
* console.log('Job completed:', result);
|
|
2131
|
+
* ```
|
|
2132
|
+
*/
|
|
2133
|
+
async waitForCompletion(jobId, onProgress) {
|
|
2134
|
+
if (typeof EventSource === "undefined") {
|
|
2135
|
+
throw new Error(
|
|
2136
|
+
"waitForCompletion() only works in browsers. For Node.js, use client.waitForResult() instead."
|
|
2137
|
+
);
|
|
2138
|
+
}
|
|
2139
|
+
return new Promise(async (resolve, reject) => {
|
|
2140
|
+
const tokenResponse = await fetch(
|
|
2141
|
+
`${this.ctx.baseUrl}/v1/auth/sse-token`,
|
|
2142
|
+
{
|
|
2143
|
+
method: "POST",
|
|
2144
|
+
headers: {
|
|
2145
|
+
Authorization: `Bearer ${this.ctx.apiKey}`,
|
|
2146
|
+
"Content-Type": "application/json"
|
|
2147
|
+
},
|
|
2148
|
+
body: JSON.stringify({ jobId })
|
|
2149
|
+
}
|
|
2150
|
+
);
|
|
2151
|
+
if (!tokenResponse.ok) {
|
|
2152
|
+
reject(new Error("Failed to generate SSE token"));
|
|
2153
|
+
return;
|
|
2154
|
+
}
|
|
2155
|
+
const { token } = await tokenResponse.json();
|
|
2156
|
+
const eventSource = this.subscribeToJob(jobId, token, {
|
|
2157
|
+
onError: (error) => {
|
|
2158
|
+
eventSource.close();
|
|
2159
|
+
reject(error);
|
|
2160
|
+
}
|
|
2161
|
+
});
|
|
2162
|
+
eventSource.addEventListener("job.progress", (event) => {
|
|
2163
|
+
const data = JSON.parse(event.data);
|
|
2164
|
+
if (onProgress) {
|
|
2165
|
+
onProgress(data.progress);
|
|
2166
|
+
}
|
|
2167
|
+
});
|
|
2168
|
+
eventSource.addEventListener("job.completed", (event) => {
|
|
2169
|
+
const data = JSON.parse(event.data);
|
|
2170
|
+
eventSource.close();
|
|
2171
|
+
resolve(data);
|
|
2172
|
+
});
|
|
2173
|
+
eventSource.addEventListener("job.failed", (event) => {
|
|
2174
|
+
const data = JSON.parse(event.data);
|
|
2175
|
+
eventSource.close();
|
|
2176
|
+
reject(new Error(data.error?.message || "Job failed"));
|
|
2177
|
+
});
|
|
2178
|
+
eventSource.addEventListener("job.cancelled", (event) => {
|
|
2179
|
+
eventSource.close();
|
|
2180
|
+
reject(new Error("Job was cancelled"));
|
|
2181
|
+
});
|
|
2182
|
+
});
|
|
2183
|
+
}
|
|
2184
|
+
};
|
|
2185
|
+
|
|
1723
2186
|
// src/resources/extract.ts
|
|
1724
2187
|
var ExtractResource = class {
|
|
1725
2188
|
constructor(ctx) {
|
|
@@ -2101,10 +2564,7 @@ var KeysResource = class {
|
|
|
2101
2564
|
* ```
|
|
2102
2565
|
*/
|
|
2103
2566
|
async revokeAll() {
|
|
2104
|
-
const result = await
|
|
2105
|
-
this.ctx,
|
|
2106
|
-
"/v1/keys/all"
|
|
2107
|
-
);
|
|
2567
|
+
const result = await post(this.ctx, "/v1/keys/revoke-all");
|
|
2108
2568
|
return result.data;
|
|
2109
2569
|
}
|
|
2110
2570
|
/**
|
|
@@ -2160,6 +2620,7 @@ var ScrapeResource = class {
|
|
|
2160
2620
|
async create(options) {
|
|
2161
2621
|
const body = {
|
|
2162
2622
|
url: options.url,
|
|
2623
|
+
noStore: options.noStore,
|
|
2163
2624
|
detectSignals: options.detectSignals ?? true,
|
|
2164
2625
|
extractWithAI: options.extractWithAI,
|
|
2165
2626
|
extractDeal: options.extractDeal,
|
|
@@ -2172,7 +2633,10 @@ var ScrapeResource = class {
|
|
|
2172
2633
|
excludeSelectors: options.excludeSelectors,
|
|
2173
2634
|
onlyMainContent: options.onlyMainContent,
|
|
2174
2635
|
headers: options.headers,
|
|
2175
|
-
timeout: options.timeout
|
|
2636
|
+
timeout: options.timeout,
|
|
2637
|
+
outputMarkdown: options.outputMarkdown,
|
|
2638
|
+
markdownBaseUrl: options.markdownBaseUrl,
|
|
2639
|
+
actions: options.actions
|
|
2176
2640
|
};
|
|
2177
2641
|
const result = await post(this.ctx, "/v1/scrape", body);
|
|
2178
2642
|
return result.data;
|
|
@@ -2248,7 +2712,8 @@ var ScrapeResource = class {
|
|
|
2248
2712
|
* { url: "https://shop1.com/product1" },
|
|
2249
2713
|
* { url: "https://shop2.com/deal", extractDeal: true }
|
|
2250
2714
|
* ],
|
|
2251
|
-
* defaults: { detectSignals: true }
|
|
2715
|
+
* defaults: { detectSignals: true },
|
|
2716
|
+
* ignoreInvalidURLs: true
|
|
2252
2717
|
* });
|
|
2253
2718
|
* console.log(batch.batchId, batch.results);
|
|
2254
2719
|
* ```
|
|
@@ -2259,7 +2724,8 @@ var ScrapeResource = class {
|
|
|
2259
2724
|
defaults: options.defaults,
|
|
2260
2725
|
webhookUrl: options.webhookUrl,
|
|
2261
2726
|
priority: options.priority,
|
|
2262
|
-
delayMs: options.
|
|
2727
|
+
delayMs: options.delayMs,
|
|
2728
|
+
ignoreInvalidURLs: options.ignoreInvalidURLs
|
|
2263
2729
|
};
|
|
2264
2730
|
const result = await post(
|
|
2265
2731
|
this.ctx,
|
|
@@ -2309,6 +2775,57 @@ var ScrapeResource = class {
|
|
|
2309
2775
|
}
|
|
2310
2776
|
};
|
|
2311
2777
|
|
|
2778
|
+
// src/resources/screenshots.ts
|
|
2779
|
+
var ScreenshotsResource = class {
|
|
2780
|
+
constructor(ctx) {
|
|
2781
|
+
this.ctx = ctx;
|
|
2782
|
+
}
|
|
2783
|
+
/**
|
|
2784
|
+
* Refresh a signed URL before expiration
|
|
2785
|
+
*
|
|
2786
|
+
* @example
|
|
2787
|
+
* ```ts
|
|
2788
|
+
* const refreshed = await client.screenshots.refresh({
|
|
2789
|
+
* path: "job_abc123/1234567890_nanoid_example.png",
|
|
2790
|
+
* ttl: 604800 // 7 days
|
|
2791
|
+
* });
|
|
2792
|
+
* console.log(refreshed.url); // New signed URL
|
|
2793
|
+
* console.log(refreshed.expiresAt); // "2026-01-25T12:00:00Z"
|
|
2794
|
+
* console.log(refreshed.tierLimits); // { min: 3600, max: 604800, default: 604800 }
|
|
2795
|
+
* ```
|
|
2796
|
+
*/
|
|
2797
|
+
async refresh(options) {
|
|
2798
|
+
const result = await post(
|
|
2799
|
+
this.ctx,
|
|
2800
|
+
"/v1/screenshots/refresh",
|
|
2801
|
+
{
|
|
2802
|
+
path: options.path,
|
|
2803
|
+
ttl: options.ttl,
|
|
2804
|
+
bucket: options.bucket
|
|
2805
|
+
}
|
|
2806
|
+
);
|
|
2807
|
+
return result.data;
|
|
2808
|
+
}
|
|
2809
|
+
/**
|
|
2810
|
+
* Get TTL limits for the current tier
|
|
2811
|
+
*
|
|
2812
|
+
* @example
|
|
2813
|
+
* ```ts
|
|
2814
|
+
* const limits = await client.screenshots.getLimits();
|
|
2815
|
+
* console.log(limits.tier); // "pro"
|
|
2816
|
+
* console.log(limits.limits.max); // 604800 (7 days in seconds)
|
|
2817
|
+
* console.log(limits.formattedLimits.max); // "7 days"
|
|
2818
|
+
* ```
|
|
2819
|
+
*/
|
|
2820
|
+
async getLimits() {
|
|
2821
|
+
const result = await get(
|
|
2822
|
+
this.ctx,
|
|
2823
|
+
"/v1/screenshots/limits"
|
|
2824
|
+
);
|
|
2825
|
+
return result.data;
|
|
2826
|
+
}
|
|
2827
|
+
};
|
|
2828
|
+
|
|
2312
2829
|
// src/resources/search.ts
|
|
2313
2830
|
var SearchResource = class {
|
|
2314
2831
|
constructor(ctx) {
|
|
@@ -2321,7 +2838,7 @@ var SearchResource = class {
|
|
|
2321
2838
|
* ```ts
|
|
2322
2839
|
* const result = await client.search.create({
|
|
2323
2840
|
* query: "laptop deals black friday",
|
|
2324
|
-
*
|
|
2841
|
+
* limit: 20,
|
|
2325
2842
|
* useDealScoring: true
|
|
2326
2843
|
* });
|
|
2327
2844
|
* ```
|
|
@@ -2329,9 +2846,9 @@ var SearchResource = class {
|
|
|
2329
2846
|
async create(options) {
|
|
2330
2847
|
const body = {
|
|
2331
2848
|
query: options.query,
|
|
2332
|
-
limit: options.
|
|
2333
|
-
scrapeResults: options.
|
|
2334
|
-
maxScrapeResults: options.
|
|
2849
|
+
limit: options.limit,
|
|
2850
|
+
scrapeResults: options.scrapeResults,
|
|
2851
|
+
maxScrapeResults: options.maxScrapeResults,
|
|
2335
2852
|
useAiOptimization: options.useAiOptimization,
|
|
2336
2853
|
aiProvider: options.aiProvider,
|
|
2337
2854
|
aiModel: options.aiModel,
|
|
@@ -2385,7 +2902,7 @@ var SearchResource = class {
|
|
|
2385
2902
|
* @example
|
|
2386
2903
|
* ```ts
|
|
2387
2904
|
* const result = await client.search.andScrape("promo codes", {
|
|
2388
|
-
*
|
|
2905
|
+
* maxScrapeResults: 5
|
|
2389
2906
|
* });
|
|
2390
2907
|
* console.log(result.data.scrapedJobIds);
|
|
2391
2908
|
* ```
|
|
@@ -2393,7 +2910,7 @@ var SearchResource = class {
|
|
|
2393
2910
|
async andScrape(query, options) {
|
|
2394
2911
|
return this.create({
|
|
2395
2912
|
query,
|
|
2396
|
-
|
|
2913
|
+
scrapeResults: true,
|
|
2397
2914
|
...options
|
|
2398
2915
|
});
|
|
2399
2916
|
}
|
|
@@ -2576,7 +3093,9 @@ var WebhooksResource = class {
|
|
|
2576
3093
|
*/
|
|
2577
3094
|
async create(options) {
|
|
2578
3095
|
const result = await post(this.ctx, "/v1/webhooks", {
|
|
3096
|
+
events: options.events ?? (options.event ? [options.event] : void 0),
|
|
2579
3097
|
event: options.event,
|
|
3098
|
+
// Legacy fallback
|
|
2580
3099
|
url: options.url,
|
|
2581
3100
|
secret: options.secret,
|
|
2582
3101
|
minDealScore: options.minDealScore,
|
|
@@ -2614,7 +3133,7 @@ var WebhooksResource = class {
|
|
|
2614
3133
|
this.ctx,
|
|
2615
3134
|
`/v1/webhooks/${webhookId}`
|
|
2616
3135
|
);
|
|
2617
|
-
return result.data;
|
|
3136
|
+
return result.data.webhook;
|
|
2618
3137
|
}
|
|
2619
3138
|
/**
|
|
2620
3139
|
* Update a webhook
|
|
@@ -2672,7 +3191,8 @@ var WebhooksResource = class {
|
|
|
2672
3191
|
async test(webhookId) {
|
|
2673
3192
|
const result = await post(
|
|
2674
3193
|
this.ctx,
|
|
2675
|
-
|
|
3194
|
+
"/v1/webhooks/test",
|
|
3195
|
+
{ webhookId }
|
|
2676
3196
|
);
|
|
2677
3197
|
return result.data;
|
|
2678
3198
|
}
|
|
@@ -2710,7 +3230,7 @@ var WebhooksResource = class {
|
|
|
2710
3230
|
*/
|
|
2711
3231
|
async getActive() {
|
|
2712
3232
|
const all = await this.list();
|
|
2713
|
-
return all.
|
|
3233
|
+
return all.data.filter((w) => w.active);
|
|
2714
3234
|
}
|
|
2715
3235
|
/**
|
|
2716
3236
|
* Get webhooks by event type
|
|
@@ -2722,7 +3242,7 @@ var WebhooksResource = class {
|
|
|
2722
3242
|
*/
|
|
2723
3243
|
async getByEvent(event) {
|
|
2724
3244
|
const all = await this.list();
|
|
2725
|
-
return all.
|
|
3245
|
+
return all.data.filter((w) => w.event === event);
|
|
2726
3246
|
}
|
|
2727
3247
|
};
|
|
2728
3248
|
|
|
@@ -2807,6 +3327,20 @@ var DealCrawl = class {
|
|
|
2807
3327
|
* ```
|
|
2808
3328
|
*/
|
|
2809
3329
|
dork;
|
|
3330
|
+
/**
|
|
3331
|
+
* Convert resource - HTML to Markdown conversion
|
|
3332
|
+
*
|
|
3333
|
+
* @example
|
|
3334
|
+
* ```ts
|
|
3335
|
+
* const result = await client.convert.htmlToMarkdown({
|
|
3336
|
+
* html: "<h1>Title</h1><p>Content</p>",
|
|
3337
|
+
* baseUrl: "https://example.com",
|
|
3338
|
+
* options: { removeNoise: true }
|
|
3339
|
+
* });
|
|
3340
|
+
* console.log(result.data.markdown);
|
|
3341
|
+
* ```
|
|
3342
|
+
*/
|
|
3343
|
+
convert;
|
|
2810
3344
|
/**
|
|
2811
3345
|
* Agent resource - AI-powered autonomous web navigation
|
|
2812
3346
|
*
|
|
@@ -2888,6 +3422,80 @@ var DealCrawl = class {
|
|
|
2888
3422
|
* ```
|
|
2889
3423
|
*/
|
|
2890
3424
|
account;
|
|
3425
|
+
/**
|
|
3426
|
+
* Screenshots resource - Screenshot signed URL management
|
|
3427
|
+
*
|
|
3428
|
+
* @example
|
|
3429
|
+
* ```ts
|
|
3430
|
+
* // Refresh a signed URL before expiration
|
|
3431
|
+
* const refreshed = await client.screenshots.refresh({
|
|
3432
|
+
* path: "job_abc123/1234567890_nanoid_example.png",
|
|
3433
|
+
* ttl: 604800 // 7 days
|
|
3434
|
+
* });
|
|
3435
|
+
*
|
|
3436
|
+
* // Get tier-specific TTL limits
|
|
3437
|
+
* const limits = await client.screenshots.getLimits();
|
|
3438
|
+
* console.log(limits.formattedLimits.max); // "7 days"
|
|
3439
|
+
* ```
|
|
3440
|
+
*/
|
|
3441
|
+
screenshots;
|
|
3442
|
+
/**
|
|
3443
|
+
* Auth resource - SSE (Server-Sent Events) authentication
|
|
3444
|
+
*
|
|
3445
|
+
* @example
|
|
3446
|
+
* ```ts
|
|
3447
|
+
* // Generate SSE token for browser EventSource
|
|
3448
|
+
* const { token, expiresAt } = await client.auth.generateSSEToken();
|
|
3449
|
+
*
|
|
3450
|
+
* // Use in browser
|
|
3451
|
+
* const eventSource = new EventSource(`/v1/events?token=${token}`);
|
|
3452
|
+
*
|
|
3453
|
+
* // Generate token for specific job
|
|
3454
|
+
* const jobToken = await client.auth.generateSSEToken({ jobId: "job_123" });
|
|
3455
|
+
*
|
|
3456
|
+
* // Check connection limits
|
|
3457
|
+
* const limits = await client.auth.getLimits();
|
|
3458
|
+
* console.log(`Available connections: ${limits.sse.available}`);
|
|
3459
|
+
* ```
|
|
3460
|
+
*/
|
|
3461
|
+
auth;
|
|
3462
|
+
/**
|
|
3463
|
+
* Events resource - Real-time SSE event streaming (Browser only)
|
|
3464
|
+
*
|
|
3465
|
+
* IMPORTANT: This resource only works in browsers. For Node.js, use polling via client.status.get()
|
|
3466
|
+
*
|
|
3467
|
+
* @example Browser Usage
|
|
3468
|
+
* ```ts
|
|
3469
|
+
* // 1. Generate SSE token
|
|
3470
|
+
* const { token } = await client.auth.generateSSEToken();
|
|
3471
|
+
*
|
|
3472
|
+
* // 2. Subscribe to all events
|
|
3473
|
+
* const eventSource = client.events.subscribe(token, {
|
|
3474
|
+
* onEvent: (event) => {
|
|
3475
|
+
* console.log('Event:', event.type, JSON.parse(event.data));
|
|
3476
|
+
* }
|
|
3477
|
+
* });
|
|
3478
|
+
*
|
|
3479
|
+
* // 3. Or subscribe to specific job
|
|
3480
|
+
* const jobEvents = client.events.subscribeToJob('job_123', token, {
|
|
3481
|
+
* onEvent: (event) => {
|
|
3482
|
+
* const data = JSON.parse(event.data);
|
|
3483
|
+
* console.log(`Progress: ${data.progress}%`);
|
|
3484
|
+
* }
|
|
3485
|
+
* });
|
|
3486
|
+
*
|
|
3487
|
+
* // 4. Listen for specific events
|
|
3488
|
+
* eventSource.addEventListener('job.completed', (event) => {
|
|
3489
|
+
* const data = JSON.parse(event.data);
|
|
3490
|
+
* console.log('Job completed!', data.summary);
|
|
3491
|
+
* eventSource.close();
|
|
3492
|
+
* });
|
|
3493
|
+
*
|
|
3494
|
+
* // 5. Clean up
|
|
3495
|
+
* eventSource.close();
|
|
3496
|
+
* ```
|
|
3497
|
+
*/
|
|
3498
|
+
events;
|
|
2891
3499
|
// ============================================
|
|
2892
3500
|
// CONSTRUCTOR
|
|
2893
3501
|
// ============================================
|
|
@@ -2938,12 +3546,16 @@ var DealCrawl = class {
|
|
|
2938
3546
|
this.crawl = new CrawlResource(this.ctx);
|
|
2939
3547
|
this.extract = new ExtractResource(this.ctx);
|
|
2940
3548
|
this.dork = new DorkResource(this.ctx);
|
|
3549
|
+
this.convert = new ConvertResource(this.ctx);
|
|
2941
3550
|
this.agent = new AgentResource(this.ctx);
|
|
2942
3551
|
this.status = new StatusResource(this.ctx);
|
|
2943
3552
|
this.data = new DataResource(this.ctx);
|
|
2944
3553
|
this.webhooks = new WebhooksResource(this.ctx);
|
|
2945
3554
|
this.keys = new KeysResource(this.ctx);
|
|
2946
3555
|
this.account = new AccountResource(this.ctx);
|
|
3556
|
+
this.screenshots = new ScreenshotsResource(this.ctx);
|
|
3557
|
+
this.auth = new AuthResource(this.ctx);
|
|
3558
|
+
this.events = new EventsResource(this.ctx);
|
|
2947
3559
|
}
|
|
2948
3560
|
// ============================================
|
|
2949
3561
|
// POLLING METHODS
|
|
@@ -3087,6 +3699,6 @@ var DealCrawl = class {
|
|
|
3087
3699
|
}
|
|
3088
3700
|
};
|
|
3089
3701
|
|
|
3090
|
-
export { AccountResource, AgentResource, CrawlResource, DEFAULT_CONFIG, DataResource, DealCrawl, DealCrawlError, DorkResource, ERROR_CODES, ERROR_MESSAGES, ExtractResource, KeysResource, ScrapeResource, SearchResource, StatusResource, WebhooksResource, DealCrawl as default, getErrorMessage, pollUntil, waitForAll, waitForAny, waitForResult };
|
|
3702
|
+
export { AccountResource, AgentResource, AuthResource, ConvertResource, CrawlResource, DEFAULT_CONFIG, DataResource, DealCrawl, DealCrawlError, DorkResource, ERROR_CODES, ERROR_MESSAGES, EventsResource, ExtractResource, KeysResource, ScrapeResource, ScreenshotsResource, SearchResource, StatusResource, WebhooksResource, DealCrawl as default, getErrorMessage, pollUntil, waitForAll, waitForAny, waitForResult };
|
|
3091
3703
|
//# sourceMappingURL=index.mjs.map
|
|
3092
3704
|
//# sourceMappingURL=index.mjs.map
|