@dealcrawl/sdk 2.10.0 → 2.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +246 -37
- package/dist/index.d.mts +1174 -50
- package/dist/index.d.ts +1174 -50
- package/dist/index.js +668 -52
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +665 -53
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1122,6 +1122,245 @@ var AgentResource = class {
|
|
|
1122
1122
|
}
|
|
1123
1123
|
};
|
|
1124
1124
|
|
|
1125
|
+
// src/resources/auth.ts
|
|
1126
|
+
var AuthResource = class {
|
|
1127
|
+
constructor(ctx) {
|
|
1128
|
+
this.ctx = ctx;
|
|
1129
|
+
}
|
|
1130
|
+
/**
|
|
1131
|
+
* Generate SSE authentication token
|
|
1132
|
+
*
|
|
1133
|
+
* Required for browser-based SSE connections because EventSource API
|
|
1134
|
+
* doesn't support custom headers. Token is short-lived (5 minutes).
|
|
1135
|
+
*
|
|
1136
|
+
* Security:
|
|
1137
|
+
* - Requires valid API key (Bearer token)
|
|
1138
|
+
* - Token expires in 5 minutes
|
|
1139
|
+
* - Token can be restricted to specific job
|
|
1140
|
+
* - Token stored in Redis (revocable)
|
|
1141
|
+
*
|
|
1142
|
+
* @example
|
|
1143
|
+
* ```ts
|
|
1144
|
+
* // 1. Generate token
|
|
1145
|
+
* const { token, expiresAt } = await client.auth.generateSSEToken();
|
|
1146
|
+
* console.log(`Token expires at: ${expiresAt}`);
|
|
1147
|
+
*
|
|
1148
|
+
* // 2. Use in browser EventSource
|
|
1149
|
+
* const eventSource = new EventSource(`/v1/events?token=${token}`);
|
|
1150
|
+
*
|
|
1151
|
+
* eventSource.addEventListener('job.completed', (event) => {
|
|
1152
|
+
* const data = JSON.parse(event.data);
|
|
1153
|
+
* console.log('Job completed:', data);
|
|
1154
|
+
* });
|
|
1155
|
+
*
|
|
1156
|
+
* // 3. For specific job only
|
|
1157
|
+
* const jobToken = await client.auth.generateSSEToken({ jobId: "job_abc123" });
|
|
1158
|
+
* const jobEvents = new EventSource(`/v1/events/job_abc123?token=${jobToken.token}`);
|
|
1159
|
+
* ```
|
|
1160
|
+
*/
|
|
1161
|
+
async generateSSEToken(options) {
|
|
1162
|
+
const result = await post(
|
|
1163
|
+
this.ctx,
|
|
1164
|
+
"/v1/auth/sse-token",
|
|
1165
|
+
options ?? {}
|
|
1166
|
+
);
|
|
1167
|
+
return result.data;
|
|
1168
|
+
}
|
|
1169
|
+
/**
|
|
1170
|
+
* Get SSE connection limits for current tier
|
|
1171
|
+
*
|
|
1172
|
+
* Shows how many concurrent SSE connections are allowed
|
|
1173
|
+
* and how many are currently active.
|
|
1174
|
+
*
|
|
1175
|
+
* Tier limits:
|
|
1176
|
+
* - Free: 10 concurrent connections
|
|
1177
|
+
* - Pro: 50 concurrent connections
|
|
1178
|
+
* - Enterprise: 200 concurrent connections
|
|
1179
|
+
*
|
|
1180
|
+
* @example
|
|
1181
|
+
* ```ts
|
|
1182
|
+
* const limits = await client.auth.getLimits();
|
|
1183
|
+
*
|
|
1184
|
+
* console.log(`Tier: ${limits.tier}`);
|
|
1185
|
+
* console.log(`Max connections: ${limits.sse.maxConnections}`);
|
|
1186
|
+
* console.log(`Current connections: ${limits.sse.currentConnections}`);
|
|
1187
|
+
* console.log(`Available: ${limits.sse.available}`);
|
|
1188
|
+
*
|
|
1189
|
+
* // Check before opening new connection
|
|
1190
|
+
* if (limits.sse.available > 0) {
|
|
1191
|
+
* const token = await client.auth.generateSSEToken();
|
|
1192
|
+
* const eventSource = new EventSource(`/v1/events?token=${token.token}`);
|
|
1193
|
+
* } else {
|
|
1194
|
+
* console.error('No available SSE connection slots');
|
|
1195
|
+
* }
|
|
1196
|
+
* ```
|
|
1197
|
+
*/
|
|
1198
|
+
async getLimits() {
|
|
1199
|
+
const result = await get(
|
|
1200
|
+
this.ctx,
|
|
1201
|
+
"/v1/auth/limits"
|
|
1202
|
+
);
|
|
1203
|
+
return result.data;
|
|
1204
|
+
}
|
|
1205
|
+
};
|
|
1206
|
+
|
|
1207
|
+
// src/resources/convert.ts
|
|
1208
|
+
var ConvertResource = class {
|
|
1209
|
+
constructor(ctx) {
|
|
1210
|
+
this.ctx = ctx;
|
|
1211
|
+
}
|
|
1212
|
+
/**
|
|
1213
|
+
* Convert HTML to Markdown
|
|
1214
|
+
*
|
|
1215
|
+
* Transforms raw HTML content into clean, readable Markdown using GitHub Flavored Markdown (GFM).
|
|
1216
|
+
* Useful for:
|
|
1217
|
+
* - Converting scraped HTML to markdown for LLM processing
|
|
1218
|
+
* - Cleaning up messy HTML from web pages
|
|
1219
|
+
* - Extracting main content while removing noise (ads, nav, footer)
|
|
1220
|
+
* - Creating documentation from HTML sources
|
|
1221
|
+
*
|
|
1222
|
+
* Features:
|
|
1223
|
+
* - GFM table, strikethrough, and task list support
|
|
1224
|
+
* - Automatic noise removal (scripts, ads, navigation)
|
|
1225
|
+
* - Relative URL resolution
|
|
1226
|
+
* - Custom element exclusion via CSS selectors
|
|
1227
|
+
* - Output length limiting
|
|
1228
|
+
*
|
|
1229
|
+
* @param options - Conversion options
|
|
1230
|
+
* @returns Conversion result with markdown, metadata, and warnings
|
|
1231
|
+
*
|
|
1232
|
+
* @example Basic usage
|
|
1233
|
+
* ```ts
|
|
1234
|
+
* const result = await client.convert.htmlToMarkdown({
|
|
1235
|
+
* html: "<h1>Product</h1><p>Price: $99</p>"
|
|
1236
|
+
* });
|
|
1237
|
+
* console.log(result.data.markdown);
|
|
1238
|
+
* ```
|
|
1239
|
+
*
|
|
1240
|
+
* @example With all options
|
|
1241
|
+
* ```ts
|
|
1242
|
+
* const result = await client.convert.htmlToMarkdown({
|
|
1243
|
+
* html: htmlContent,
|
|
1244
|
+
* baseUrl: "https://shop.example.com",
|
|
1245
|
+
* options: {
|
|
1246
|
+
* gfmTables: true,
|
|
1247
|
+
* removeNoise: true,
|
|
1248
|
+
* excludeSelectors: [".advertisement", "#sidebar"],
|
|
1249
|
+
* absoluteUrls: true,
|
|
1250
|
+
* maxLength: 100000,
|
|
1251
|
+
* includeImages: true,
|
|
1252
|
+
* includeLinks: true
|
|
1253
|
+
* }
|
|
1254
|
+
* });
|
|
1255
|
+
*
|
|
1256
|
+
* // Check metadata
|
|
1257
|
+
* console.log(`Words: ${result.data.metadata.wordCount}`);
|
|
1258
|
+
* console.log(`Links: ${result.data.metadata.linkCount}`);
|
|
1259
|
+
* console.log(`Images: ${result.data.metadata.imageCount}`);
|
|
1260
|
+
* console.log(`Conversion time: ${result.data.metadata.conversionTimeMs}ms`);
|
|
1261
|
+
*
|
|
1262
|
+
* // Check for warnings
|
|
1263
|
+
* if (result.data.warnings?.length) {
|
|
1264
|
+
* console.warn("Conversion warnings:", result.data.warnings);
|
|
1265
|
+
* }
|
|
1266
|
+
* ```
|
|
1267
|
+
*
|
|
1268
|
+
* @example Converting scraped HTML
|
|
1269
|
+
* ```ts
|
|
1270
|
+
* // First scrape a page
|
|
1271
|
+
* const scrapeJob = await client.scrape.create({
|
|
1272
|
+
* url: "https://example.com/article"
|
|
1273
|
+
* });
|
|
1274
|
+
* const scrapeResult = await client.waitForResult(scrapeJob.jobId);
|
|
1275
|
+
*
|
|
1276
|
+
* // Then convert HTML to markdown
|
|
1277
|
+
* const markdown = await client.convert.htmlToMarkdown({
|
|
1278
|
+
* html: scrapeResult.data.html,
|
|
1279
|
+
* baseUrl: scrapeResult.data.url,
|
|
1280
|
+
* options: {
|
|
1281
|
+
* removeNoise: true,
|
|
1282
|
+
* onlyMainContent: true
|
|
1283
|
+
* }
|
|
1284
|
+
* });
|
|
1285
|
+
* ```
|
|
1286
|
+
*/
|
|
1287
|
+
async htmlToMarkdown(options) {
|
|
1288
|
+
const body = {
|
|
1289
|
+
html: options.html,
|
|
1290
|
+
baseUrl: options.baseUrl,
|
|
1291
|
+
options: options.options
|
|
1292
|
+
};
|
|
1293
|
+
const result = await post(this.ctx, "/v1/convert", body);
|
|
1294
|
+
return result.data;
|
|
1295
|
+
}
|
|
1296
|
+
/**
|
|
1297
|
+
* Alias for htmlToMarkdown() for convenience
|
|
1298
|
+
*
|
|
1299
|
+
* @example
|
|
1300
|
+
* ```ts
|
|
1301
|
+
* const result = await client.convert.toMarkdown({
|
|
1302
|
+
* html: "<h1>Hello</h1>"
|
|
1303
|
+
* });
|
|
1304
|
+
* ```
|
|
1305
|
+
*/
|
|
1306
|
+
async toMarkdown(options) {
|
|
1307
|
+
return this.htmlToMarkdown(options);
|
|
1308
|
+
}
|
|
1309
|
+
/**
|
|
1310
|
+
* Convert HTML with minimal options (just the HTML content)
|
|
1311
|
+
* Uses all default settings
|
|
1312
|
+
*
|
|
1313
|
+
* @param html - HTML content to convert
|
|
1314
|
+
* @param baseUrl - Optional base URL for resolving relative links
|
|
1315
|
+
* @returns Conversion result
|
|
1316
|
+
*
|
|
1317
|
+
* @example
|
|
1318
|
+
* ```ts
|
|
1319
|
+
* const result = await client.convert.quick(
|
|
1320
|
+
* "<h1>Title</h1><p>Content</p>",
|
|
1321
|
+
* "https://example.com"
|
|
1322
|
+
* );
|
|
1323
|
+
* console.log(result.data.markdown);
|
|
1324
|
+
* ```
|
|
1325
|
+
*/
|
|
1326
|
+
async quick(html, baseUrl) {
|
|
1327
|
+
return this.htmlToMarkdown({ html, baseUrl });
|
|
1328
|
+
}
|
|
1329
|
+
/**
|
|
1330
|
+
* Convert HTML with noise removal enabled
|
|
1331
|
+
* Removes navigation, footer, ads, scripts, and other clutter
|
|
1332
|
+
*
|
|
1333
|
+
* @param html - HTML content to convert
|
|
1334
|
+
* @param baseUrl - Optional base URL
|
|
1335
|
+
* @returns Conversion result with clean markdown
|
|
1336
|
+
*
|
|
1337
|
+
* @example
|
|
1338
|
+
* ```ts
|
|
1339
|
+
* // Extract just the main content from a messy page
|
|
1340
|
+
* const result = await client.convert.clean(messyHtml, "https://example.com");
|
|
1341
|
+
* console.log(result.data.markdown); // Clean, readable markdown
|
|
1342
|
+
* ```
|
|
1343
|
+
*/
|
|
1344
|
+
async clean(html, baseUrl) {
|
|
1345
|
+
return this.htmlToMarkdown({
|
|
1346
|
+
html,
|
|
1347
|
+
baseUrl,
|
|
1348
|
+
options: {
|
|
1349
|
+
removeNoise: true,
|
|
1350
|
+
excludeSelectors: [
|
|
1351
|
+
"nav",
|
|
1352
|
+
"footer",
|
|
1353
|
+
"aside",
|
|
1354
|
+
".advertisement",
|
|
1355
|
+
".ad",
|
|
1356
|
+
".sidebar",
|
|
1357
|
+
"#comments"
|
|
1358
|
+
]
|
|
1359
|
+
}
|
|
1360
|
+
});
|
|
1361
|
+
}
|
|
1362
|
+
};
|
|
1363
|
+
|
|
1125
1364
|
// src/resources/crawl.ts
|
|
1126
1365
|
var CRAWL_TEMPLATES = {
|
|
1127
1366
|
ecommerce: {
|
|
@@ -1294,12 +1533,10 @@ var CrawlResource = class {
|
|
|
1294
1533
|
* ```
|
|
1295
1534
|
*/
|
|
1296
1535
|
async analyze(url) {
|
|
1297
|
-
const result = await
|
|
1536
|
+
const result = await post(
|
|
1298
1537
|
this.ctx,
|
|
1299
1538
|
"/v1/crawl/analyze",
|
|
1300
|
-
{
|
|
1301
|
-
url
|
|
1302
|
-
}
|
|
1539
|
+
{ url }
|
|
1303
1540
|
);
|
|
1304
1541
|
return result.data;
|
|
1305
1542
|
}
|
|
@@ -1465,31 +1702,28 @@ var DataResource = class {
|
|
|
1465
1702
|
});
|
|
1466
1703
|
}
|
|
1467
1704
|
/**
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
)
|
|
1478
|
-
|
|
1479
|
-
throw new Error("category is required and cannot be empty");
|
|
1480
|
-
}
|
|
1481
|
-
return this.listDeals({ category, ...options });
|
|
1482
|
-
} return this.listDeals({ category, ...options });
|
|
1705
|
+
* Get deals by category
|
|
1706
|
+
* Convenience method for filtering by category
|
|
1707
|
+
*
|
|
1708
|
+
* @example
|
|
1709
|
+
* ```ts
|
|
1710
|
+
* const electronicsDeals = await client.data.getDealsByCategory("electronics");
|
|
1711
|
+
* ```
|
|
1712
|
+
*/
|
|
1713
|
+
async getDealsByCategory(category, options) {
|
|
1714
|
+
if (!category || !category.trim()) {
|
|
1715
|
+
throw new Error("category is required and cannot be empty");
|
|
1483
1716
|
}
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1717
|
+
return this.listDeals({ category, ...options });
|
|
1718
|
+
}
|
|
1719
|
+
/**
|
|
1720
|
+
* Get unsynced deals (not yet sent to DealUp)
|
|
1721
|
+
*
|
|
1722
|
+
* @example
|
|
1723
|
+
* ```ts
|
|
1724
|
+
* const unsyncedDeals = await client.data.getUnsyncedDeals();
|
|
1725
|
+
* ```
|
|
1726
|
+
*/
|
|
1493
1727
|
async getUnsyncedDeals(options) {
|
|
1494
1728
|
return this.listDeals({ synced: false, ...options });
|
|
1495
1729
|
}
|
|
@@ -1545,9 +1779,11 @@ var DataResource = class {
|
|
|
1545
1779
|
{
|
|
1546
1780
|
format: options.format || "json",
|
|
1547
1781
|
minScore: options.minScore,
|
|
1548
|
-
maxPrice: options.maxPrice,
|
|
1549
1782
|
category: options.category,
|
|
1550
|
-
|
|
1783
|
+
synced: options.synced?.toString(),
|
|
1784
|
+
fromDate: options.fromDate,
|
|
1785
|
+
toDate: options.toDate,
|
|
1786
|
+
limit: options.limit
|
|
1551
1787
|
}
|
|
1552
1788
|
);
|
|
1553
1789
|
return result.data;
|
|
@@ -1592,8 +1828,8 @@ var DorkResource = class {
|
|
|
1592
1828
|
async create(options) {
|
|
1593
1829
|
const body = {
|
|
1594
1830
|
query: this.buildQuery(options),
|
|
1595
|
-
|
|
1596
|
-
|
|
1831
|
+
site: options.site,
|
|
1832
|
+
maxResults: options.maxResults
|
|
1597
1833
|
};
|
|
1598
1834
|
const result = await post(this.ctx, "/v1/dork", body);
|
|
1599
1835
|
return result.data;
|
|
@@ -1708,9 +1944,6 @@ var DorkResource = class {
|
|
|
1708
1944
|
if (typeof options.query === "string" && options.query.trim() !== "") {
|
|
1709
1945
|
parts.push(options.query);
|
|
1710
1946
|
}
|
|
1711
|
-
if (options.site) {
|
|
1712
|
-
parts.push(`site:${options.site}`);
|
|
1713
|
-
}
|
|
1714
1947
|
if (options.fileType) {
|
|
1715
1948
|
parts.push(`filetype:${options.fileType}`);
|
|
1716
1949
|
}
|
|
@@ -1724,6 +1957,236 @@ var DorkResource = class {
|
|
|
1724
1957
|
}
|
|
1725
1958
|
};
|
|
1726
1959
|
|
|
1960
|
+
// src/resources/events.ts
|
|
1961
|
+
var EventsResource = class {
|
|
1962
|
+
constructor(ctx) {
|
|
1963
|
+
this.ctx = ctx;
|
|
1964
|
+
}
|
|
1965
|
+
/**
|
|
1966
|
+
* Subscribe to all events for authenticated client
|
|
1967
|
+
*
|
|
1968
|
+
* Opens an SSE connection to receive real-time events for all jobs.
|
|
1969
|
+
* Requires an SSE token obtained via client.auth.generateSSEToken().
|
|
1970
|
+
*
|
|
1971
|
+
* Event Types:
|
|
1972
|
+
* - Job lifecycle: job.created, job.queued, job.started, job.progress,
|
|
1973
|
+
* job.completed, job.failed, job.cancelled
|
|
1974
|
+
* - Job details: job.log, job.metric, job.alert, job.checkpoint
|
|
1975
|
+
* - Deals: deal.found, deal.validated
|
|
1976
|
+
* - System: ping, connection.open, connection.close, error
|
|
1977
|
+
*
|
|
1978
|
+
* Features:
|
|
1979
|
+
* - Automatic reconnection on disconnect
|
|
1980
|
+
* - Event replay via Last-Event-ID
|
|
1981
|
+
* - Keepalive pings every 15 seconds
|
|
1982
|
+
* - Max connection time: 1 hour
|
|
1983
|
+
*
|
|
1984
|
+
* @param token - SSE authentication token from client.auth.generateSSEToken()
|
|
1985
|
+
* @param options - Subscription options (callbacks, reconnection settings)
|
|
1986
|
+
*
|
|
1987
|
+
* @example
|
|
1988
|
+
* ```ts
|
|
1989
|
+
* // Generate token
|
|
1990
|
+
* const { token } = await client.auth.generateSSEToken();
|
|
1991
|
+
*
|
|
1992
|
+
* // Subscribe with event handlers
|
|
1993
|
+
* const eventSource = client.events.subscribe(token, {
|
|
1994
|
+
* onEvent: (event) => {
|
|
1995
|
+
* // Handle all events
|
|
1996
|
+
* console.log('Event:', event.type);
|
|
1997
|
+
* const data = JSON.parse(event.data);
|
|
1998
|
+
*
|
|
1999
|
+
* if (data.jobId) {
|
|
2000
|
+
* console.log(`Job ${data.jobId}:`, data);
|
|
2001
|
+
* }
|
|
2002
|
+
* },
|
|
2003
|
+
* onError: (error) => {
|
|
2004
|
+
* console.error('SSE error:', error);
|
|
2005
|
+
* },
|
|
2006
|
+
* onOpen: () => {
|
|
2007
|
+
* console.log('SSE connection opened');
|
|
2008
|
+
* }
|
|
2009
|
+
* });
|
|
2010
|
+
*
|
|
2011
|
+
* // Listen for specific event types
|
|
2012
|
+
* eventSource.addEventListener('job.completed', (event) => {
|
|
2013
|
+
* const data = JSON.parse(event.data);
|
|
2014
|
+
* console.log('Job completed:', data);
|
|
2015
|
+
* });
|
|
2016
|
+
*
|
|
2017
|
+
* // Clean up
|
|
2018
|
+
* eventSource.close();
|
|
2019
|
+
* ```
|
|
2020
|
+
*/
|
|
2021
|
+
subscribe(token, options) {
|
|
2022
|
+
if (typeof EventSource === "undefined") {
|
|
2023
|
+
throw new Error(
|
|
2024
|
+
"EventSource is not available. SSE subscriptions only work in browsers. For Node.js, use polling via client.status.get() instead."
|
|
2025
|
+
);
|
|
2026
|
+
}
|
|
2027
|
+
const url = new URL("/v1/events", this.ctx.baseUrl);
|
|
2028
|
+
url.searchParams.set("token", token);
|
|
2029
|
+
const eventSource = new EventSource(url.toString());
|
|
2030
|
+
if (options?.onEvent) {
|
|
2031
|
+
eventSource.onmessage = options.onEvent;
|
|
2032
|
+
}
|
|
2033
|
+
if (options?.onError) {
|
|
2034
|
+
eventSource.onerror = (event) => {
|
|
2035
|
+
options.onError(
|
|
2036
|
+
new Error("SSE connection error. Will auto-reconnect if enabled.")
|
|
2037
|
+
);
|
|
2038
|
+
};
|
|
2039
|
+
}
|
|
2040
|
+
if (options?.onOpen) {
|
|
2041
|
+
eventSource.onopen = options.onOpen;
|
|
2042
|
+
}
|
|
2043
|
+
return eventSource;
|
|
2044
|
+
}
|
|
2045
|
+
/**
|
|
2046
|
+
* Subscribe to events for a specific job
|
|
2047
|
+
*
|
|
2048
|
+
* Opens an SSE connection filtered to a single job.
|
|
2049
|
+
* More efficient than global subscription when tracking one job.
|
|
2050
|
+
*
|
|
2051
|
+
* @param jobId - Job ID to subscribe to
|
|
2052
|
+
* @param token - SSE authentication token
|
|
2053
|
+
* @param options - Subscription options
|
|
2054
|
+
*
|
|
2055
|
+
* @example
|
|
2056
|
+
* ```ts
|
|
2057
|
+
* // Start a scrape job
|
|
2058
|
+
* const job = await client.scrape.create({ url: "https://example.com" });
|
|
2059
|
+
*
|
|
2060
|
+
* // Generate SSE token for this job
|
|
2061
|
+
* const { token } = await client.auth.generateSSEToken({ jobId: job.jobId });
|
|
2062
|
+
*
|
|
2063
|
+
* // Subscribe to job events
|
|
2064
|
+
* const eventSource = client.events.subscribeToJob(job.jobId, token, {
|
|
2065
|
+
* onEvent: (event) => {
|
|
2066
|
+
* const data = JSON.parse(event.data);
|
|
2067
|
+
* console.log(`[${event.type}]`, data);
|
|
2068
|
+
* }
|
|
2069
|
+
* });
|
|
2070
|
+
*
|
|
2071
|
+
* // Listen for completion
|
|
2072
|
+
* eventSource.addEventListener('job.completed', (event) => {
|
|
2073
|
+
* const data = JSON.parse(event.data);
|
|
2074
|
+
* console.log('Scrape completed!', data.summary);
|
|
2075
|
+
* eventSource.close();
|
|
2076
|
+
* });
|
|
2077
|
+
*
|
|
2078
|
+
* // Listen for progress
|
|
2079
|
+
* eventSource.addEventListener('job.progress', (event) => {
|
|
2080
|
+
* const data = JSON.parse(event.data);
|
|
2081
|
+
* console.log(`Progress: ${data.progress}%`);
|
|
2082
|
+
* });
|
|
2083
|
+
*
|
|
2084
|
+
* // Listen for errors
|
|
2085
|
+
* eventSource.addEventListener('job.failed', (event) => {
|
|
2086
|
+
* const data = JSON.parse(event.data);
|
|
2087
|
+
* console.error('Job failed:', data.error);
|
|
2088
|
+
* eventSource.close();
|
|
2089
|
+
* });
|
|
2090
|
+
* ```
|
|
2091
|
+
*/
|
|
2092
|
+
subscribeToJob(jobId, token, options) {
|
|
2093
|
+
if (typeof EventSource === "undefined") {
|
|
2094
|
+
throw new Error(
|
|
2095
|
+
"EventSource is not available. SSE subscriptions only work in browsers. For Node.js, use polling via client.status.get() instead."
|
|
2096
|
+
);
|
|
2097
|
+
}
|
|
2098
|
+
const url = new URL(`/v1/events/${jobId}`, this.ctx.baseUrl);
|
|
2099
|
+
url.searchParams.set("token", token);
|
|
2100
|
+
const eventSource = new EventSource(url.toString());
|
|
2101
|
+
if (options?.onEvent) {
|
|
2102
|
+
eventSource.onmessage = options.onEvent;
|
|
2103
|
+
}
|
|
2104
|
+
if (options?.onError) {
|
|
2105
|
+
eventSource.onerror = (event) => {
|
|
2106
|
+
options.onError(
|
|
2107
|
+
new Error("SSE connection error. Will auto-reconnect if enabled.")
|
|
2108
|
+
);
|
|
2109
|
+
};
|
|
2110
|
+
}
|
|
2111
|
+
if (options?.onOpen) {
|
|
2112
|
+
eventSource.onopen = options.onOpen;
|
|
2113
|
+
}
|
|
2114
|
+
return eventSource;
|
|
2115
|
+
}
|
|
2116
|
+
/**
|
|
2117
|
+
* Helper: Wait for job completion via SSE
|
|
2118
|
+
*
|
|
2119
|
+
* Convenience method that subscribes to a job and resolves when complete.
|
|
2120
|
+
* Automatically handles token generation and cleanup.
|
|
2121
|
+
*
|
|
2122
|
+
* @param jobId - Job ID to wait for
|
|
2123
|
+
* @param onProgress - Optional progress callback
|
|
2124
|
+
*
|
|
2125
|
+
* @example
|
|
2126
|
+
* ```ts
|
|
2127
|
+
* const job = await client.scrape.create({ url: "https://example.com" });
|
|
2128
|
+
*
|
|
2129
|
+
* // Wait for completion with progress updates
|
|
2130
|
+
* const result = await client.events.waitForCompletion(job.jobId, (progress) => {
|
|
2131
|
+
* console.log(`Progress: ${progress}%`);
|
|
2132
|
+
* });
|
|
2133
|
+
*
|
|
2134
|
+
* console.log('Job completed:', result);
|
|
2135
|
+
* ```
|
|
2136
|
+
*/
|
|
2137
|
+
async waitForCompletion(jobId, onProgress) {
|
|
2138
|
+
if (typeof EventSource === "undefined") {
|
|
2139
|
+
throw new Error(
|
|
2140
|
+
"waitForCompletion() only works in browsers. For Node.js, use client.waitForResult() instead."
|
|
2141
|
+
);
|
|
2142
|
+
}
|
|
2143
|
+
return new Promise(async (resolve, reject) => {
|
|
2144
|
+
const tokenResponse = await fetch(
|
|
2145
|
+
`${this.ctx.baseUrl}/v1/auth/sse-token`,
|
|
2146
|
+
{
|
|
2147
|
+
method: "POST",
|
|
2148
|
+
headers: {
|
|
2149
|
+
Authorization: `Bearer ${this.ctx.apiKey}`,
|
|
2150
|
+
"Content-Type": "application/json"
|
|
2151
|
+
},
|
|
2152
|
+
body: JSON.stringify({ jobId })
|
|
2153
|
+
}
|
|
2154
|
+
);
|
|
2155
|
+
if (!tokenResponse.ok) {
|
|
2156
|
+
reject(new Error("Failed to generate SSE token"));
|
|
2157
|
+
return;
|
|
2158
|
+
}
|
|
2159
|
+
const { token } = await tokenResponse.json();
|
|
2160
|
+
const eventSource = this.subscribeToJob(jobId, token, {
|
|
2161
|
+
onError: (error) => {
|
|
2162
|
+
eventSource.close();
|
|
2163
|
+
reject(error);
|
|
2164
|
+
}
|
|
2165
|
+
});
|
|
2166
|
+
eventSource.addEventListener("job.progress", (event) => {
|
|
2167
|
+
const data = JSON.parse(event.data);
|
|
2168
|
+
if (onProgress) {
|
|
2169
|
+
onProgress(data.progress);
|
|
2170
|
+
}
|
|
2171
|
+
});
|
|
2172
|
+
eventSource.addEventListener("job.completed", (event) => {
|
|
2173
|
+
const data = JSON.parse(event.data);
|
|
2174
|
+
eventSource.close();
|
|
2175
|
+
resolve(data);
|
|
2176
|
+
});
|
|
2177
|
+
eventSource.addEventListener("job.failed", (event) => {
|
|
2178
|
+
const data = JSON.parse(event.data);
|
|
2179
|
+
eventSource.close();
|
|
2180
|
+
reject(new Error(data.error?.message || "Job failed"));
|
|
2181
|
+
});
|
|
2182
|
+
eventSource.addEventListener("job.cancelled", (event) => {
|
|
2183
|
+
eventSource.close();
|
|
2184
|
+
reject(new Error("Job was cancelled"));
|
|
2185
|
+
});
|
|
2186
|
+
});
|
|
2187
|
+
}
|
|
2188
|
+
};
|
|
2189
|
+
|
|
1727
2190
|
// src/resources/extract.ts
|
|
1728
2191
|
var ExtractResource = class {
|
|
1729
2192
|
constructor(ctx) {
|
|
@@ -2105,10 +2568,7 @@ var KeysResource = class {
|
|
|
2105
2568
|
* ```
|
|
2106
2569
|
*/
|
|
2107
2570
|
async revokeAll() {
|
|
2108
|
-
const result = await
|
|
2109
|
-
this.ctx,
|
|
2110
|
-
"/v1/keys/all"
|
|
2111
|
-
);
|
|
2571
|
+
const result = await post(this.ctx, "/v1/keys/revoke-all");
|
|
2112
2572
|
return result.data;
|
|
2113
2573
|
}
|
|
2114
2574
|
/**
|
|
@@ -2164,6 +2624,7 @@ var ScrapeResource = class {
|
|
|
2164
2624
|
async create(options) {
|
|
2165
2625
|
const body = {
|
|
2166
2626
|
url: options.url,
|
|
2627
|
+
noStore: options.noStore,
|
|
2167
2628
|
detectSignals: options.detectSignals ?? true,
|
|
2168
2629
|
extractWithAI: options.extractWithAI,
|
|
2169
2630
|
extractDeal: options.extractDeal,
|
|
@@ -2176,7 +2637,10 @@ var ScrapeResource = class {
|
|
|
2176
2637
|
excludeSelectors: options.excludeSelectors,
|
|
2177
2638
|
onlyMainContent: options.onlyMainContent,
|
|
2178
2639
|
headers: options.headers,
|
|
2179
|
-
timeout: options.timeout
|
|
2640
|
+
timeout: options.timeout,
|
|
2641
|
+
outputMarkdown: options.outputMarkdown,
|
|
2642
|
+
markdownBaseUrl: options.markdownBaseUrl,
|
|
2643
|
+
actions: options.actions
|
|
2180
2644
|
};
|
|
2181
2645
|
const result = await post(this.ctx, "/v1/scrape", body);
|
|
2182
2646
|
return result.data;
|
|
@@ -2252,7 +2716,8 @@ var ScrapeResource = class {
|
|
|
2252
2716
|
* { url: "https://shop1.com/product1" },
|
|
2253
2717
|
* { url: "https://shop2.com/deal", extractDeal: true }
|
|
2254
2718
|
* ],
|
|
2255
|
-
* defaults: { detectSignals: true }
|
|
2719
|
+
* defaults: { detectSignals: true },
|
|
2720
|
+
* ignoreInvalidURLs: true
|
|
2256
2721
|
* });
|
|
2257
2722
|
* console.log(batch.batchId, batch.results);
|
|
2258
2723
|
* ```
|
|
@@ -2263,7 +2728,8 @@ var ScrapeResource = class {
|
|
|
2263
2728
|
defaults: options.defaults,
|
|
2264
2729
|
webhookUrl: options.webhookUrl,
|
|
2265
2730
|
priority: options.priority,
|
|
2266
|
-
delayMs: options.
|
|
2731
|
+
delayMs: options.delayMs,
|
|
2732
|
+
ignoreInvalidURLs: options.ignoreInvalidURLs
|
|
2267
2733
|
};
|
|
2268
2734
|
const result = await post(
|
|
2269
2735
|
this.ctx,
|
|
@@ -2313,6 +2779,57 @@ var ScrapeResource = class {
|
|
|
2313
2779
|
}
|
|
2314
2780
|
};
|
|
2315
2781
|
|
|
2782
|
+
// src/resources/screenshots.ts
|
|
2783
|
+
var ScreenshotsResource = class {
|
|
2784
|
+
constructor(ctx) {
|
|
2785
|
+
this.ctx = ctx;
|
|
2786
|
+
}
|
|
2787
|
+
/**
|
|
2788
|
+
* Refresh a signed URL before expiration
|
|
2789
|
+
*
|
|
2790
|
+
* @example
|
|
2791
|
+
* ```ts
|
|
2792
|
+
* const refreshed = await client.screenshots.refresh({
|
|
2793
|
+
* path: "job_abc123/1234567890_nanoid_example.png",
|
|
2794
|
+
* ttl: 604800 // 7 days
|
|
2795
|
+
* });
|
|
2796
|
+
* console.log(refreshed.url); // New signed URL
|
|
2797
|
+
* console.log(refreshed.expiresAt); // "2026-01-25T12:00:00Z"
|
|
2798
|
+
* console.log(refreshed.tierLimits); // { min: 3600, max: 604800, default: 604800 }
|
|
2799
|
+
* ```
|
|
2800
|
+
*/
|
|
2801
|
+
async refresh(options) {
|
|
2802
|
+
const result = await post(
|
|
2803
|
+
this.ctx,
|
|
2804
|
+
"/v1/screenshots/refresh",
|
|
2805
|
+
{
|
|
2806
|
+
path: options.path,
|
|
2807
|
+
ttl: options.ttl,
|
|
2808
|
+
bucket: options.bucket
|
|
2809
|
+
}
|
|
2810
|
+
);
|
|
2811
|
+
return result.data;
|
|
2812
|
+
}
|
|
2813
|
+
/**
|
|
2814
|
+
* Get TTL limits for the current tier
|
|
2815
|
+
*
|
|
2816
|
+
* @example
|
|
2817
|
+
* ```ts
|
|
2818
|
+
* const limits = await client.screenshots.getLimits();
|
|
2819
|
+
* console.log(limits.tier); // "pro"
|
|
2820
|
+
* console.log(limits.limits.max); // 604800 (7 days in seconds)
|
|
2821
|
+
* console.log(limits.formattedLimits.max); // "7 days"
|
|
2822
|
+
* ```
|
|
2823
|
+
*/
|
|
2824
|
+
async getLimits() {
|
|
2825
|
+
const result = await get(
|
|
2826
|
+
this.ctx,
|
|
2827
|
+
"/v1/screenshots/limits"
|
|
2828
|
+
);
|
|
2829
|
+
return result.data;
|
|
2830
|
+
}
|
|
2831
|
+
};
|
|
2832
|
+
|
|
2316
2833
|
// src/resources/search.ts
|
|
2317
2834
|
var SearchResource = class {
|
|
2318
2835
|
constructor(ctx) {
|
|
@@ -2325,7 +2842,7 @@ var SearchResource = class {
|
|
|
2325
2842
|
* ```ts
|
|
2326
2843
|
* const result = await client.search.create({
|
|
2327
2844
|
* query: "laptop deals black friday",
|
|
2328
|
-
*
|
|
2845
|
+
* limit: 20,
|
|
2329
2846
|
* useDealScoring: true
|
|
2330
2847
|
* });
|
|
2331
2848
|
* ```
|
|
@@ -2333,9 +2850,9 @@ var SearchResource = class {
|
|
|
2333
2850
|
async create(options) {
|
|
2334
2851
|
const body = {
|
|
2335
2852
|
query: options.query,
|
|
2336
|
-
limit: options.
|
|
2337
|
-
scrapeResults: options.
|
|
2338
|
-
maxScrapeResults: options.
|
|
2853
|
+
limit: options.limit,
|
|
2854
|
+
scrapeResults: options.scrapeResults,
|
|
2855
|
+
maxScrapeResults: options.maxScrapeResults,
|
|
2339
2856
|
useAiOptimization: options.useAiOptimization,
|
|
2340
2857
|
aiProvider: options.aiProvider,
|
|
2341
2858
|
aiModel: options.aiModel,
|
|
@@ -2389,7 +2906,7 @@ var SearchResource = class {
|
|
|
2389
2906
|
* @example
|
|
2390
2907
|
* ```ts
|
|
2391
2908
|
* const result = await client.search.andScrape("promo codes", {
|
|
2392
|
-
*
|
|
2909
|
+
* maxScrapeResults: 5
|
|
2393
2910
|
* });
|
|
2394
2911
|
* console.log(result.data.scrapedJobIds);
|
|
2395
2912
|
* ```
|
|
@@ -2397,7 +2914,7 @@ var SearchResource = class {
|
|
|
2397
2914
|
async andScrape(query, options) {
|
|
2398
2915
|
return this.create({
|
|
2399
2916
|
query,
|
|
2400
|
-
|
|
2917
|
+
scrapeResults: true,
|
|
2401
2918
|
...options
|
|
2402
2919
|
});
|
|
2403
2920
|
}
|
|
@@ -2580,7 +3097,9 @@ var WebhooksResource = class {
|
|
|
2580
3097
|
*/
|
|
2581
3098
|
async create(options) {
|
|
2582
3099
|
const result = await post(this.ctx, "/v1/webhooks", {
|
|
3100
|
+
events: options.events ?? (options.event ? [options.event] : void 0),
|
|
2583
3101
|
event: options.event,
|
|
3102
|
+
// Legacy fallback
|
|
2584
3103
|
url: options.url,
|
|
2585
3104
|
secret: options.secret,
|
|
2586
3105
|
minDealScore: options.minDealScore,
|
|
@@ -2618,7 +3137,7 @@ var WebhooksResource = class {
|
|
|
2618
3137
|
this.ctx,
|
|
2619
3138
|
`/v1/webhooks/${webhookId}`
|
|
2620
3139
|
);
|
|
2621
|
-
return result.data;
|
|
3140
|
+
return result.data.webhook;
|
|
2622
3141
|
}
|
|
2623
3142
|
/**
|
|
2624
3143
|
* Update a webhook
|
|
@@ -2676,7 +3195,8 @@ var WebhooksResource = class {
|
|
|
2676
3195
|
async test(webhookId) {
|
|
2677
3196
|
const result = await post(
|
|
2678
3197
|
this.ctx,
|
|
2679
|
-
|
|
3198
|
+
"/v1/webhooks/test",
|
|
3199
|
+
{ webhookId }
|
|
2680
3200
|
);
|
|
2681
3201
|
return result.data;
|
|
2682
3202
|
}
|
|
@@ -2714,7 +3234,7 @@ var WebhooksResource = class {
|
|
|
2714
3234
|
*/
|
|
2715
3235
|
async getActive() {
|
|
2716
3236
|
const all = await this.list();
|
|
2717
|
-
return all.
|
|
3237
|
+
return all.data.filter((w) => w.active);
|
|
2718
3238
|
}
|
|
2719
3239
|
/**
|
|
2720
3240
|
* Get webhooks by event type
|
|
@@ -2726,7 +3246,7 @@ var WebhooksResource = class {
|
|
|
2726
3246
|
*/
|
|
2727
3247
|
async getByEvent(event) {
|
|
2728
3248
|
const all = await this.list();
|
|
2729
|
-
return all.
|
|
3249
|
+
return all.data.filter((w) => w.event === event);
|
|
2730
3250
|
}
|
|
2731
3251
|
};
|
|
2732
3252
|
|
|
@@ -2811,6 +3331,20 @@ var DealCrawl = class {
|
|
|
2811
3331
|
* ```
|
|
2812
3332
|
*/
|
|
2813
3333
|
dork;
|
|
3334
|
+
/**
|
|
3335
|
+
* Convert resource - HTML to Markdown conversion
|
|
3336
|
+
*
|
|
3337
|
+
* @example
|
|
3338
|
+
* ```ts
|
|
3339
|
+
* const result = await client.convert.htmlToMarkdown({
|
|
3340
|
+
* html: "<h1>Title</h1><p>Content</p>",
|
|
3341
|
+
* baseUrl: "https://example.com",
|
|
3342
|
+
* options: { removeNoise: true }
|
|
3343
|
+
* });
|
|
3344
|
+
* console.log(result.data.markdown);
|
|
3345
|
+
* ```
|
|
3346
|
+
*/
|
|
3347
|
+
convert;
|
|
2814
3348
|
/**
|
|
2815
3349
|
* Agent resource - AI-powered autonomous web navigation
|
|
2816
3350
|
*
|
|
@@ -2892,6 +3426,80 @@ var DealCrawl = class {
|
|
|
2892
3426
|
* ```
|
|
2893
3427
|
*/
|
|
2894
3428
|
account;
|
|
3429
|
+
/**
|
|
3430
|
+
* Screenshots resource - Screenshot signed URL management
|
|
3431
|
+
*
|
|
3432
|
+
* @example
|
|
3433
|
+
* ```ts
|
|
3434
|
+
* // Refresh a signed URL before expiration
|
|
3435
|
+
* const refreshed = await client.screenshots.refresh({
|
|
3436
|
+
* path: "job_abc123/1234567890_nanoid_example.png",
|
|
3437
|
+
* ttl: 604800 // 7 days
|
|
3438
|
+
* });
|
|
3439
|
+
*
|
|
3440
|
+
* // Get tier-specific TTL limits
|
|
3441
|
+
* const limits = await client.screenshots.getLimits();
|
|
3442
|
+
* console.log(limits.formattedLimits.max); // "7 days"
|
|
3443
|
+
* ```
|
|
3444
|
+
*/
|
|
3445
|
+
screenshots;
|
|
3446
|
+
/**
|
|
3447
|
+
* Auth resource - SSE (Server-Sent Events) authentication
|
|
3448
|
+
*
|
|
3449
|
+
* @example
|
|
3450
|
+
* ```ts
|
|
3451
|
+
* // Generate SSE token for browser EventSource
|
|
3452
|
+
* const { token, expiresAt } = await client.auth.generateSSEToken();
|
|
3453
|
+
*
|
|
3454
|
+
* // Use in browser
|
|
3455
|
+
* const eventSource = new EventSource(`/v1/events?token=${token}`);
|
|
3456
|
+
*
|
|
3457
|
+
* // Generate token for specific job
|
|
3458
|
+
* const jobToken = await client.auth.generateSSEToken({ jobId: "job_123" });
|
|
3459
|
+
*
|
|
3460
|
+
* // Check connection limits
|
|
3461
|
+
* const limits = await client.auth.getLimits();
|
|
3462
|
+
* console.log(`Available connections: ${limits.sse.available}`);
|
|
3463
|
+
* ```
|
|
3464
|
+
*/
|
|
3465
|
+
auth;
|
|
3466
|
+
/**
|
|
3467
|
+
* Events resource - Real-time SSE event streaming (Browser only)
|
|
3468
|
+
*
|
|
3469
|
+
* IMPORTANT: This resource only works in browsers. For Node.js, use polling via client.status.get()
|
|
3470
|
+
*
|
|
3471
|
+
* @example Browser Usage
|
|
3472
|
+
* ```ts
|
|
3473
|
+
* // 1. Generate SSE token
|
|
3474
|
+
* const { token } = await client.auth.generateSSEToken();
|
|
3475
|
+
*
|
|
3476
|
+
* // 2. Subscribe to all events
|
|
3477
|
+
* const eventSource = client.events.subscribe(token, {
|
|
3478
|
+
* onEvent: (event) => {
|
|
3479
|
+
* console.log('Event:', event.type, JSON.parse(event.data));
|
|
3480
|
+
* }
|
|
3481
|
+
* });
|
|
3482
|
+
*
|
|
3483
|
+
* // 3. Or subscribe to specific job
|
|
3484
|
+
* const jobEvents = client.events.subscribeToJob('job_123', token, {
|
|
3485
|
+
* onEvent: (event) => {
|
|
3486
|
+
* const data = JSON.parse(event.data);
|
|
3487
|
+
* console.log(`Progress: ${data.progress}%`);
|
|
3488
|
+
* }
|
|
3489
|
+
* });
|
|
3490
|
+
*
|
|
3491
|
+
* // 4. Listen for specific events
|
|
3492
|
+
* eventSource.addEventListener('job.completed', (event) => {
|
|
3493
|
+
* const data = JSON.parse(event.data);
|
|
3494
|
+
* console.log('Job completed!', data.summary);
|
|
3495
|
+
* eventSource.close();
|
|
3496
|
+
* });
|
|
3497
|
+
*
|
|
3498
|
+
* // 5. Clean up
|
|
3499
|
+
* eventSource.close();
|
|
3500
|
+
* ```
|
|
3501
|
+
*/
|
|
3502
|
+
events;
|
|
2895
3503
|
// ============================================
|
|
2896
3504
|
// CONSTRUCTOR
|
|
2897
3505
|
// ============================================
|
|
@@ -2942,12 +3550,16 @@ var DealCrawl = class {
|
|
|
2942
3550
|
this.crawl = new CrawlResource(this.ctx);
|
|
2943
3551
|
this.extract = new ExtractResource(this.ctx);
|
|
2944
3552
|
this.dork = new DorkResource(this.ctx);
|
|
3553
|
+
this.convert = new ConvertResource(this.ctx);
|
|
2945
3554
|
this.agent = new AgentResource(this.ctx);
|
|
2946
3555
|
this.status = new StatusResource(this.ctx);
|
|
2947
3556
|
this.data = new DataResource(this.ctx);
|
|
2948
3557
|
this.webhooks = new WebhooksResource(this.ctx);
|
|
2949
3558
|
this.keys = new KeysResource(this.ctx);
|
|
2950
3559
|
this.account = new AccountResource(this.ctx);
|
|
3560
|
+
this.screenshots = new ScreenshotsResource(this.ctx);
|
|
3561
|
+
this.auth = new AuthResource(this.ctx);
|
|
3562
|
+
this.events = new EventsResource(this.ctx);
|
|
2951
3563
|
}
|
|
2952
3564
|
// ============================================
|
|
2953
3565
|
// POLLING METHODS
|
|
@@ -3093,6 +3705,8 @@ var DealCrawl = class {
|
|
|
3093
3705
|
|
|
3094
3706
|
exports.AccountResource = AccountResource;
|
|
3095
3707
|
exports.AgentResource = AgentResource;
|
|
3708
|
+
exports.AuthResource = AuthResource;
|
|
3709
|
+
exports.ConvertResource = ConvertResource;
|
|
3096
3710
|
exports.CrawlResource = CrawlResource;
|
|
3097
3711
|
exports.DEFAULT_CONFIG = DEFAULT_CONFIG;
|
|
3098
3712
|
exports.DataResource = DataResource;
|
|
@@ -3101,9 +3715,11 @@ exports.DealCrawlError = DealCrawlError;
|
|
|
3101
3715
|
exports.DorkResource = DorkResource;
|
|
3102
3716
|
exports.ERROR_CODES = ERROR_CODES;
|
|
3103
3717
|
exports.ERROR_MESSAGES = ERROR_MESSAGES;
|
|
3718
|
+
exports.EventsResource = EventsResource;
|
|
3104
3719
|
exports.ExtractResource = ExtractResource;
|
|
3105
3720
|
exports.KeysResource = KeysResource;
|
|
3106
3721
|
exports.ScrapeResource = ScrapeResource;
|
|
3722
|
+
exports.ScreenshotsResource = ScreenshotsResource;
|
|
3107
3723
|
exports.SearchResource = SearchResource;
|
|
3108
3724
|
exports.StatusResource = StatusResource;
|
|
3109
3725
|
exports.WebhooksResource = WebhooksResource;
|