@dealcrawl/sdk 2.1.3 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +147 -14
- package/dist/index.d.mts +445 -11
- package/dist/index.d.ts +445 -11
- package/dist/index.js +252 -3
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +252 -4
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1544,11 +1544,21 @@ var KeysResource = class {
|
|
|
1544
1544
|
*
|
|
1545
1545
|
* @example
|
|
1546
1546
|
* ```ts
|
|
1547
|
+
* // Simple revoke
|
|
1547
1548
|
* await client.keys.revoke("key_abc123");
|
|
1549
|
+
*
|
|
1550
|
+
* // With reason
|
|
1551
|
+
* await client.keys.revoke("key_abc123", {
|
|
1552
|
+
* reason: "Key compromised"
|
|
1553
|
+
* });
|
|
1548
1554
|
* ```
|
|
1549
1555
|
*/
|
|
1550
|
-
async revoke(keyId) {
|
|
1551
|
-
const result = await del(
|
|
1556
|
+
async revoke(keyId, options = {}) {
|
|
1557
|
+
const result = await del(
|
|
1558
|
+
this.ctx,
|
|
1559
|
+
`/v1/keys/${keyId}`,
|
|
1560
|
+
options.reason ? { reason: options.reason } : void 0
|
|
1561
|
+
);
|
|
1552
1562
|
return result.data;
|
|
1553
1563
|
}
|
|
1554
1564
|
/**
|
|
@@ -1624,6 +1634,8 @@ var ScrapeResource = class {
|
|
|
1624
1634
|
detectSignals: options.detectSignals ?? true,
|
|
1625
1635
|
extractWithAI: options.extractWithAI,
|
|
1626
1636
|
extractDeal: options.extractDeal,
|
|
1637
|
+
extractMultipleDeals: options.extractMultipleDeals,
|
|
1638
|
+
maxDeals: options.maxDeals,
|
|
1627
1639
|
useAdvancedModel: options.useAdvancedModel,
|
|
1628
1640
|
minDealScore: options.minDealScore,
|
|
1629
1641
|
screenshot: options.screenshot,
|
|
@@ -1652,6 +1664,25 @@ var ScrapeResource = class {
|
|
|
1652
1664
|
...options
|
|
1653
1665
|
});
|
|
1654
1666
|
}
|
|
1667
|
+
/**
|
|
1668
|
+
* Scrape a list page and extract multiple deals
|
|
1669
|
+
* Use for category pages, deal lists, search results
|
|
1670
|
+
*
|
|
1671
|
+
* @example
|
|
1672
|
+
* ```ts
|
|
1673
|
+
* const job = await client.scrape.extractDeals(
|
|
1674
|
+
* "https://amazon.fr/deals",
|
|
1675
|
+
* { maxDeals: 30, useAdvancedModel: true }
|
|
1676
|
+
* );
|
|
1677
|
+
* ```
|
|
1678
|
+
*/
|
|
1679
|
+
async extractDeals(url, options) {
|
|
1680
|
+
return this.create({
|
|
1681
|
+
url,
|
|
1682
|
+
extractMultipleDeals: true,
|
|
1683
|
+
...options
|
|
1684
|
+
});
|
|
1685
|
+
}
|
|
1655
1686
|
/**
|
|
1656
1687
|
* Scrape a URL with screenshot capture
|
|
1657
1688
|
* Convenience method for screenshot capture
|
|
@@ -1674,6 +1705,185 @@ var ScrapeResource = class {
|
|
|
1674
1705
|
...options
|
|
1675
1706
|
});
|
|
1676
1707
|
}
|
|
1708
|
+
// ============================================
|
|
1709
|
+
// BATCH SCRAPING METHODS
|
|
1710
|
+
// ============================================
|
|
1711
|
+
/**
|
|
1712
|
+
* Create a batch scrape job for multiple URLs
|
|
1713
|
+
* Scrapes 1-100 URLs in a single request
|
|
1714
|
+
*
|
|
1715
|
+
* @example
|
|
1716
|
+
* ```ts
|
|
1717
|
+
* const batch = await client.scrape.batch({
|
|
1718
|
+
* urls: [
|
|
1719
|
+
* { url: "https://shop1.com/product1" },
|
|
1720
|
+
* { url: "https://shop2.com/deal", extractDeal: true }
|
|
1721
|
+
* ],
|
|
1722
|
+
* defaults: { detectSignals: true }
|
|
1723
|
+
* });
|
|
1724
|
+
* console.log(batch.batchId, batch.results);
|
|
1725
|
+
* ```
|
|
1726
|
+
*/
|
|
1727
|
+
async batch(options) {
|
|
1728
|
+
const body = {
|
|
1729
|
+
urls: options.urls,
|
|
1730
|
+
defaults: options.defaults,
|
|
1731
|
+
webhookUrl: options.webhookUrl,
|
|
1732
|
+
priority: options.priority,
|
|
1733
|
+
delayMs: options.delay
|
|
1734
|
+
};
|
|
1735
|
+
const result = await post(
|
|
1736
|
+
this.ctx,
|
|
1737
|
+
"/v1/scrape/batch",
|
|
1738
|
+
body
|
|
1739
|
+
);
|
|
1740
|
+
return result.data;
|
|
1741
|
+
}
|
|
1742
|
+
/**
|
|
1743
|
+
* Get status of a batch scrape operation
|
|
1744
|
+
*
|
|
1745
|
+
* @example
|
|
1746
|
+
* ```ts
|
|
1747
|
+
* const status = await client.scrape.getBatchStatus(batchId);
|
|
1748
|
+
* console.log(`Completed: ${status.completed}/${status.total}`);
|
|
1749
|
+
* ```
|
|
1750
|
+
*/
|
|
1751
|
+
async getBatchStatus(batchId) {
|
|
1752
|
+
const result = await get(
|
|
1753
|
+
this.ctx,
|
|
1754
|
+
`/v1/scrape/batch/${batchId}`
|
|
1755
|
+
);
|
|
1756
|
+
return result.data;
|
|
1757
|
+
}
|
|
1758
|
+
/**
|
|
1759
|
+
* Batch scrape with deal extraction enabled for all URLs
|
|
1760
|
+
* Convenience method for deal-focused batch scraping
|
|
1761
|
+
*
|
|
1762
|
+
* @example
|
|
1763
|
+
* ```ts
|
|
1764
|
+
* const batch = await client.scrape.batchForDeals([
|
|
1765
|
+
* "https://shop1.com/sale",
|
|
1766
|
+
* "https://shop2.com/deals"
|
|
1767
|
+
* ]);
|
|
1768
|
+
* ```
|
|
1769
|
+
*/
|
|
1770
|
+
async batchForDeals(urls, options) {
|
|
1771
|
+
return this.batch({
|
|
1772
|
+
urls: urls.map((url) => ({ url })),
|
|
1773
|
+
defaults: {
|
|
1774
|
+
extractDeal: true,
|
|
1775
|
+
detectSignals: true,
|
|
1776
|
+
...options?.defaults
|
|
1777
|
+
},
|
|
1778
|
+
...options
|
|
1779
|
+
});
|
|
1780
|
+
}
|
|
1781
|
+
};
|
|
1782
|
+
|
|
1783
|
+
// src/resources/search.ts
|
|
1784
|
+
var SearchResource = class {
|
|
1785
|
+
constructor(ctx) {
|
|
1786
|
+
this.ctx = ctx;
|
|
1787
|
+
}
|
|
1788
|
+
/**
|
|
1789
|
+
* Create a new search job
|
|
1790
|
+
*
|
|
1791
|
+
* @example
|
|
1792
|
+
* ```ts
|
|
1793
|
+
* const result = await client.search.create({
|
|
1794
|
+
* query: "laptop deals black friday",
|
|
1795
|
+
* maxResults: 20,
|
|
1796
|
+
* useDealScoring: true
|
|
1797
|
+
* });
|
|
1798
|
+
* ```
|
|
1799
|
+
*/
|
|
1800
|
+
async create(options) {
|
|
1801
|
+
const body = {
|
|
1802
|
+
query: options.query,
|
|
1803
|
+
limit: options.maxResults,
|
|
1804
|
+
scrapeResults: options.autoScrape,
|
|
1805
|
+
maxScrapeResults: options.autoScrapeLimit,
|
|
1806
|
+
useAiOptimization: options.useAiOptimization,
|
|
1807
|
+
aiProvider: options.aiProvider,
|
|
1808
|
+
aiModel: options.aiModel,
|
|
1809
|
+
useDealScoring: options.useDealScoring,
|
|
1810
|
+
skipCache: options.skipCache,
|
|
1811
|
+
filters: options.filters,
|
|
1812
|
+
webhook: options.webhook
|
|
1813
|
+
};
|
|
1814
|
+
const result = await post(this.ctx, "/v1/search", body);
|
|
1815
|
+
return result.data;
|
|
1816
|
+
}
|
|
1817
|
+
/**
|
|
1818
|
+
* Search with AI query optimization
|
|
1819
|
+
* Convenience method for AI-enhanced searches
|
|
1820
|
+
*
|
|
1821
|
+
* @example
|
|
1822
|
+
* ```ts
|
|
1823
|
+
* const result = await client.search.withAI("iphone discount", {
|
|
1824
|
+
* aiProvider: "openai",
|
|
1825
|
+
* aiModel: "gpt-4o-mini"
|
|
1826
|
+
* });
|
|
1827
|
+
* ```
|
|
1828
|
+
*/
|
|
1829
|
+
async withAI(query, options) {
|
|
1830
|
+
return this.create({
|
|
1831
|
+
query,
|
|
1832
|
+
useAiOptimization: true,
|
|
1833
|
+
...options
|
|
1834
|
+
});
|
|
1835
|
+
}
|
|
1836
|
+
/**
|
|
1837
|
+
* Search with deal scoring enabled
|
|
1838
|
+
* Scores each result for deal relevance (0-100)
|
|
1839
|
+
*
|
|
1840
|
+
* @example
|
|
1841
|
+
* ```ts
|
|
1842
|
+
* const result = await client.search.forDeals("gaming laptop");
|
|
1843
|
+
* ```
|
|
1844
|
+
*/
|
|
1845
|
+
async forDeals(query, options) {
|
|
1846
|
+
return this.create({
|
|
1847
|
+
query,
|
|
1848
|
+
useDealScoring: true,
|
|
1849
|
+
...options
|
|
1850
|
+
});
|
|
1851
|
+
}
|
|
1852
|
+
/**
|
|
1853
|
+
* Search and auto-scrape top results
|
|
1854
|
+
* Creates scrape jobs for the best matching URLs
|
|
1855
|
+
*
|
|
1856
|
+
* @example
|
|
1857
|
+
* ```ts
|
|
1858
|
+
* const result = await client.search.andScrape("promo codes", {
|
|
1859
|
+
* autoScrapeLimit: 5
|
|
1860
|
+
* });
|
|
1861
|
+
* console.log(result.data.scrapedJobIds);
|
|
1862
|
+
* ```
|
|
1863
|
+
*/
|
|
1864
|
+
async andScrape(query, options) {
|
|
1865
|
+
return this.create({
|
|
1866
|
+
query,
|
|
1867
|
+
autoScrape: true,
|
|
1868
|
+
...options
|
|
1869
|
+
});
|
|
1870
|
+
}
|
|
1871
|
+
/**
|
|
1872
|
+
* Check search API status
|
|
1873
|
+
* Returns availability and features info
|
|
1874
|
+
*
|
|
1875
|
+
* @example
|
|
1876
|
+
* ```ts
|
|
1877
|
+
* const status = await client.search.getStatus();
|
|
1878
|
+
* if (status.available) {
|
|
1879
|
+
* console.log(`Provider: ${status.provider}`);
|
|
1880
|
+
* }
|
|
1881
|
+
* ```
|
|
1882
|
+
*/
|
|
1883
|
+
async getStatus() {
|
|
1884
|
+
const result = await get(this.ctx, "/v1/search/status");
|
|
1885
|
+
return result.data;
|
|
1886
|
+
}
|
|
1677
1887
|
};
|
|
1678
1888
|
|
|
1679
1889
|
// src/resources/status.ts
|
|
@@ -1995,17 +2205,38 @@ var DealCrawl = class {
|
|
|
1995
2205
|
// RESOURCES
|
|
1996
2206
|
// ============================================
|
|
1997
2207
|
/**
|
|
1998
|
-
* Scrape resource - Single page scraping
|
|
2208
|
+
* Scrape resource - Single page and batch scraping
|
|
1999
2209
|
*
|
|
2000
2210
|
* @example
|
|
2001
2211
|
* ```ts
|
|
2212
|
+
* // Single page
|
|
2002
2213
|
* const job = await client.scrape.create({
|
|
2003
2214
|
* url: "https://example.com",
|
|
2004
2215
|
* extractDeal: true
|
|
2005
2216
|
* });
|
|
2217
|
+
*
|
|
2218
|
+
* // Batch scraping
|
|
2219
|
+
* const batch = await client.scrape.batch({
|
|
2220
|
+
* urls: [{ url: "https://shop1.com" }, { url: "https://shop2.com" }]
|
|
2221
|
+
* });
|
|
2006
2222
|
* ```
|
|
2007
2223
|
*/
|
|
2008
2224
|
scrape;
|
|
2225
|
+
/**
|
|
2226
|
+
* Search resource - Web search with AI optimization
|
|
2227
|
+
*
|
|
2228
|
+
* @example
|
|
2229
|
+
* ```ts
|
|
2230
|
+
* const result = await client.search.create({
|
|
2231
|
+
* query: "laptop deals",
|
|
2232
|
+
* useDealScoring: true
|
|
2233
|
+
* });
|
|
2234
|
+
*
|
|
2235
|
+
* // With AI optimization
|
|
2236
|
+
* const result = await client.search.withAI("iphone discount");
|
|
2237
|
+
* ```
|
|
2238
|
+
*/
|
|
2239
|
+
search;
|
|
2009
2240
|
/**
|
|
2010
2241
|
* Crawl resource - Website crawling
|
|
2011
2242
|
*
|
|
@@ -2142,6 +2373,7 @@ var DealCrawl = class {
|
|
|
2142
2373
|
onRateLimit: config.onRateLimit
|
|
2143
2374
|
};
|
|
2144
2375
|
this.scrape = new ScrapeResource(this.ctx);
|
|
2376
|
+
this.search = new SearchResource(this.ctx);
|
|
2145
2377
|
this.crawl = new CrawlResource(this.ctx);
|
|
2146
2378
|
this.extract = new ExtractResource(this.ctx);
|
|
2147
2379
|
this.dork = new DorkResource(this.ctx);
|
|
@@ -2257,6 +2489,22 @@ var DealCrawl = class {
|
|
|
2257
2489
|
const job = await this.extract.create(options);
|
|
2258
2490
|
return this.waitForResult(job.jobId, waitOptions);
|
|
2259
2491
|
}
|
|
2492
|
+
/**
|
|
2493
|
+
* Search and return results directly
|
|
2494
|
+
* Note: Search is synchronous, no waiting needed
|
|
2495
|
+
*
|
|
2496
|
+
* @example
|
|
2497
|
+
* ```ts
|
|
2498
|
+
* const result = await client.searchAndWait({
|
|
2499
|
+
* query: "gaming laptop deals",
|
|
2500
|
+
* useDealScoring: true
|
|
2501
|
+
* });
|
|
2502
|
+
* console.log(result.data.results);
|
|
2503
|
+
* ```
|
|
2504
|
+
*/
|
|
2505
|
+
async searchAndWait(options) {
|
|
2506
|
+
return this.search.create(options);
|
|
2507
|
+
}
|
|
2260
2508
|
};
|
|
2261
2509
|
|
|
2262
2510
|
exports.AccountResource = AccountResource;
|
|
@@ -2270,6 +2518,7 @@ exports.ERROR_CODES = ERROR_CODES;
|
|
|
2270
2518
|
exports.ExtractResource = ExtractResource;
|
|
2271
2519
|
exports.KeysResource = KeysResource;
|
|
2272
2520
|
exports.ScrapeResource = ScrapeResource;
|
|
2521
|
+
exports.SearchResource = SearchResource;
|
|
2273
2522
|
exports.StatusResource = StatusResource;
|
|
2274
2523
|
exports.WebhooksResource = WebhooksResource;
|
|
2275
2524
|
exports.default = DealCrawl;
|