@dealcrawl/sdk 2.1.3 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +147 -14
- package/dist/index.d.mts +445 -11
- package/dist/index.d.ts +445 -11
- package/dist/index.js +252 -3
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +252 -4
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -1540,11 +1540,21 @@ var KeysResource = class {
|
|
|
1540
1540
|
*
|
|
1541
1541
|
* @example
|
|
1542
1542
|
* ```ts
|
|
1543
|
+
* // Simple revoke
|
|
1543
1544
|
* await client.keys.revoke("key_abc123");
|
|
1545
|
+
*
|
|
1546
|
+
* // With reason
|
|
1547
|
+
* await client.keys.revoke("key_abc123", {
|
|
1548
|
+
* reason: "Key compromised"
|
|
1549
|
+
* });
|
|
1544
1550
|
* ```
|
|
1545
1551
|
*/
|
|
1546
|
-
async revoke(keyId) {
|
|
1547
|
-
const result = await del(
|
|
1552
|
+
async revoke(keyId, options = {}) {
|
|
1553
|
+
const result = await del(
|
|
1554
|
+
this.ctx,
|
|
1555
|
+
`/v1/keys/${keyId}`,
|
|
1556
|
+
options.reason ? { reason: options.reason } : void 0
|
|
1557
|
+
);
|
|
1548
1558
|
return result.data;
|
|
1549
1559
|
}
|
|
1550
1560
|
/**
|
|
@@ -1620,6 +1630,8 @@ var ScrapeResource = class {
|
|
|
1620
1630
|
detectSignals: options.detectSignals ?? true,
|
|
1621
1631
|
extractWithAI: options.extractWithAI,
|
|
1622
1632
|
extractDeal: options.extractDeal,
|
|
1633
|
+
extractMultipleDeals: options.extractMultipleDeals,
|
|
1634
|
+
maxDeals: options.maxDeals,
|
|
1623
1635
|
useAdvancedModel: options.useAdvancedModel,
|
|
1624
1636
|
minDealScore: options.minDealScore,
|
|
1625
1637
|
screenshot: options.screenshot,
|
|
@@ -1648,6 +1660,25 @@ var ScrapeResource = class {
|
|
|
1648
1660
|
...options
|
|
1649
1661
|
});
|
|
1650
1662
|
}
|
|
1663
|
+
/**
|
|
1664
|
+
* Scrape a list page and extract multiple deals
|
|
1665
|
+
* Use for category pages, deal lists, search results
|
|
1666
|
+
*
|
|
1667
|
+
* @example
|
|
1668
|
+
* ```ts
|
|
1669
|
+
* const job = await client.scrape.extractDeals(
|
|
1670
|
+
* "https://amazon.fr/deals",
|
|
1671
|
+
* { maxDeals: 30, useAdvancedModel: true }
|
|
1672
|
+
* );
|
|
1673
|
+
* ```
|
|
1674
|
+
*/
|
|
1675
|
+
async extractDeals(url, options) {
|
|
1676
|
+
return this.create({
|
|
1677
|
+
url,
|
|
1678
|
+
extractMultipleDeals: true,
|
|
1679
|
+
...options
|
|
1680
|
+
});
|
|
1681
|
+
}
|
|
1651
1682
|
/**
|
|
1652
1683
|
* Scrape a URL with screenshot capture
|
|
1653
1684
|
* Convenience method for screenshot capture
|
|
@@ -1670,6 +1701,185 @@ var ScrapeResource = class {
|
|
|
1670
1701
|
...options
|
|
1671
1702
|
});
|
|
1672
1703
|
}
|
|
1704
|
+
// ============================================
|
|
1705
|
+
// BATCH SCRAPING METHODS
|
|
1706
|
+
// ============================================
|
|
1707
|
+
/**
|
|
1708
|
+
* Create a batch scrape job for multiple URLs
|
|
1709
|
+
* Scrapes 1-100 URLs in a single request
|
|
1710
|
+
*
|
|
1711
|
+
* @example
|
|
1712
|
+
* ```ts
|
|
1713
|
+
* const batch = await client.scrape.batch({
|
|
1714
|
+
* urls: [
|
|
1715
|
+
* { url: "https://shop1.com/product1" },
|
|
1716
|
+
* { url: "https://shop2.com/deal", extractDeal: true }
|
|
1717
|
+
* ],
|
|
1718
|
+
* defaults: { detectSignals: true }
|
|
1719
|
+
* });
|
|
1720
|
+
* console.log(batch.batchId, batch.results);
|
|
1721
|
+
* ```
|
|
1722
|
+
*/
|
|
1723
|
+
async batch(options) {
|
|
1724
|
+
const body = {
|
|
1725
|
+
urls: options.urls,
|
|
1726
|
+
defaults: options.defaults,
|
|
1727
|
+
webhookUrl: options.webhookUrl,
|
|
1728
|
+
priority: options.priority,
|
|
1729
|
+
delayMs: options.delay
|
|
1730
|
+
};
|
|
1731
|
+
const result = await post(
|
|
1732
|
+
this.ctx,
|
|
1733
|
+
"/v1/scrape/batch",
|
|
1734
|
+
body
|
|
1735
|
+
);
|
|
1736
|
+
return result.data;
|
|
1737
|
+
}
|
|
1738
|
+
/**
|
|
1739
|
+
* Get status of a batch scrape operation
|
|
1740
|
+
*
|
|
1741
|
+
* @example
|
|
1742
|
+
* ```ts
|
|
1743
|
+
* const status = await client.scrape.getBatchStatus(batchId);
|
|
1744
|
+
* console.log(`Completed: ${status.completed}/${status.total}`);
|
|
1745
|
+
* ```
|
|
1746
|
+
*/
|
|
1747
|
+
async getBatchStatus(batchId) {
|
|
1748
|
+
const result = await get(
|
|
1749
|
+
this.ctx,
|
|
1750
|
+
`/v1/scrape/batch/${batchId}`
|
|
1751
|
+
);
|
|
1752
|
+
return result.data;
|
|
1753
|
+
}
|
|
1754
|
+
/**
|
|
1755
|
+
* Batch scrape with deal extraction enabled for all URLs
|
|
1756
|
+
* Convenience method for deal-focused batch scraping
|
|
1757
|
+
*
|
|
1758
|
+
* @example
|
|
1759
|
+
* ```ts
|
|
1760
|
+
* const batch = await client.scrape.batchForDeals([
|
|
1761
|
+
* "https://shop1.com/sale",
|
|
1762
|
+
* "https://shop2.com/deals"
|
|
1763
|
+
* ]);
|
|
1764
|
+
* ```
|
|
1765
|
+
*/
|
|
1766
|
+
async batchForDeals(urls, options) {
|
|
1767
|
+
return this.batch({
|
|
1768
|
+
urls: urls.map((url) => ({ url })),
|
|
1769
|
+
defaults: {
|
|
1770
|
+
extractDeal: true,
|
|
1771
|
+
detectSignals: true,
|
|
1772
|
+
...options?.defaults
|
|
1773
|
+
},
|
|
1774
|
+
...options
|
|
1775
|
+
});
|
|
1776
|
+
}
|
|
1777
|
+
};
|
|
1778
|
+
|
|
1779
|
+
// src/resources/search.ts
|
|
1780
|
+
var SearchResource = class {
|
|
1781
|
+
constructor(ctx) {
|
|
1782
|
+
this.ctx = ctx;
|
|
1783
|
+
}
|
|
1784
|
+
/**
|
|
1785
|
+
* Create a new search job
|
|
1786
|
+
*
|
|
1787
|
+
* @example
|
|
1788
|
+
* ```ts
|
|
1789
|
+
* const result = await client.search.create({
|
|
1790
|
+
* query: "laptop deals black friday",
|
|
1791
|
+
* maxResults: 20,
|
|
1792
|
+
* useDealScoring: true
|
|
1793
|
+
* });
|
|
1794
|
+
* ```
|
|
1795
|
+
*/
|
|
1796
|
+
async create(options) {
|
|
1797
|
+
const body = {
|
|
1798
|
+
query: options.query,
|
|
1799
|
+
limit: options.maxResults,
|
|
1800
|
+
scrapeResults: options.autoScrape,
|
|
1801
|
+
maxScrapeResults: options.autoScrapeLimit,
|
|
1802
|
+
useAiOptimization: options.useAiOptimization,
|
|
1803
|
+
aiProvider: options.aiProvider,
|
|
1804
|
+
aiModel: options.aiModel,
|
|
1805
|
+
useDealScoring: options.useDealScoring,
|
|
1806
|
+
skipCache: options.skipCache,
|
|
1807
|
+
filters: options.filters,
|
|
1808
|
+
webhook: options.webhook
|
|
1809
|
+
};
|
|
1810
|
+
const result = await post(this.ctx, "/v1/search", body);
|
|
1811
|
+
return result.data;
|
|
1812
|
+
}
|
|
1813
|
+
/**
|
|
1814
|
+
* Search with AI query optimization
|
|
1815
|
+
* Convenience method for AI-enhanced searches
|
|
1816
|
+
*
|
|
1817
|
+
* @example
|
|
1818
|
+
* ```ts
|
|
1819
|
+
* const result = await client.search.withAI("iphone discount", {
|
|
1820
|
+
* aiProvider: "openai",
|
|
1821
|
+
* aiModel: "gpt-4o-mini"
|
|
1822
|
+
* });
|
|
1823
|
+
* ```
|
|
1824
|
+
*/
|
|
1825
|
+
async withAI(query, options) {
|
|
1826
|
+
return this.create({
|
|
1827
|
+
query,
|
|
1828
|
+
useAiOptimization: true,
|
|
1829
|
+
...options
|
|
1830
|
+
});
|
|
1831
|
+
}
|
|
1832
|
+
/**
|
|
1833
|
+
* Search with deal scoring enabled
|
|
1834
|
+
* Scores each result for deal relevance (0-100)
|
|
1835
|
+
*
|
|
1836
|
+
* @example
|
|
1837
|
+
* ```ts
|
|
1838
|
+
* const result = await client.search.forDeals("gaming laptop");
|
|
1839
|
+
* ```
|
|
1840
|
+
*/
|
|
1841
|
+
async forDeals(query, options) {
|
|
1842
|
+
return this.create({
|
|
1843
|
+
query,
|
|
1844
|
+
useDealScoring: true,
|
|
1845
|
+
...options
|
|
1846
|
+
});
|
|
1847
|
+
}
|
|
1848
|
+
/**
|
|
1849
|
+
* Search and auto-scrape top results
|
|
1850
|
+
* Creates scrape jobs for the best matching URLs
|
|
1851
|
+
*
|
|
1852
|
+
* @example
|
|
1853
|
+
* ```ts
|
|
1854
|
+
* const result = await client.search.andScrape("promo codes", {
|
|
1855
|
+
* autoScrapeLimit: 5
|
|
1856
|
+
* });
|
|
1857
|
+
* console.log(result.data.scrapedJobIds);
|
|
1858
|
+
* ```
|
|
1859
|
+
*/
|
|
1860
|
+
async andScrape(query, options) {
|
|
1861
|
+
return this.create({
|
|
1862
|
+
query,
|
|
1863
|
+
autoScrape: true,
|
|
1864
|
+
...options
|
|
1865
|
+
});
|
|
1866
|
+
}
|
|
1867
|
+
/**
|
|
1868
|
+
* Check search API status
|
|
1869
|
+
* Returns availability and features info
|
|
1870
|
+
*
|
|
1871
|
+
* @example
|
|
1872
|
+
* ```ts
|
|
1873
|
+
* const status = await client.search.getStatus();
|
|
1874
|
+
* if (status.available) {
|
|
1875
|
+
* console.log(`Provider: ${status.provider}`);
|
|
1876
|
+
* }
|
|
1877
|
+
* ```
|
|
1878
|
+
*/
|
|
1879
|
+
async getStatus() {
|
|
1880
|
+
const result = await get(this.ctx, "/v1/search/status");
|
|
1881
|
+
return result.data;
|
|
1882
|
+
}
|
|
1673
1883
|
};
|
|
1674
1884
|
|
|
1675
1885
|
// src/resources/status.ts
|
|
@@ -1991,17 +2201,38 @@ var DealCrawl = class {
|
|
|
1991
2201
|
// RESOURCES
|
|
1992
2202
|
// ============================================
|
|
1993
2203
|
/**
|
|
1994
|
-
* Scrape resource - Single page scraping
|
|
2204
|
+
* Scrape resource - Single page and batch scraping
|
|
1995
2205
|
*
|
|
1996
2206
|
* @example
|
|
1997
2207
|
* ```ts
|
|
2208
|
+
* // Single page
|
|
1998
2209
|
* const job = await client.scrape.create({
|
|
1999
2210
|
* url: "https://example.com",
|
|
2000
2211
|
* extractDeal: true
|
|
2001
2212
|
* });
|
|
2213
|
+
*
|
|
2214
|
+
* // Batch scraping
|
|
2215
|
+
* const batch = await client.scrape.batch({
|
|
2216
|
+
* urls: [{ url: "https://shop1.com" }, { url: "https://shop2.com" }]
|
|
2217
|
+
* });
|
|
2002
2218
|
* ```
|
|
2003
2219
|
*/
|
|
2004
2220
|
scrape;
|
|
2221
|
+
/**
|
|
2222
|
+
* Search resource - Web search with AI optimization
|
|
2223
|
+
*
|
|
2224
|
+
* @example
|
|
2225
|
+
* ```ts
|
|
2226
|
+
* const result = await client.search.create({
|
|
2227
|
+
* query: "laptop deals",
|
|
2228
|
+
* useDealScoring: true
|
|
2229
|
+
* });
|
|
2230
|
+
*
|
|
2231
|
+
* // With AI optimization
|
|
2232
|
+
* const result = await client.search.withAI("iphone discount");
|
|
2233
|
+
* ```
|
|
2234
|
+
*/
|
|
2235
|
+
search;
|
|
2005
2236
|
/**
|
|
2006
2237
|
* Crawl resource - Website crawling
|
|
2007
2238
|
*
|
|
@@ -2138,6 +2369,7 @@ var DealCrawl = class {
|
|
|
2138
2369
|
onRateLimit: config.onRateLimit
|
|
2139
2370
|
};
|
|
2140
2371
|
this.scrape = new ScrapeResource(this.ctx);
|
|
2372
|
+
this.search = new SearchResource(this.ctx);
|
|
2141
2373
|
this.crawl = new CrawlResource(this.ctx);
|
|
2142
2374
|
this.extract = new ExtractResource(this.ctx);
|
|
2143
2375
|
this.dork = new DorkResource(this.ctx);
|
|
@@ -2253,8 +2485,24 @@ var DealCrawl = class {
|
|
|
2253
2485
|
const job = await this.extract.create(options);
|
|
2254
2486
|
return this.waitForResult(job.jobId, waitOptions);
|
|
2255
2487
|
}
|
|
2488
|
+
/**
|
|
2489
|
+
* Search and return results directly
|
|
2490
|
+
* Note: Search is synchronous, no waiting needed
|
|
2491
|
+
*
|
|
2492
|
+
* @example
|
|
2493
|
+
* ```ts
|
|
2494
|
+
* const result = await client.searchAndWait({
|
|
2495
|
+
* query: "gaming laptop deals",
|
|
2496
|
+
* useDealScoring: true
|
|
2497
|
+
* });
|
|
2498
|
+
* console.log(result.data.results);
|
|
2499
|
+
* ```
|
|
2500
|
+
*/
|
|
2501
|
+
async searchAndWait(options) {
|
|
2502
|
+
return this.search.create(options);
|
|
2503
|
+
}
|
|
2256
2504
|
};
|
|
2257
2505
|
|
|
2258
|
-
export { AccountResource, CrawlResource, DEFAULT_CONFIG, DataResource, DealCrawl, DealCrawlError, DorkResource, ERROR_CODES, ExtractResource, KeysResource, ScrapeResource, StatusResource, WebhooksResource, DealCrawl as default, pollUntil, waitForAll, waitForAny, waitForResult };
|
|
2506
|
+
export { AccountResource, CrawlResource, DEFAULT_CONFIG, DataResource, DealCrawl, DealCrawlError, DorkResource, ERROR_CODES, ExtractResource, KeysResource, ScrapeResource, SearchResource, StatusResource, WebhooksResource, DealCrawl as default, pollUntil, waitForAll, waitForAny, waitForResult };
|
|
2259
2507
|
//# sourceMappingURL=index.mjs.map
|
|
2260
2508
|
//# sourceMappingURL=index.mjs.map
|