@dealcrawl/sdk 2.0.0

package/README.md

@@ -0,0 +1,540 @@
+# @dealcrawl/sdk
+
+Official TypeScript SDK for the DealCrawl web scraping and crawling API.
+
+[![npm version](https://img.shields.io/npm/v/@dealcrawl/sdk.svg)](https://www.npmjs.com/package/@dealcrawl/sdk)
+[![TypeScript](https://img.shields.io/badge/TypeScript-5.0+-blue.svg)](https://www.typescriptlang.org/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+## Features
+
+- 🚀 **Full API Coverage** - Access all 41 DealCrawl API endpoints
+- 📦 **Zero Dependencies** - Uses native `fetch`, works everywhere
+- 🔒 **Type-Safe** - Complete TypeScript definitions
+- ⚡ **Automatic Retries** - Built-in retry logic with exponential backoff
+- 🔄 **Polling Helpers** - `waitForResult()` for async job completion
+- 🎯 **Resource Pattern** - Stripe/Twilio-style API design
+
+## Installation
+
+```bash
+# npm
+npm install @dealcrawl/sdk
+
+# bun
+bun add @dealcrawl/sdk
+
+# pnpm
+pnpm add @dealcrawl/sdk
+
+# yarn
+yarn add @dealcrawl/sdk
+```
+
+## Quick Start
+
+```typescript
+import { DealCrawl } from "@dealcrawl/sdk";
+
+const client = new DealCrawl({
+  apiKey: process.env.DEALCRAWL_API_KEY!,
+});
+
+// Scrape a single page with deal extraction
+const job = await client.scrape.create({
+  url: "https://shop.example.com/product",
+  extractDeal: true,
+});
+
+// Wait for result with automatic polling
+const result = await client.waitForResult(job.jobId);
+console.log(result);
+```
+
+## Configuration
+
+```typescript
+const client = new DealCrawl({
+  apiKey: "sk_xxx", // Required
+  baseUrl: "https://api.dealcrawl.dev", // Optional (default)
+  timeout: 30000, // Request timeout in ms
+  maxRetries: 3, // Retry attempts
+  retryDelay: 1000, // Base retry delay in ms
+  onRateLimit: (info) => console.log(info), // Rate limit callback
+});
+```
+
+## Resources
+
+### Scrape - Single Page Scraping
+
+```typescript
+// Basic scrape
+const job = await client.scrape.create({
+  url: "https://example.com",
+  detectSignals: true,
+});
+
+// With deal extraction
+const job = await client.scrape.extractDeal("https://shop.example.com/sale");
+
+// With screenshot
+const job = await client.scrape.withScreenshot("https://example.com", {
+  format: "webp",
+  fullPage: true,
+});
+```
+
+**Options:**
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `url` | string | required | URL to scrape |
+| `detectSignals` | boolean | true | Detect prices, discounts, urgency |
+| `extractDeal` | boolean | false | Extract deal information |
+| `extractWithAI` | boolean | false | Use AI for extraction |
+| `useAdvancedModel` | boolean | false | Use GPT-4o (higher cost) |
+| `minDealScore` | number | 0 | Minimum deal score (0-100) |
+| `screenshot` | object | - | Screenshot options |
+| `excludeTags` | string[] | - | HTML tags to exclude |
+| `onlyMainContent` | boolean | true | Extract main content only |
+
+### Crawl - Website Crawling
+
+```typescript
+// Basic crawl
+const job = await client.crawl.create({
+  url: "https://shop.example.com",
+  maxDepth: 3,
+  maxPages: 100,
+  extractDeal: true,
+});
+
+// Using templates
+const job = await client.crawl.withTemplate("ecommerce", {
+  url: "https://shop.example.com",
+});
+
+// Analyze before crawling
+const analysis = await client.crawl.analyze("https://shop.example.com");
+console.log(analysis.recommendedTemplate);
+console.log(analysis.estimatedPages);
+
+// Find deals (convenience method)
+const job = await client.crawl.forDeals("https://shop.example.com", {
+  minDealScore: 70,
+});
+```
+
+**Available Templates:**
+
+- `ecommerce` - Product pages and online stores
+- `marketplace` - Multi-vendor marketplaces
+- `blog` - Blog posts and articles
+- `docs` - Documentation sites
+- `custom` - No preset, use your own settings
+
+### Extract - LLM-Based Extraction
+
+```typescript
+// Schema-based extraction
+const job = await client.extract.withSchema("https://example.com/product", {
+  type: "object",
+  properties: {
+    name: { type: "string" },
+    price: { type: "number" },
+    features: { type: "array", items: { type: "string" } },
+  },
+});
+
+// Prompt-based extraction
+const job = await client.extract.withPrompt(
+  "https://example.com/article",
+  "Extract the article title, author, and main points"
+);
+
+// Pre-built extractors
+const job = await client.extract.product("https://shop.example.com/item");
+const job = await client.extract.article("https://blog.example.com/post");
+const job = await client.extract.contact("https://example.com/contact");
+```
+
+### Dork - Google Dork Searches
+
+```typescript
+// Basic dork search
+const job = await client.dork.create({
+  query: "discount coupon",
+  site: "amazon.com",
+  maxResults: 50,
+});
+
+// Find deals on a site
+const job = await client.dork.findDeals("amazon.com");
+
+// Find products
+const job = await client.dork.findProducts("shop.example.com");
+
+// Find PDFs
+const job = await client.dork.findPDFs("docs.example.com", "user guide");
+
+// Build query string (for preview)
+const query = client.dork.buildQuery({
+  query: "laptop deals",
+  site: "amazon.com",
+  inTitle: "discount",
+});
+// Returns: "laptop deals site:amazon.com intitle:discount"
+```
+
+### Status - Job Management
+
+```typescript
+// Get job status
+const status = await client.status.get(jobId);
+
+// Get deals from a job
+const deals = await client.status.getDeals(jobId, {
+  minScore: 70,
+  limit: 20,
+});
+
+// Resume a failed/paused job
+const resumed = await client.status.resume(jobId);
+
+// Get job metrics
+const metrics = await client.status.getMetrics(jobId);
+
+// Cancel a job
+await client.status.cancel(jobId);
+
+// Convenience methods
+const isComplete = await client.status.isComplete(jobId);
+const succeeded = await client.status.succeeded(jobId);
+const result = await client.status.getResult(jobId);
+```
+
+### Data - Jobs & Deals Access
+
+```typescript
+// List jobs with filtering
+const jobs = await client.data.listJobs({
+  status: "completed",
+  type: "crawl",
+  page: 1,
+  limit: 20,
+  sortBy: "created_at",
+  sortOrder: "desc",
+});
+
+// List deals with filtering
+const deals = await client.data.listDeals({
+  minScore: 70,
+  category: "electronics",
+  sortBy: "deal_score",
+});
+
+// Get top deals
+const topDeals = await client.data.getTopDeals(20, 80);
+
+// Export data
+const jsonExport = await client.data.exportDeals({ format: "json" });
+const csvExport = await client.data.exportDeals({ format: "csv" });
+
+// Get statistics
+const stats = await client.data.getStats();
+```
+
+### Webhooks - Event Notifications
+
+```typescript
+// Create a webhook
+const webhook = await client.webhooks.create({
+  event: "deal.found",
+  url: "https://my-server.com/webhooks/deals",
+  secret: "my-webhook-secret",
+  minDealScore: 70,
+});
+
+// List webhooks
+const webhooks = await client.webhooks.list();
+
+// Test a webhook
+const result = await client.webhooks.test(webhookId);
+
+// Enable/disable
+await client.webhooks.enable(webhookId);
+await client.webhooks.disable(webhookId);
+
+// Delete
+await client.webhooks.delete(webhookId);
+```
+
+**Events:**
+
+- `deal.found` - New deal discovered
+- `deal.synced` - Deal synced to DealUp
+- `crawl.completed` - Crawl job finished
+- `crawl.failed` - Crawl job failed
+
+### Keys - API Key Management
+
+```typescript
+// List all keys
+const keys = await client.keys.list();
+
+// Create a new key
+const newKey = await client.keys.create({
+  name: "Production Key",
+  scopes: ["scrape:write", "crawl:write", "data:read"],
+  expiresInDays: 365,
+});
+// ⚠️ Save newKey.key immediately - it won't be shown again!
+
+// Rotate a key
+const rotated = await client.keys.rotate(keyId, {
+  newName: "Production Key v2",
+});
+
+// Revoke a key
+await client.keys.revoke(keyId);
+
+// Get key stats
+const stats = await client.keys.getStats(keyId, { days: 30 });
+```
+
+### Account - Profile & Preferences
+
+```typescript
+// Get account info
+const account = await client.account.get();
+console.log(account.tier); // "free" | "pro" | "enterprise"
+console.log(account.usage);
+
+// Get metrics
+const metrics = await client.account.getMetrics();
+
+// Get recommendations
+const recommendations = await client.account.getRecommendations();
+
+// Update preferences
+await client.account.updatePreferences({
+  minDealScore: 70,
+  autoSync: true,
+  preferredCategories: ["software", "courses"],
+});
+
+// Convenience methods
+const remaining = await client.account.getRemainingQuota("scrapes");
+const hasQuota = await client.account.hasQuota("crawls", 5);
+const isPremium = await client.account.isPremium();
+```
+
+## Polling & Waiting
+
+```typescript
+// Wait for a single job
+const result = await client.waitForResult(jobId, {
+  pollInterval: 2000, // Check every 2 seconds
+  timeout: 300000, // 5 minute timeout
+  onProgress: (status) => console.log(`Progress: ${status.progress}%`),
+  onStatusChange: (newStatus, oldStatus) => {
+    console.log(`Status changed: ${oldStatus} → ${newStatus}`);
+  },
+});
+
+// Wait for multiple jobs
+const results = await client.waitForAll([jobId1, jobId2, jobId3]);
+
+// Wait for any job to complete
+const firstResult = await client.waitForAny([jobId1, jobId2, jobId3]);
+
+// Convenience: scrape and wait
+const result = await client.scrapeAndWait({
+  url: "https://example.com",
+  extractDeal: true,
+});
+
+// Convenience: crawl and wait
+const result = await client.crawlAndWait({
+  url: "https://shop.example.com",
+  maxPages: 50,
+});
+```
+
+## Error Handling
+
+```typescript
+import { DealCrawl, DealCrawlError, ERROR_CODES } from "@dealcrawl/sdk";
+
+try {
+  const result = await client.scrape.create({ url: "..." });
+} catch (error) {
+  if (error instanceof DealCrawlError) {
+    console.log(error.code); // e.g., "RATE_LIMIT_EXCEEDED"
+    console.log(error.statusCode); // HTTP status code
+    console.log(error.message); // Human-readable message
+    console.log(error.details); // Additional details
+
+    // Check error type
+    if (error.isRateLimited()) {
+      console.log(`Retry after ${error.retryAfter} seconds`);
+    }
+    if (error.isAuthError()) {
+      console.log("Check your API key");
+    }
+    if (error.isRetryable()) {
+      // Automatic retry was attempted
+    }
+  }
+}
+```
+
+**Error Codes:**
+
+- `INVALID_API_KEY` - API key is invalid or missing
+- `RATE_LIMIT_EXCEEDED` - Too many requests
+- `QUOTA_EXCEEDED` - Monthly quota exceeded
+- `JOB_NOT_FOUND` - Job ID doesn't exist
+- `JOB_TIMEOUT` - Job didn't complete in time
+- `FETCH_FAILED` - Network request failed
+
+## TypeScript Types
+
+All types are exported for your convenience:
+
+```typescript
+import type {
+  // Configuration
+  DealCrawlConfig,
+
+  // Request Options
+  ScrapeOptions,
+  CrawlOptions,
+  ExtractOptions,
+  DorkOptions,
+
+  // Responses
+  JobStatusResponse,
+  ListDealsResponse,
+  DealItem,
+
+  // Re-exports from @dealcrawl/shared
+  ScrapeResult,
+  CrawlResult,
+  ExtractedDeal,
+  Signal,
+} from "@dealcrawl/sdk";
+```
+
+## Examples
+
+### Find Best Deals on a Site
+
+```typescript
+async function findBestDeals(siteUrl: string) {
+  const client = new DealCrawl({ apiKey: process.env.DEALCRAWL_API_KEY! });
+
+  // Crawl the site for deals
+  const job = await client.crawl.forDeals(siteUrl, {
+    maxPages: 200,
+    minDealScore: 60,
+  });
+
+  // Wait for completion
+  const result = await client.waitForResult(job.jobId, {
+    timeout: 600000, // 10 minutes
+    onProgress: (s) => console.log(`Crawled: ${s.progress}%`),
+  });
+
+  if (result.status === "failed") {
+    throw new Error(result.error);
+  }
+
+  // Get the best deals
+  const deals = await client.status.getDeals(job.jobId, {
+    minScore: 80,
+    limit: 50,
+  });
+
+  return deals.deals.sort((a, b) => b.dealScore - a.dealScore);
+}
+```
+
+### Extract Product Information
+
+```typescript
+async function extractProduct(productUrl: string) {
+  const client = new DealCrawl({ apiKey: process.env.DEALCRAWL_API_KEY! });
+
+  const result = await client.extractAndWait({
+    url: productUrl,
+    schema: {
+      type: "object",
+      properties: {
+        name: { type: "string" },
+        price: { type: "number" },
+        originalPrice: { type: "number" },
+        discount: { type: "string" },
+        rating: { type: "number" },
+        reviews: { type: "number" },
+        availability: { type: "string" },
+        features: { type: "array", items: { type: "string" } },
+      },
+      required: ["name", "price"],
+    },
+    model: "gpt-4o-mini",
+  });
+
+  return result.result;
+}
+```
+
+### Monitor for New Deals
+
+```typescript
+async function setupDealMonitoring(webhookUrl: string) {
+  const client = new DealCrawl({ apiKey: process.env.DEALCRAWL_API_KEY! });
+
+  // Create webhook for high-score deals
+  await client.webhooks.create({
+    event: "deal.found",
+    url: webhookUrl,
+    minDealScore: 85,
+    categories: ["software", "courses"],
+  });
+
+  // Set up preferences
+  await client.account.updatePreferences({
+    minDealScore: 70,
+    webhookEnabled: true,
+  });
+
+  console.log("Deal monitoring configured!");
+}
+```
+
+## Browser Usage
+
+The SDK works in browsers with native `fetch` support:
+
+```typescript
+// In a browser environment
+import { DealCrawl } from "@dealcrawl/sdk";
+
+const client = new DealCrawl({
+  apiKey: "your-api-key", // ⚠️ Don't expose keys in client-side code!
+});
+```
+
+> **Warning:** Never expose your API key in client-side code. Use a backend proxy or edge function.
+
+## Compatibility
+
+- **Node.js**: 18.0+
+- **Bun**: All versions
+- **Browser**: Modern browsers with `fetch` support
+- **TypeScript**: 5.0+
+
+## License
+
+MIT © [DealUp](https://dealup.cc)