@dealcrawl/sdk 2.12.0 → 2.14.0

package/README.md

@@ -6,7 +6,83 @@ Official TypeScript SDK for the DealCrawl web scraping and crawling API.
 [![TypeScript](https://img.shields.io/badge/TypeScript-5.0+-blue.svg)](https://www.typescriptlang.org/)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 
-## What's New in v2.12.0 (January 2026) ✨
+## What's New in v2.13.0 (January 2026) ✨
+
+### New Resources
+
+**Postprocessors Resource (`client.postprocessors.*`)**
+
+Discover and check site-specific enrichment postprocessors (Amazon, Fnac, eBay):
+
+```typescript
+// List all available postprocessors
+const list = await client.postprocessors.list();
+console.log(list.postprocessors);
+// [{ name: "amazon", domains: [...], extractedFields: [...] }, ...]
+
+// Check if a URL will be enriched
+const check = await client.postprocessors.check({
+  url: "https://www.amazon.fr/dp/B09V3KXJPB"
+});
+console.log(check.hasPostprocessor); // true
+console.log(check.postprocessor);    // "amazon"
+```
+
+**Usage Resource (`client.usage.*`)**
+
+Monitor usage, quotas, and LLM token consumption:
+
+```typescript
+// Get current usage and quotas
+const usage = await client.usage.current();
+console.log(usage.usage.scrapes);     // 150
+console.log(usage.quotas.scrapes);    // 10000
+console.log(usage.percentUsed.scrapes); // 1.5
+
+// Get LLM token usage
+const tokens = await client.usage.tokens({ days: 30 });
+console.log(tokens.totals.estimatedCostUsd); // 12.34
+
+// Get daily token breakdown
+const daily = await client.usage.dailyTokens({ days: 7 });
+```
+
+### New Agent Vision Capabilities
+
+Enable multi-modal vision for better page understanding:
+
+```typescript
+const job = await client.agent.create({
+  url: "https://complex-site.com",
+  prompt: "Navigate and extract data",
+  enableVision: true,  // NEW: Enable vision-based navigation
+  visionOptions: {     // NEW: Configure vision behavior
+    quality: 80,                  // JPEG quality (1-100)
+    annotateElements: true,       // Label clickable elements
+    captureFrequency: "on_stuck"  // When to capture: every_step | on_demand | on_stuck
+  }
+});
+```
+
+### New Scrape Options
+
+Control scraping behavior with new options:
+
+```typescript
+const job = await client.scrape.create({
+  url: "https://example.com",
+  // NEW: Control postprocessor execution
+  runPostprocessors: true,  // Enable/disable site enrichment (default: true)
+  // NEW: Force browser-based scraping (uses 'render' quota)
+  forceDynamic: false,
+  // NEW: Force stealth mode for anti-bot sites
+  forceStealth: false,
+});
+```
+
+---
+
+## What's New in v2.12.0 (January 2026)
 
 ### New Methods - 100% API Coverage
 
@@ -362,6 +438,222 @@ const deals = await client.status.getDeals(job.jobId, {
 console.log(deals.deals); // Array of ExtractedDeal objects
 ```
 
+### 🏪 Site-Specific Data (Postprocessors)
+
+Automatic enrichment for major e-commerce sites. When scraping Amazon, Fnac, or eBay, you get structured site-specific data:
+
+```typescript
+// Scrape an Amazon product
+const result = await client.scrapeAndWait({
+  url: "https://www.amazon.fr/dp/B0EXAMPLE",
+  extractDeal: true,
+});
+
+// Site-specific data automatically extracted
+console.log(result.data.siteData);
+// {
+//   site: "amazon",
+//   productId: "B0EXAMPLE",
+//   seller: {
+//     name: "Official Store",
+//     rating: 4.8,
+//     reviewCount: 12543,
+//     isPrime: true
+//   },
+//   shipping: {
+//     free: true,
+//     prime: true,
+//     estimatedDelivery: "Tomorrow"
+//   },
+//   availability: {
+//     inStock: true,
+//     stockLevel: "in_stock",
+//     quantity: 50
+//   },
+//   condition: "new",
+//   buyBox: {
+//     seller: "Official Store",
+//     price: 29.99,
+//     isPrime: true
+//   },
+//   coupon: {
+//     discount: "10%",
+//     autoApplied: true
+//   },
+//   subscribeAndSave: {
+//     available: true,
+//     discountPercent: 15,
+//     price: 25.49
+//   },
+//   categories: ["Electronics", "Accessories", "Headphones"]
+// }
+
+// Check which postprocessors ran
+console.log(result.data.postprocessorsUsed);
+// ["amazon"]
+```
+
+**Amazon-Specific Fields:**
+
+```typescript
+// Amazon product with all enrichments
+const result = await client.scrapeAndWait({
+  url: "https://www.amazon.com/dp/B0EXAMPLE",
+});
+
+if (result.data.siteData?.site === "amazon") {
+  const { siteData } = result.data;
+
+  // Prime eligibility
+  console.log("Prime:", siteData.shipping?.prime);
+  console.log("Free shipping:", siteData.shipping?.free);
+
+  // Buy Box winner
+  if (siteData.buyBox) {
+    console.log(`Best price: $${siteData.buyBox.price} from ${siteData.buyBox.seller}`);
+  }
+
+  // Subscribe & Save
+  if (siteData.subscribeAndSave?.available) {
+    console.log(`S&S: ${siteData.subscribeAndSave.discountPercent}% off`);
+  }
+
+  // Lightning deals
+  if (siteData.flashDeal?.active) {
+    console.log(`Flash deal ends: ${siteData.flashDeal.endsAt}`);
+    console.log(`${siteData.flashDeal.percentClaimed}% claimed`);
+  }
+
+  // Coupons
+  if (siteData.coupon) {
+    console.log(`Coupon: ${siteData.coupon.discount}`);
+  }
+}
+```
+
+**Fnac-Specific Fields:**
+
+```typescript
+// Fnac product
+const result = await client.scrapeAndWait({
+  url: "https://www.fnac.com/product/12345",
+});
+
+if (result.data.siteData?.site === "fnac") {
+  const { siteData } = result.data;
+
+  // Fnac Pro sellers
+  console.log("Pro Seller:", siteData.seller?.isProSeller);
+
+  // Express delivery
+  console.log("Express available:", siteData.shipping?.expressAvailable);
+
+  // Product condition
+  console.log("Condition:", siteData.condition);
+  // "new" | "refurbished" | "used" | "like_new" | "acceptable"
+
+  // Product variations
+  if (siteData.variations) {
+    siteData.variations.forEach(v => {
+      console.log(`${v.type}: ${v.options.join(", ")}`);
+    });
+  }
+}
+```
+
+**eBay-Specific Fields:**
+
+```typescript
+// eBay listing
+const result = await client.scrapeAndWait({
+  url: "https://www.ebay.com/itm/123456789",
+});
+
+if (result.data.siteData?.site === "ebay") {
+  const { siteData } = result.data;
+
+  // Top-rated seller badge
+  console.log("Top Rated:", siteData.seller?.topRated);
+
+  // Item condition
+  console.log("Condition:", siteData.condition);
+
+  // Stock level
+  if (siteData.availability) {
+    console.log("In stock:", siteData.availability.inStock);
+    console.log("Quantity:", siteData.availability.quantity);
+  }
+}
+```
+
+**Supported Domains:**
+
+| Site   | Domains                                                                                   |
+|--------|-------------------------------------------------------------------------------------------|
+| Amazon | amazon.com, amazon.fr, amazon.de, amazon.co.uk, amazon.es, amazon.it, amazon.ca, +12 more |
+| Fnac   | fnac.com, fnac.be, fnac.ch, fnac.pt, fnac.es                                              |
+| eBay   | ebay.com, ebay.fr, ebay.de, ebay.co.uk, ebay.es, ebay.it, ebay.ca, ebay.com.au, +6 more   |
+
+**SiteSpecificData Type:**
+
+```typescript
+import type { SiteSpecificData } from "@dealcrawl/sdk";
+
+interface SiteSpecificData {
+  site: string;                    // "amazon" | "fnac" | "ebay"
+  productId?: string;              // ASIN, SKU, or item ID
+  seller?: {
+    name: string;
+    url?: string;
+    rating?: number;
+    reviewCount?: number;
+    isPrime?: boolean;             // Amazon
+    isProSeller?: boolean;         // Fnac
+    topRated?: boolean;            // eBay
+  };
+  shipping?: {
+    free: boolean;
+    estimatedDelivery?: string;
+    prime?: boolean;               // Amazon Prime
+    expressAvailable?: boolean;    // Fnac Express
+  };
+  availability?: {
+    inStock: boolean;
+    stockLevel?: "in_stock" | "low_stock" | "out_of_stock" | "preorder";
+    quantity?: number;
+    message?: string;
+  };
+  condition?: "new" | "refurbished" | "used" | "like_new" | "acceptable";
+  buyBox?: {                       // Amazon only
+    seller: string;
+    price: number;
+    isPrime: boolean;
+  };
+  coupon?: {
+    code?: string;
+    discount: string;
+    autoApplied: boolean;
+  };
+  flashDeal?: {                    // Lightning deals
+    active: boolean;
+    endsAt?: string;
+    percentClaimed?: number;
+  };
+  subscribeAndSave?: {             // Amazon only
+    available: boolean;
+    discountPercent?: number;
+    price?: number;
+  };
+  categories?: string[];
+  variations?: Array<{
+    type: string;
+    options: string[];
+    selected?: string;
+  }>;
+  rawData?: Record<string, unknown>;
+}
+```
+
 ### 📝 Markdown Output
 
 Convert HTML to clean, structured markdown:
@@ -471,6 +763,9 @@ const job = await client.scrape.withScreenshot("https://example.com", {
 | `outputMarkdown`       | boolean  | false    | Convert content to Markdown (GFM)                         |
 | `markdownBaseUrl`      | string   | -        | Base URL for resolving relative URLs in markdown          |
 | `actions`              | array    | -        | Browser actions to execute before scraping                |
+| `runPostprocessors`    | boolean  | true     | Run site-specific postprocessors (Amazon, Fnac, eBay)     |
+| `forceDynamic`         | boolean  | false    | Force browser-based scraping (uses 'render' quota)        |
+| `forceStealth`         | boolean  | false    | Force stealth mode for anti-bot sites                     |
 
 ### Batch Scrape - Bulk URL Scraping
 
@@ -788,6 +1083,16 @@ const job = await client.agent.withClaude(
 | `timeout`         | number  | 30000    | Per-step timeout in ms (max: 60000)           |
 | `takeScreenshots` | boolean | false    | Capture screenshot at each step               |
 | `onlyMainContent` | boolean | true     | Extract main content only                     |
+| `enableVision`    | boolean | tier     | Enable vision-based navigation (screenshots for LLM) |
+| `visionOptions`   | object  | -        | Vision capture configuration (see below)      |
+
+**Vision Options:**
+
+| Option              | Type   | Default | Description                                      |
+| ------------------- | ------ | ------- | ------------------------------------------------ |
+| `quality`           | number | tier    | JPEG quality for screenshots (1-100)             |
+| `annotateElements`  | boolean| false   | Label clickable elements on screenshots          |
+| `captureFrequency`  | string | tier    | When to capture: `every_step`, `on_demand`, `on_stuck` |
 
 **Action Types:**
 
@@ -1039,6 +1344,90 @@ await client.keys.create({
 });
 ```
 
+### Postprocessors - Site Enrichment Discovery
+
+Discover which postprocessors are available and check URL applicability:
+
+```typescript
+// List all available postprocessors
+const list = await client.postprocessors.list();
+console.log(list.postprocessors);
+// [
+//   {
+//     name: "amazon",
+//     domains: ["amazon.com", "amazon.fr", "amazon.de", ...],
+//     extractedFields: ["productId", "seller", "shipping", "availability", ...],
+//     description: "Amazon product page enrichment..."
+//   },
+//   ...
+// ]
+console.log(list.totalDomains); // 30+ supported domains
+
+// Check if a specific URL will be enriched
+const amazonCheck = await client.postprocessors.check({
+  url: "https://www.amazon.fr/dp/B09V3KXJPB"
+});
+console.log(amazonCheck.hasPostprocessor); // true
+console.log(amazonCheck.postprocessor);    // "amazon"
+console.log(amazonCheck.extractedFields);  // ["productId", "seller", ...]
+
+// Check unsupported URL
+const genericCheck = await client.postprocessors.check({
+  url: "https://example.com/product"
+});
+console.log(genericCheck.hasPostprocessor); // false
+console.log(genericCheck.message);          // "No postprocessor available..."
+```
+
+**Disable postprocessors for faster scraping:**
+
+```typescript
+const job = await client.scrape.create({
+  url: "https://amazon.com/dp/B123",
+  runPostprocessors: false, // Skip enrichment for speed
+});
+```
+
+### Usage - Quota & Token Monitoring
+
+Monitor your usage, quotas, and LLM token consumption:
+
+```typescript
+// Get current usage and quotas
+const usage = await client.usage.current();
+console.log(usage.tier);                    // "pro"
+console.log(usage.usage.scrapes);           // 150
+console.log(usage.quotas.scrapes);          // 10000
+console.log(usage.percentUsed.scrapes);     // 1.5
+console.log(usage.billingPeriod.daysRemaining); // 15
+
+// Get historical usage (for billing analysis)
+const history = await client.usage.history({ months: 6 });
+history.data.forEach(period => {
+  console.log(`${period.periodStart}: ${period.usage.scrapes} scrapes`);
+});
+
+// Get LLM token usage summary
+const tokens = await client.usage.tokens({ days: 30 });
+console.log(tokens.totals.totalTokens);      // 1234567
+console.log(tokens.totals.estimatedCostUsd); // 12.34
+tokens.breakdown.forEach(item => {
+  console.log(`${item.provider}/${item.model}: ${item.totalTokens} tokens`);
+});
+
+// Filter by provider
+const openaiTokens = await client.usage.tokens({
+  days: 30,
+  provider: "openai"
+});
+
+// Get daily token breakdown (for charts)
+const daily = await client.usage.dailyTokens({ days: 7 });
+daily.daily.forEach(day => {
+  console.log(`${day.date}: ${day.totalTokens} tokens ($${day.estimatedCostUsd})`);
+});
+```
+
 ### Account - Profile & Preferences
 
 ```typescript
@@ -1208,6 +1597,8 @@ import type {
   ExtractOptions,
   DorkOptions,
   AgentOptions,
+  VisionOptions,           // NEW: Vision configuration
+  VisionCaptureFrequency,  // NEW: "every_step" | "on_demand" | "on_stuck"
   SchemaGenerationOptions,
 
   // Responses
@@ -1217,10 +1608,25 @@ import type {
   AgentJobResponse,
   AgentStatusResponse,
   AgentResultResponse,
+  AgentMemoryStats,        // NEW: Memory stats from agent sessions
   SchemaGenerationResponse,
   SearchJobResponse,
   BatchScrapeResponse,
 
+  // Postprocessor Types (NEW)
+  PostprocessorInfo,
+  PostprocessorsListResponse,
+  PostprocessorCheckResponse,
+
+  // Usage Types (NEW)
+  CurrentUsageResponse,
+  UsageHistoryResponse,
+  TokenUsageResponse,
+  DailyTokenUsageResponse,
+  BillingPeriod,
+  QuotaLimits,
+  UsagePercentages,
+
   // Action Types
   ActionInput,
   ClickAction,
@@ -1244,6 +1650,7 @@ import type {
   ExtractedDeal,
   Signal,
   ParsedPage,              // Includes markdown field
+  SiteSpecificData,        // Postprocessor enrichment data
 } from "@dealcrawl/sdk";
 ```