@hydrafetch/node-sdk 0.3.0 → 0.5.0

package/README.md

@@ -25,7 +25,7 @@ const page = await hf.scrape('https://example.com/article');
 console.log(page.markdown);
 ```
 
-Create a key at [app.hydrafetch.com](https://app.hydrafetch.com). The constructor reads `HYDRAFETCH_API_KEY` when no key is passed.
+Copy your key from [app.hydrafetch.com](https://app.hydrafetch.com). The constructor reads `HYDRAFETCH_API_KEY` when no key is passed.
 
 ## Scraping
 
@@ -82,6 +82,16 @@ out.results.forEach((item) => console.log(item.url, item.data?.name));
 
 Pass `prompt` instead of, or alongside, `schema` to describe the fields in plain language.
 
+Sites that price or stock by delivery area serve a different page for each area. Pass `location` to fetch as if from a country and language, and `headers` to send the cookie that picks the store or area, the same options `scrape` takes.
+
+```ts
+const out = await hf.extract(['https://shop.example/search?q=ghee'], {
+  prompt: 'every product with its size and price',
+  location: { country: 'in' },
+  headers: { Cookie: 'store_id=42' },
+});
+```
+
 ### Zod schemas
 
 If you already use Zod, pass the schema directly. Zod stays optional and is imported only when you
@@ -167,6 +177,24 @@ await hf.styleguide('https://stripe.com/pricing', { maxAge: 0 }); // one page, r
 
 For logos in a browser use [`@hydrafetch/client-sdk`](https://github.com/Hydrafetch/client-sdk) or [`@hydrafetch/react`](https://github.com/Hydrafetch/react-sdk) with a publishable key. Those bill against logo pulls rather than credits.
 
+## Rating results
+
+Tell us when a result was wrong, so it gets looked at. It is free, and rating the same request again replaces the earlier rating.
+
+```ts
+const page = await hf.scrape('https://stripe.com/pricing');
+await hf.rateResult({
+  requestId: hf.lastRequestId,
+  rating: 'bad',
+  reasons: ['missing_content'],
+  note: 'The enterprise tier is missing',
+});
+
+await hf.rateResult({ url: 'https://stripe.com/pricing', rating: 'good' }); // by URL, last 24 hours
+```
+
+Reasons are `missing_content`, `wrong_content`, `blocked`, `slow`, `formatting` and `other`.
+
 ## Error handling
 
 All failures throw `HydrafetchError`, carrying the API's error code, HTTP status and request id.
@@ -235,6 +263,7 @@ const hf = new Hydrafetch({
 | `startCrawl(url, options?)` | `string` | 1 per page |
 | `startBatch(urls, options?)` | `string` | 1 per page |
 | `crawlStatus(id)`, `batchStatus(id)` | `JobResult` | free |
+| `rateResult(input)` | `RateResultResponse` | free |
 
 Failed requests are not billed. Pricing does not vary with page difficulty, so there is no render, stealth or proxy option to set.
 

package/dist/index.cjs

@@ -129,6 +129,10 @@ var Hydrafetch = class {
   timeoutMs;
   maxRetries;
   fetchImpl;
+  latestRequestId;
+  get lastRequestId() {
+    return this.latestRequestId;
+  }
   constructor(options = {}) {
     const opts = typeof options === "string" ? { apiKey: options } : options;
     const env = globalThis.process?.env;
@@ -136,7 +140,7 @@ var Hydrafetch = class {
     if (!apiKey) {
       throw new HydrafetchError({
         code: "MISSING_API_KEY",
-        message: "No API key. Pass one to the constructor or set HYDRAFETCH_API_KEY. Create a key at https://app.hydrafetch.com.",
+        message: "No API key. Pass one to the constructor or set HYDRAFETCH_API_KEY. Copy your key from https://app.hydrafetch.com.",
         status: 401
       });
     }
@@ -174,6 +178,7 @@ var Hydrafetch = class {
           body: body === void 0 ? void 0 : JSON.stringify(body),
           signal: controller.signal
         });
+        this.latestRequestId = res.headers.get("x-request-id") ?? this.latestRequestId;
         const text = await res.text();
         let parsed;
         try {
@@ -221,6 +226,9 @@ var Hydrafetch = class {
     const page = await this.scrape(url, { ...options, formats: ["markdown"] });
     return page.markdown;
   }
+  rateResult(input) {
+    return this.request("POST", "/v1/web/feedback", input);
+  }
   map(url, options = {}) {
     return this.request("POST", "/v1/web/map", { url, ...options });
   }

package/dist/index.d.cts

@@ -34,6 +34,10 @@ interface ScrapeBase {
 type ScrapeResult<F extends Format = 'markdown'> = ScrapeBase & {
     [K in F]: ScrapeContent[K];
 } & Partial<Omit<ScrapeContent, F>>;
+interface Location {
+    country?: string;
+    languages?: string[];
+}
 interface ScrapeOptions<F extends readonly Format[] = readonly Format[]> {
     formats?: F;
     maxAge?: number;
@@ -48,6 +52,7 @@ interface ScrapeOptions<F extends readonly Format[] = readonly Format[]> {
     includeLinks?: boolean;
     waitFor?: number;
     timeout?: number;
+    location?: Location;
     headers?: Record<string, string>;
     jsonOptions?: Record<string, unknown>;
 }
@@ -114,6 +119,8 @@ interface ExtractOptions {
     showConfidence?: boolean;
     mergeEntities?: boolean;
     maxAge?: number;
+    location?: Location;
+    headers?: Record<string, string>;
 }
 interface ExtractItem<T = unknown> {
     url: string;
@@ -231,6 +238,20 @@ interface WaitOptions {
     pollIntervalMs?: number;
     onProgress?: (job: JobResult) => void;
 }
+type ResultRating = 'good' | 'bad';
+type ResultFeedbackReason = 'missing_content' | 'wrong_content' | 'blocked' | 'slow' | 'formatting' | 'other';
+interface RateResultInput {
+    requestId?: string;
+    url?: string;
+    rating: ResultRating;
+    reasons?: ResultFeedbackReason[];
+    note?: string;
+}
+interface RateResultResponse {
+    id: string;
+    requestId: string;
+    rating: ResultRating;
+}
 
 interface HydrafetchOptions {
     apiKey?: string;
@@ -245,10 +266,13 @@ declare class Hydrafetch {
     private readonly timeoutMs;
     private readonly maxRetries;
     private readonly fetchImpl;
+    private latestRequestId;
+    get lastRequestId(): string | undefined;
     constructor(options?: HydrafetchOptions | string);
     private request;
     scrape<const F extends readonly Format[] = ['markdown']>(url: string, options?: ScrapeOptions<F>): Promise<ScrapeResult<F[number]>>;
     markdown(url: string, options?: Omit<ScrapeOptions, 'formats'>): Promise<string>;
+    rateResult(input: RateResultInput): Promise<RateResultResponse>;
     map(url: string, options?: MapOptions): Promise<MapResult>;
     search(query: string, options?: SearchOptions): Promise<SearchResult>;
     extract<T = unknown>(urls: string | string[], options?: ExtractOptions): Promise<ExtractResult<T>>;
@@ -290,4 +314,4 @@ declare class HydrafetchTimeoutError extends HydrafetchError {
     constructor(message: string);
 }
 
-export { type BatchOptions, type BrandOptions, type BrandResult, type CrawlOptions, type ExtractItem, type ExtractOptions, type ExtractResult, type ExtractSchema, type Format, Hydrafetch, HydrafetchError, type HydrafetchOptions, HydrafetchTimeoutError, type ImagesResult, type JobPage, type JobResult, type JobStatus, type LogoResult, type MapOptions, type MapResult, type PageImage, type PageMetadata, type ScrapeBase, type ScrapeContent, type ScrapeOptions, type ScrapeResult, type ScreenshotOptions, type ScreenshotResult, type SearchOptions, type SearchResult, type SearchResultItem, type StyleguideOptions, type StyleguideResult, type Usage, type WaitOptions, type ZodLike };
+export { type BatchOptions, type BrandOptions, type BrandResult, type CrawlOptions, type ExtractItem, type ExtractOptions, type ExtractResult, type ExtractSchema, type Format, Hydrafetch, HydrafetchError, type HydrafetchOptions, HydrafetchTimeoutError, type ImagesResult, type JobPage, type JobResult, type JobStatus, type Location, type LogoResult, type MapOptions, type MapResult, type PageImage, type PageMetadata, type RateResultInput, type RateResultResponse, type ResultFeedbackReason, type ResultRating, type ScrapeBase, type ScrapeContent, type ScrapeOptions, type ScrapeResult, type ScreenshotOptions, type ScreenshotResult, type SearchOptions, type SearchResult, type SearchResultItem, type StyleguideOptions, type StyleguideResult, type Usage, type WaitOptions, type ZodLike };

package/dist/index.d.ts

@@ -34,6 +34,10 @@ interface ScrapeBase {
 type ScrapeResult<F extends Format = 'markdown'> = ScrapeBase & {
     [K in F]: ScrapeContent[K];
 } & Partial<Omit<ScrapeContent, F>>;
+interface Location {
+    country?: string;
+    languages?: string[];
+}
 interface ScrapeOptions<F extends readonly Format[] = readonly Format[]> {
     formats?: F;
     maxAge?: number;
@@ -48,6 +52,7 @@ interface ScrapeOptions<F extends readonly Format[] = readonly Format[]> {
     includeLinks?: boolean;
     waitFor?: number;
     timeout?: number;
+    location?: Location;
     headers?: Record<string, string>;
     jsonOptions?: Record<string, unknown>;
 }
@@ -114,6 +119,8 @@ interface ExtractOptions {
     showConfidence?: boolean;
     mergeEntities?: boolean;
     maxAge?: number;
+    location?: Location;
+    headers?: Record<string, string>;
 }
 interface ExtractItem<T = unknown> {
     url: string;
@@ -231,6 +238,20 @@ interface WaitOptions {
     pollIntervalMs?: number;
     onProgress?: (job: JobResult) => void;
 }
+type ResultRating = 'good' | 'bad';
+type ResultFeedbackReason = 'missing_content' | 'wrong_content' | 'blocked' | 'slow' | 'formatting' | 'other';
+interface RateResultInput {
+    requestId?: string;
+    url?: string;
+    rating: ResultRating;
+    reasons?: ResultFeedbackReason[];
+    note?: string;
+}
+interface RateResultResponse {
+    id: string;
+    requestId: string;
+    rating: ResultRating;
+}
 
 interface HydrafetchOptions {
     apiKey?: string;
@@ -245,10 +266,13 @@ declare class Hydrafetch {
     private readonly timeoutMs;
     private readonly maxRetries;
     private readonly fetchImpl;
+    private latestRequestId;
+    get lastRequestId(): string | undefined;
     constructor(options?: HydrafetchOptions | string);
     private request;
     scrape<const F extends readonly Format[] = ['markdown']>(url: string, options?: ScrapeOptions<F>): Promise<ScrapeResult<F[number]>>;
     markdown(url: string, options?: Omit<ScrapeOptions, 'formats'>): Promise<string>;
+    rateResult(input: RateResultInput): Promise<RateResultResponse>;
     map(url: string, options?: MapOptions): Promise<MapResult>;
     search(query: string, options?: SearchOptions): Promise<SearchResult>;
     extract<T = unknown>(urls: string | string[], options?: ExtractOptions): Promise<ExtractResult<T>>;
@@ -290,4 +314,4 @@ declare class HydrafetchTimeoutError extends HydrafetchError {
     constructor(message: string);
 }
 
-export { type BatchOptions, type BrandOptions, type BrandResult, type CrawlOptions, type ExtractItem, type ExtractOptions, type ExtractResult, type ExtractSchema, type Format, Hydrafetch, HydrafetchError, type HydrafetchOptions, HydrafetchTimeoutError, type ImagesResult, type JobPage, type JobResult, type JobStatus, type LogoResult, type MapOptions, type MapResult, type PageImage, type PageMetadata, type ScrapeBase, type ScrapeContent, type ScrapeOptions, type ScrapeResult, type ScreenshotOptions, type ScreenshotResult, type SearchOptions, type SearchResult, type SearchResultItem, type StyleguideOptions, type StyleguideResult, type Usage, type WaitOptions, type ZodLike };
+export { type BatchOptions, type BrandOptions, type BrandResult, type CrawlOptions, type ExtractItem, type ExtractOptions, type ExtractResult, type ExtractSchema, type Format, Hydrafetch, HydrafetchError, type HydrafetchOptions, HydrafetchTimeoutError, type ImagesResult, type JobPage, type JobResult, type JobStatus, type Location, type LogoResult, type MapOptions, type MapResult, type PageImage, type PageMetadata, type RateResultInput, type RateResultResponse, type ResultFeedbackReason, type ResultRating, type ScrapeBase, type ScrapeContent, type ScrapeOptions, type ScrapeResult, type ScreenshotOptions, type ScreenshotResult, type SearchOptions, type SearchResult, type SearchResultItem, type StyleguideOptions, type StyleguideResult, type Usage, type WaitOptions, type ZodLike };

package/dist/index.js

@@ -101,6 +101,10 @@ var Hydrafetch = class {
   timeoutMs;
   maxRetries;
   fetchImpl;
+  latestRequestId;
+  get lastRequestId() {
+    return this.latestRequestId;
+  }
   constructor(options = {}) {
     const opts = typeof options === "string" ? { apiKey: options } : options;
     const env = globalThis.process?.env;
@@ -108,7 +112,7 @@ var Hydrafetch = class {
     if (!apiKey) {
       throw new HydrafetchError({
         code: "MISSING_API_KEY",
-        message: "No API key. Pass one to the constructor or set HYDRAFETCH_API_KEY. Create a key at https://app.hydrafetch.com.",
+        message: "No API key. Pass one to the constructor or set HYDRAFETCH_API_KEY. Copy your key from https://app.hydrafetch.com.",
         status: 401
       });
     }
@@ -146,6 +150,7 @@ var Hydrafetch = class {
           body: body === void 0 ? void 0 : JSON.stringify(body),
           signal: controller.signal
         });
+        this.latestRequestId = res.headers.get("x-request-id") ?? this.latestRequestId;
         const text = await res.text();
         let parsed;
         try {
@@ -193,6 +198,9 @@ var Hydrafetch = class {
     const page = await this.scrape(url, { ...options, formats: ["markdown"] });
     return page.markdown;
   }
+  rateResult(input) {
+    return this.request("POST", "/v1/web/feedback", input);
+  }
   map(url, options = {}) {
     return this.request("POST", "/v1/web/map", { url, ...options });
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hydrafetch/node-sdk",
-  "version": "0.3.0",
+  "version": "0.5.0",
   "description": "Official Node/TypeScript client for the Hydrafetch web data API. Turn any URL into clean Markdown and structured data.",
   "keywords": [
     "hydrafetch",