@hydrafetch/node-sdk 0.1.0

package/dist/index.js

@@ -0,0 +1,269 @@
+// src/errors.ts
+var HydrafetchError = class extends Error {
+  code;
+  status;
+  requestId;
+  details;
+  constructor(args) {
+    super(args.message);
+    this.name = "HydrafetchError";
+    this.code = args.code;
+    this.status = args.status;
+    this.requestId = args.requestId;
+    this.details = args.details;
+  }
+  /** 401 and 403. Retrying without changing the key is pointless. */
+  get isAuth() {
+    return this.status === 401 || this.status === 403;
+  }
+  /** 402. The workspace is out of credits. */
+  get isOutOfCredits() {
+    return this.status === 402;
+  }
+  /** 400 and 422. The request itself is wrong; a repeat costs another call and fails again. */
+  get isInvalidRequest() {
+    return this.status === 400 || this.status === 422;
+  }
+  /** 429 and 5xx. Worth another attempt after a wait. */
+  get isRetryable() {
+    return this.status === 429 || this.status >= 500;
+  }
+};
+var HydrafetchTimeoutError = class extends HydrafetchError {
+  constructor(message) {
+    super({ code: "TIMEOUT", message, status: 408 });
+    this.name = "HydrafetchTimeoutError";
+  }
+};
+
+// src/client.ts
+var DEFAULT_BASE_URL = "https://api.hydrafetch.com";
+var DEFAULT_TIMEOUT_MS = 12e4;
+var DEFAULT_MAX_RETRIES = 2;
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+var Hydrafetch = class {
+  apiKey;
+  baseUrl;
+  timeoutMs;
+  maxRetries;
+  fetchImpl;
+  constructor(options = {}) {
+    const opts = typeof options === "string" ? { apiKey: options } : options;
+    const env = globalThis.process?.env;
+    const apiKey = opts.apiKey ?? env?.HYDRAFETCH_API_KEY;
+    if (!apiKey) {
+      throw new HydrafetchError({
+        code: "MISSING_API_KEY",
+        message: "No API key. Pass one to the constructor or set HYDRAFETCH_API_KEY. Create a key at https://app.hydrafetch.com.",
+        status: 401
+      });
+    }
+    this.apiKey = apiKey;
+    this.baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
+    this.timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    this.maxRetries = opts.maxRetries ?? DEFAULT_MAX_RETRIES;
+    this.fetchImpl = opts.fetch ?? globalThis.fetch;
+    if (typeof this.fetchImpl !== "function") {
+      throw new HydrafetchError({
+        code: "NO_FETCH",
+        message: "No global fetch. Use Node 18 or newer, or pass a fetch implementation.",
+        status: 500
+      });
+    }
+  }
+  async request(method, path, body, query) {
+    const url = new URL(this.baseUrl + path);
+    for (const [k, v] of Object.entries(query ?? {})) {
+      if (v === void 0 || v === null) continue;
+      url.searchParams.set(k, String(v));
+    }
+    let lastError;
+    for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(), this.timeoutMs);
+      try {
+        const res = await this.fetchImpl(url.toString(), {
+          method,
+          headers: {
+            "X-API-Key": this.apiKey,
+            "content-type": "application/json",
+            accept: "application/json"
+          },
+          body: body === void 0 ? void 0 : JSON.stringify(body),
+          signal: controller.signal
+        });
+        const text = await res.text();
+        let parsed;
+        try {
+          parsed = text ? JSON.parse(text) : void 0;
+        } catch {
+          parsed = void 0;
+        }
+        if (!res.ok || parsed?.success === false) {
+          const err = new HydrafetchError({
+            code: parsed?.error?.code ?? `HTTP_${res.status}`,
+            message: parsed?.error?.message ?? text.slice(0, 300) ?? res.statusText,
+            status: res.status,
+            requestId: parsed?.meta?.requestId,
+            details: parsed?.error?.details
+          });
+          if (!err.isRetryable || attempt === this.maxRetries) throw err;
+          lastError = err;
+          await sleep(2 ** attempt * 500 + Math.floor(Math.random() * 250));
+          continue;
+        }
+        return parsed?.data ?? parsed;
+      } catch (error) {
+        if (error instanceof HydrafetchError) {
+          if (!error.isRetryable || attempt === this.maxRetries) throw error;
+          lastError = error;
+          continue;
+        }
+        if (error?.name === "AbortError") {
+          throw new HydrafetchTimeoutError(
+            `Request to ${path} exceeded ${this.timeoutMs}ms. Raise timeoutMs, or use async job endpoints for long work.`
+          );
+        }
+        if (attempt === this.maxRetries) throw error;
+        await sleep(2 ** attempt * 500);
+      } finally {
+        clearTimeout(timer);
+      }
+    }
+    throw lastError ?? new HydrafetchError({ code: "UNKNOWN", message: "Request failed", status: 500 });
+  }
+  /**
+   * One page as clean Markdown and structured data. One credit.
+   *
+   * The return type follows the formats you ask for, so the ones you requested
+   * are not optional:
+   *
+   * ```ts
+   * const a = await hf.scrape(url);                            // a.markdown: string
+   * const b = await hf.scrape(url, { formats: ['html'] });     // b.html: string
+   * ```
+   */
+  scrape(url, options = {}) {
+    return this.request("POST", "/v1/web/scrape", { url, ...options });
+  }
+  /** A page straight to a Markdown string, for when that is all you want. */
+  async markdown(url, options = {}) {
+    const page = await this.scrape(url, { ...options, formats: ["markdown"] });
+    return page.markdown;
+  }
+  /** A site's URLs from its sitemap and links, without fetching the pages. One credit. */
+  map(url, options = {}) {
+    return this.request("POST", "/v1/web/map", { url, ...options });
+  }
+  /** Ranked web results. One credit, plus one per result when scrapeResults is set. */
+  search(query, options = {}) {
+    return this.request("POST", "/v1/web/search", { query, ...options });
+  }
+  /** Typed JSON from one or more URLs. Five credits per URL. */
+  extract(urls, options = {}) {
+    return this.request("POST", "/v1/web/extract", {
+      urls: Array.isArray(urls) ? urls : [urls],
+      ...options
+    });
+  }
+  /** A company's brand from its domain. Five credits. */
+  brand(domain) {
+    return this.request("GET", "/v1/web/brand", void 0, { domain });
+  }
+  /** One embeddable logo. One credit, a fifth of a full brand resolve. */
+  logo(domain, options = {}) {
+    return this.request("GET", "/v1/web/brand/logo", void 0, { domain, ...options });
+  }
+  /** A site's design system, read from computed styles in a real browser. Ten credits. */
+  styleguide(domain) {
+    return this.request("GET", "/v1/web/styleguide", void 0, { domain });
+  }
+  /** A rendered PNG of the page, returned as a public URL. Five credits. */
+  screenshot(url, options = {}) {
+    return this.request("POST", "/v1/web/screenshot", { url, ...options });
+  }
+  /** A page's images with metadata, without rendering it. One credit. */
+  images(url, options = {}) {
+    return this.request("POST", "/v1/web/images", { url, ...options });
+  }
+  /** A page's links. One credit. */
+  links(url, options = {}) {
+    return this.request("POST", "/v1/web/links", { url, ...options });
+  }
+  /**
+   * Start a crawl from a seed URL. Returns a job id immediately; poll with
+   * crawlStatus, or use crawlAndWait to do both.
+   */
+  async startCrawl(url, options = {}) {
+    const res = await this.request("POST", "/v1/web/crawl", {
+      url,
+      ...options
+    });
+    const id = res.crawlId ?? res.id;
+    if (!id) {
+      throw new HydrafetchError({
+        code: "NO_JOB_ID",
+        message: "Crawl accepted but returned no id.",
+        status: 502
+      });
+    }
+    return id;
+  }
+  crawlStatus(id) {
+    return this.request("GET", `/v1/web/crawl/${encodeURIComponent(id)}`);
+  }
+  /**
+   * Start a batch over a known list of URLs. Per-page options belong in scrapeOptions,
+   * not at the top level.
+   */
+  async startBatch(urls, options = {}) {
+    const res = await this.request("POST", "/v1/web/batch", {
+      urls,
+      ...options
+    });
+    const id = res.batchId ?? res.id;
+    if (!id) {
+      throw new HydrafetchError({
+        code: "NO_JOB_ID",
+        message: "Batch accepted but returned no id.",
+        status: 502
+      });
+    }
+    return id;
+  }
+  batchStatus(id) {
+    return this.request("GET", `/v1/web/batch/${encodeURIComponent(id)}`);
+  }
+  /** Start a crawl and poll until it reaches a terminal state. */
+  crawlAndWait(url, options = {}, wait = {}) {
+    return this.startCrawl(url, options).then((id) => this.waitForJob(() => this.crawlStatus(id), wait));
+  }
+  /** Start a batch and poll until it reaches a terminal state. */
+  batchAndWait(urls, options = {}, wait = {}) {
+    return this.startBatch(urls, options).then((id) => this.waitForJob(() => this.batchStatus(id), wait));
+  }
+  async waitForJob(poll, wait) {
+    const timeoutMs = wait.timeoutMs ?? 3e5;
+    const interval = wait.pollIntervalMs ?? 2e3;
+    const deadline = Date.now() + timeoutMs;
+    const terminal = /* @__PURE__ */ new Set(["completed", "failed", "cancelled"]);
+    for (; ; ) {
+      const job = await poll();
+      wait.onProgress?.(job);
+      if (terminal.has(String(job.status))) return job;
+      if (Date.now() + interval > deadline) {
+        throw new HydrafetchTimeoutError(
+          `Job did not finish within ${timeoutMs}ms. It is still running server-side; poll its status directly, or pass a webhook.`
+        );
+      }
+      await sleep(interval);
+    }
+  }
+};
+export {
+  Hydrafetch,
+  HydrafetchError,
+  HydrafetchTimeoutError
+};

package/package.json

@@ -0,0 +1,55 @@
+{
+  "name": "@hydrafetch/node-sdk",
+  "version": "0.1.0",
+  "description": "Official Node/TypeScript client for the Hydrafetch web data API. Turn any URL into clean Markdown and structured data.",
+  "keywords": [
+    "hydrafetch",
+    "web-scraping",
+    "markdown",
+    "llm",
+    "rag",
+    "crawler",
+    "structured-data",
+    "agents"
+  ],
+  "homepage": "https://hydrafetch.com",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Hydrafetch/node-sdk.git"
+  },
+  "bugs": "https://github.com/Hydrafetch/node-sdk/issues",
+  "license": "MIT",
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "scripts": {
+    "build": "tsup src/index.ts --format esm,cjs --dts --clean",
+    "test": "vitest run",
+    "typecheck": "tsc --noEmit",
+    "prepublishOnly": "npm run typecheck && npm run test && npm run build"
+  },
+  "devDependencies": {
+    "tsup": "^8.5.0",
+    "typescript": "^5.9.2",
+    "vitest": "^3.2.4"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}