@webcrawl/client 0.0.0-stage → 1.2.0

package/README.md

@@ -1,3 +1,48 @@
-# Temporary Holding Version
+# @webcrawl/client
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Zero-dependency TypeScript & Node.js SDK for **[Webcrawl Anything](https://github.com/brainfish-ai/webcrawl-anything)** and **[webcrawl.app](https://webcrawl.app)**.
+
+## Install
+
+```bash
+npm install @webcrawl/client
+```
+
+## Quickstart
+
+```ts
+import { WebcrawlClient } from "@webcrawl/client";
+
+const client = new WebcrawlClient({
+  apiKey: process.env.WEBCRAWL_API_KEY, // or run `npx webcrawl login`
+});
+
+// 1. Search the web or structured verticals (shopping, flights, hotels, real-estate, security CVEs)
+const results = await client.search({
+  query: "CVE-2024-3094 xz utils",
+  category: "security",
+  subcategory: "cve",
+  limit: 5,
+});
+
+// 2. Scrape any URL into clean Markdown
+const page = await client.scrape({
+  url: "https://github.com/brainfish-ai/webcrawl-anything",
+  formats: ["markdown"],
+});
+
+// 3. Parse PDFs, Office files (.docx, .xlsx, .pptx), or YouTube transcripts
+const doc = await client.parse({
+  url: "https://arxiv.org/pdf/1706.03762.pdf",
+  ocr: "auto",
+});
+
+// 4. List all supported crawl & search targets
+const targets = await client.targets();
+```
+
+## Documentation & Source
+
+- **GitHub Repository:** [brainfish-ai/webcrawl-anything](https://github.com/brainfish-ai/webcrawl-anything)
+- **Cloud Platform:** [https://webcrawl.app](https://webcrawl.app)
+- **License:** MIT

package/dist/client.d.ts

@@ -0,0 +1,35 @@
+import type { ApiResponse, CrawlParams, ExtractParams, MapParams, ParseParams, ParseResponse, RerankParams, ScrapeParams, SearchParams, SearchResponse } from "./types.js";
+export interface ClientOptions {
+    apiKey?: string;
+    baseUrl?: string;
+    /** Per-request timeout in ms. Default 60s. */
+    timeoutMs?: number;
+    fetch?: typeof fetch;
+    userAgent?: string;
+    /** IANA timezone sent as `X-User-Timezone` so results are local. Defaults to this machine's. */
+    timezone?: string;
+}
+export declare class WebcrawlClient {
+    readonly baseUrl: string;
+    private readonly apiKey?;
+    private readonly timeoutMs;
+    private readonly fetchImpl;
+    private readonly userAgent;
+    private readonly timezone?;
+    constructor(opts?: ClientOptions);
+    request<T = ApiResponse>(method: "GET" | "POST", path: string, body?: unknown): Promise<T>;
+    search(p: SearchParams): Promise<SearchResponse>;
+    scrape(p: ScrapeParams): Promise<ApiResponse>;
+    /** PDF, DOCX/XLSX/PPTX, image or audio/video to Markdown. Long files: pass `async: true` and poll `run()`. */
+    parse(p: ParseParams): Promise<ParseResponse>;
+    /** Status, trace and (when finished) result of an async run. */
+    run(id: string): Promise<ApiResponse>;
+    map(p: MapParams): Promise<ApiResponse>;
+    crawl(p: CrawlParams): Promise<ApiResponse>;
+    extract(p: ExtractParams): Promise<ApiResponse>;
+    rerank(p: RerankParams): Promise<ApiResponse>;
+    categories(): Promise<ApiResponse>;
+    targets(): Promise<ApiResponse>;
+    usage(): Promise<ApiResponse>;
+}
+export declare function createClient(opts?: ClientOptions): WebcrawlClient;

package/dist/client.js

@@ -0,0 +1,130 @@
+import { DEFAULT_BASE_URL } from "./config.js";
+import { WebcrawlError, kindForStatus } from "./errors.js";
+function localTimezone() {
+    try {
+        return Intl.DateTimeFormat().resolvedOptions().timeZone;
+    }
+    catch {
+        return undefined;
+    }
+}
+function bytesToBase64(bytes) {
+    if (typeof Buffer !== "undefined") {
+        return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString("base64");
+    }
+    let s = "";
+    for (let i = 0; i < bytes.length; i += 0x8000) {
+        s += String.fromCharCode(...bytes.subarray(i, i + 0x8000));
+    }
+    return btoa(s);
+}
+const compact = (o) => Object.fromEntries(Object.entries(o).filter(([, v]) => v !== undefined));
+export class WebcrawlClient {
+    baseUrl;
+    apiKey;
+    timeoutMs;
+    fetchImpl;
+    userAgent;
+    timezone;
+    constructor(opts = {}) {
+        this.baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
+        this.apiKey = opts.apiKey;
+        this.timeoutMs = opts.timeoutMs ?? 60_000;
+        this.fetchImpl = opts.fetch ?? globalThis.fetch;
+        this.userAgent = opts.userAgent ?? "webcrawl-client/0.1";
+        this.timezone = opts.timezone ?? localTimezone();
+    }
+    async request(method, path, body) {
+        const headers = {
+            accept: "application/json",
+            "user-agent": this.userAgent,
+        };
+        if (this.apiKey) {
+            headers.authorization = `Bearer ${this.apiKey}`;
+        }
+        if (this.timezone) {
+            headers["x-user-timezone"] = this.timezone;
+        }
+        if (body !== undefined) {
+            headers["content-type"] = "application/json";
+        }
+        let res;
+        try {
+            res = await this.fetchImpl(`${this.baseUrl}${path}`, {
+                method,
+                headers,
+                body: body === undefined ? undefined : JSON.stringify(body),
+                signal: AbortSignal.timeout(this.timeoutMs),
+            });
+        }
+        catch (e) {
+            const name = e instanceof Error ? e.name : "";
+            if (name === "TimeoutError" || name === "AbortError") {
+                throw new WebcrawlError("timeout", `Request to ${path} timed out after ${this.timeoutMs}ms`);
+            }
+            throw new WebcrawlError("network", e instanceof Error ? e.message : String(e));
+        }
+        const text = await res.text();
+        let json;
+        try {
+            json = text ? JSON.parse(text) : {};
+        }
+        catch {
+            json = { error: text.slice(0, 300) };
+        }
+        if (!res.ok) {
+            const obj = (json && typeof json === "object" ? json : {});
+            const retry = Number(res.headers.get("retry-after"));
+            throw new WebcrawlError(kindForStatus(res.status), typeof obj.error === "string" ? obj.error : `HTTP ${res.status}`, { status: res.status, retryAfter: Number.isFinite(retry) && retry > 0 ? retry : undefined, body: json });
+        }
+        return json;
+    }
+    search(p) {
+        const { sites, scrapeContent, ...rest } = p;
+        return this.request("POST", "/v1/search", {
+            ...compact(rest),
+            ...(sites?.length ? { filters: { sites } } : {}),
+            ...(scrapeContent ? { scrapeOptions: { formats: ["markdown"] } } : {}),
+        });
+    }
+    scrape(p) {
+        return this.request("POST", "/v1/scrape", compact({ url: p.url, formats: p.formats, noCache: p.noCache }));
+    }
+    /** PDF, DOCX/XLSX/PPTX, image or audio/video to Markdown. Long files: pass `async: true` and poll `run()`. */
+    parse(p) {
+        const { bytes, noCache, ...rest } = p;
+        if (!p.url && !bytes && !p.base64) {
+            throw new WebcrawlError("bad_request", "parse needs `url`, `bytes` or `base64`");
+        }
+        const base64 = bytes ? bytesToBase64(bytes) : p.base64;
+        return this.request("POST", "/v1/parse", compact({ ...rest, base64, ...(noCache ? { noCache: true } : {}) }));
+    }
+    /** Status, trace and (when finished) result of an async run. */
+    run(id) {
+        return this.request("GET", `/v1/runs/${encodeURIComponent(id)}`);
+    }
+    map(p) {
+        return this.request("POST", "/v1/map", compact({ ...p }));
+    }
+    crawl(p) {
+        return this.request("POST", "/v1/crawl", compact({ ...p, maxPages: p.maxPages === undefined ? undefined : Math.min(25, Math.max(1, p.maxPages)) }));
+    }
+    extract(p) {
+        return this.request("POST", "/v1/extract", compact({ ...p }));
+    }
+    rerank(p) {
+        return this.request("POST", "/v1/rerank", compact({ ...p }));
+    }
+    categories() {
+        return this.request("GET", "/v1/search/categories");
+    }
+    targets() {
+        return this.request("GET", "/v1/targets");
+    }
+    usage() {
+        return this.request("GET", "/v1/account/usage");
+    }
+}
+export function createClient(opts = {}) {
+    return new WebcrawlClient(opts);
+}

package/dist/config.d.ts

@@ -0,0 +1,27 @@
+export declare const DEFAULT_BASE_URL = "https://webcrawl.app";
+export type ExecutionMode = "local" | "cloud" | "auto";
+export interface FileConfig {
+    apiKey?: string;
+    baseUrl?: string;
+    mode?: ExecutionMode;
+}
+export interface ResolvedConfig {
+    apiKey?: string;
+    baseUrl: string;
+    mode: ExecutionMode;
+    effectiveMode: "local" | "cloud";
+    source: {
+        apiKey: "flag" | "env" | "file" | "none";
+        baseUrl: "flag" | "env" | "file" | "default";
+        mode: "flag" | "env" | "file" | "default";
+    };
+}
+export declare function configPath(env?: NodeJS.ProcessEnv): string;
+export declare function readConfigFile(path?: string): FileConfig;
+export declare function writeConfigFile(cfg: FileConfig, path?: string): void;
+/** Precedence: explicit flags > environment > config file > defaults. */
+export declare function resolveConfig(flags?: {
+    apiKey?: string;
+    baseUrl?: string;
+    mode?: string;
+}, env?: NodeJS.ProcessEnv, file?: FileConfig): ResolvedConfig;

package/dist/config.js

@@ -0,0 +1,58 @@
+import { chmodSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+export const DEFAULT_BASE_URL = "https://webcrawl.app";
+export function configPath(env = process.env) {
+    const base = env.XDG_CONFIG_HOME || join(homedir(), ".config");
+    return join(base, "webcrawl", "config.json");
+}
+export function readConfigFile(path = configPath()) {
+    try {
+        const parsed = JSON.parse(readFileSync(path, "utf8"));
+        if (parsed && typeof parsed === "object") {
+            const o = parsed;
+            return {
+                apiKey: typeof o.apiKey === "string" ? o.apiKey : undefined,
+                baseUrl: typeof o.baseUrl === "string" ? o.baseUrl : undefined,
+                mode: o.mode === "local" || o.mode === "cloud" || o.mode === "auto" ? o.mode : undefined,
+            };
+        }
+    }
+    catch {
+        // missing or unreadable config is fine
+    }
+    return {};
+}
+export function writeConfigFile(cfg, path = configPath()) {
+    mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+    writeFileSync(path, `${JSON.stringify(cfg, null, 2)}\n`, { mode: 0o600 });
+    chmodSync(path, 0o600);
+}
+const clean = (v) => (v && v.trim() ? v.trim() : undefined);
+/** Precedence: explicit flags > environment > config file > defaults. */
+export function resolveConfig(flags = {}, env = process.env, file = readConfigFile(configPath(env))) {
+    const pick = (f, e, fl, none) => {
+        if (clean(f)) {
+            return [clean(f), "flag"];
+        }
+        if (clean(e)) {
+            return [clean(e), "env"];
+        }
+        if (clean(fl)) {
+            return [clean(fl), "file"];
+        }
+        return [undefined, none];
+    };
+    const [apiKey, keySrc] = pick(flags.apiKey, env.WEBCRAWL_API_KEY, file.apiKey, "none");
+    const [baseUrl, urlSrc] = pick(flags.baseUrl, env.WEBCRAWL_URL, file.baseUrl, "default");
+    const [rawMode, modeSrc] = pick(flags.mode, env.WEBCRAWL_MODE, file.mode, "default");
+    const mode = rawMode === "local" || rawMode === "cloud" || rawMode === "auto" ? rawMode : "auto";
+    const effectiveMode = mode === "local" ? "local" : mode === "cloud" ? "cloud" : apiKey ? "cloud" : "local";
+    return {
+        apiKey,
+        baseUrl: (baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, ""),
+        mode,
+        effectiveMode,
+        source: { apiKey: keySrc, baseUrl: urlSrc, mode: modeSrc },
+    };
+}

package/dist/errors.d.ts

@@ -0,0 +1,16 @@
+export type WebcrawlErrorKind = "auth" | "credits" | "rate_limit" | "bad_request" | "not_found" | "server" | "network" | "timeout";
+export declare class WebcrawlError extends Error {
+    readonly kind: WebcrawlErrorKind;
+    readonly status?: number;
+    /** Seconds to wait before retrying, from `Retry-After` on 429. */
+    readonly retryAfter?: number;
+    readonly body?: unknown;
+    constructor(kind: WebcrawlErrorKind, message: string, opts?: {
+        status?: number;
+        retryAfter?: number;
+        body?: unknown;
+    });
+}
+export declare function kindForStatus(status: number): WebcrawlErrorKind;
+/** One actionable sentence for people (and models) reading the error. */
+export declare function describeError(err: unknown, baseUrl?: string): string;

package/dist/errors.js

@@ -0,0 +1,55 @@
+export class WebcrawlError extends Error {
+    kind;
+    status;
+    /** Seconds to wait before retrying, from `Retry-After` on 429. */
+    retryAfter;
+    body;
+    constructor(kind, message, opts = {}) {
+        super(message);
+        this.name = "WebcrawlError";
+        this.kind = kind;
+        this.status = opts.status;
+        this.retryAfter = opts.retryAfter;
+        this.body = opts.body;
+    }
+}
+export function kindForStatus(status) {
+    if (status === 401 || status === 403) {
+        return "auth";
+    }
+    if (status === 402) {
+        return "credits";
+    }
+    if (status === 429) {
+        return "rate_limit";
+    }
+    if (status === 404) {
+        return "not_found";
+    }
+    if (status >= 500) {
+        return "server";
+    }
+    return "bad_request";
+}
+/** One actionable sentence for people (and models) reading the error. */
+export function describeError(err, baseUrl = "https://webcrawl.app") {
+    if (!(err instanceof WebcrawlError)) {
+        return err instanceof Error ? err.message : String(err);
+    }
+    switch (err.kind) {
+        case "auth":
+            return "Your API key is missing or invalid. Set WEBCRAWL_API_KEY or run `webcrawl login`.";
+        case "credits":
+            return `You are out of credits. Add more at ${baseUrl}/account.`;
+        case "rate_limit":
+            return err.retryAfter
+                ? `Rate limited. Retry in ${err.retryAfter}s.`
+                : "Rate limited. Wait a moment and retry.";
+        case "timeout":
+            return "The request timed out. Try again, or narrow the request.";
+        case "network":
+            return `Could not reach ${baseUrl}. Check your connection or WEBCRAWL_URL.`;
+        default:
+            return err.message;
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export { WebcrawlClient, createClient, type ClientOptions } from "./client.js";
+export { DEFAULT_BASE_URL, configPath, readConfigFile, resolveConfig, writeConfigFile, type ExecutionMode, type FileConfig, type ResolvedConfig, } from "./config.js";
+export { WebcrawlError, describeError, kindForStatus, type WebcrawlErrorKind } from "./errors.js";
+export type * from "./types.js";

package/dist/index.js

@@ -0,0 +1,3 @@
+export { WebcrawlClient, createClient } from "./client.js";
+export { DEFAULT_BASE_URL, configPath, readConfigFile, resolveConfig, writeConfigFile, } from "./config.js";
+export { WebcrawlError, describeError, kindForStatus } from "./errors.js";

package/dist/types.d.ts

@@ -0,0 +1,126 @@
+/** Fields every API response may carry. Responses are intentionally loose: the API grows faster than this client. */
+export interface CreditInfo {
+    charged?: number;
+    used?: number;
+    remaining?: number;
+    limit?: number;
+}
+export interface ApiResponse {
+    success?: boolean;
+    credits?: CreditInfo;
+    [key: string]: unknown;
+}
+export interface Location {
+    country?: string;
+    region?: string;
+    city?: string;
+    timezone?: string;
+}
+export interface SearchParams extends Location {
+    query: string;
+    category?: string;
+    subcategory?: string;
+    limit?: number;
+    /** Restrict to these domains. */
+    sites?: string[];
+    includeDomains?: string[];
+    excludeDomains?: string[];
+    /** Time filter, e.g. `qdr:w`. */
+    tbs?: string;
+    scrapeContent?: boolean;
+}
+export interface SearchResponse extends ApiResponse {
+    results?: Array<{
+        title?: string;
+        url?: string;
+        description?: string;
+        markdown?: string;
+    }>;
+    data?: unknown;
+    offers?: unknown[];
+    products?: unknown[];
+    listings?: unknown[];
+    flights?: unknown[];
+    hotels?: unknown[];
+    businesses?: unknown[];
+    /** Structured CVE records (category "security"): id, cvssScore, severity, affected, fixed versions, kev, references. */
+    cves?: unknown[];
+}
+export interface ScrapeParams {
+    url: string;
+    formats?: Array<"markdown" | "links" | "html">;
+    noCache?: boolean;
+}
+/** Parse a file (PDF, Office, image, audio/video) from a URL, raw bytes or base64. */
+export interface ParseParams {
+    /** File URL. Provide one of `url`, `bytes` or `base64`. */
+    url?: string;
+    /** Raw file bytes (sent as base64 JSON). */
+    bytes?: Uint8Array;
+    base64?: string;
+    fileName?: string;
+    contentType?: string;
+    /** `auto` OCRs only scanned pages; `force` OCRs every page; `off` never OCRs. */
+    ocr?: "auto" | "force" | "off";
+    /** 1-indexed PDF pages to read. */
+    pages?: number[];
+    maxOcrPages?: number;
+    /** ISO-639-1 hint for transcription. */
+    language?: string;
+    /** WebVTT/SRT captions to use instead of transcribing. */
+    captions?: string;
+    describeVideo?: boolean;
+    /** Run as a durable job (returns a run id), for big PDFs and long media. */
+    async?: boolean;
+    noCache?: boolean;
+}
+export interface ParseMetadata {
+    kind?: string;
+    method?: "native" | "ocr" | "vision" | "transcript";
+    pages?: number;
+    pagesOcr?: number[];
+    pagesSkipped?: Array<{
+        page: number;
+        reason: string;
+    }>;
+    durationSeconds?: number;
+    warnings?: string[];
+}
+export interface ParseResponse extends ApiResponse {
+    title?: string;
+    markdown?: string;
+    /** Present when the parse runs as a durable job. */
+    id?: string;
+    status?: string;
+    statusUrl?: string;
+    metadata?: {
+        parse?: ParseMetadata;
+        [key: string]: unknown;
+    };
+}
+export interface MapParams {
+    url: string;
+    limit?: number;
+    search?: string;
+}
+export interface CrawlParams {
+    url: string;
+    maxPages?: number;
+    extractStructured?: boolean;
+}
+export interface ExtractParams {
+    url: string;
+    prompt?: string;
+    schema?: Record<string, unknown>;
+}
+export type RerankDocument = string | {
+    title?: string;
+    url?: string;
+    text?: string;
+    description?: string;
+};
+export interface RerankParams {
+    query: string;
+    documents: RerankDocument[];
+    topK?: number;
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,52 @@
 {
   "name": "@webcrawl/client",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "1.2.0",
+  "description": "Zero-dependency TypeScript & Node.js SDK for Webcrawl Anything (search, scrape, parse, map, crawl, extract, rerank, targets)",
+  "type": "module",
+  "license": "MIT",
+  "author": "Brainfish AI <support@webcrawl.app> (https://webcrawl.app)",
+  "homepage": "https://webcrawl.app",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/brainfish-ai/webcrawl-anything.git",
+    "directory": "packages/client"
+  },
+  "bugs": {
+    "url": "https://github.com/brainfish-ai/webcrawl-anything/issues"
+  },
+  "keywords": [
+    "webcrawl",
+    "web-crawler",
+    "web-scraper",
+    "search-api",
+    "markdown",
+    "pdf-parser",
+    "cve",
+    "mcp",
+    "ai-agents"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "development": "./src/index.ts",
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "prepack": "npm run build",
+    "test": "node --conditions=development --import tsx --test test/*.test.ts"
+  },
+  "engines": {
+    "node": ">=20"
+  }
+}