@webcrawl/client 0.0.0-stage → 1.2.2

package/README.md

@@ -1,3 +1,48 @@
-# Temporary Holding Version
+# @webcrawl/client
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Zero-dependency TypeScript & Node.js SDK for **[Webcrawl Anything](https://github.com/brainfish-ai/webcrawl-anything)** and **[webcrawl.app](https://webcrawl.app)**.
+
+## Install
+
+```bash
+npm install @webcrawl/client
+```
+
+## Quickstart
+
+```ts
+import { WebcrawlClient } from "@webcrawl/client";
+
+const client = new WebcrawlClient({
+  apiKey: process.env.WEBCRAWL_API_KEY, // or run `npx webcrawl login`
+});
+
+// 1. Search the web or structured verticals (shopping, flights, hotels, real-estate, security CVEs)
+const results = await client.search({
+  query: "CVE-2024-3094 xz utils",
+  category: "security",
+  subcategory: "cve",
+  limit: 5,
+});
+
+// 2. Scrape any URL into clean Markdown
+const page = await client.scrape({
+  url: "https://github.com/brainfish-ai/webcrawl-anything",
+  formats: ["markdown"],
+});
+
+// 3. Parse PDFs, Office files (.docx, .xlsx, .pptx), or YouTube transcripts
+const doc = await client.parse({
+  url: "https://arxiv.org/pdf/1706.03762.pdf",
+  ocr: "auto",
+});
+
+// 4. List all supported crawl & search targets
+const targets = await client.targets();
+```
+
+## Documentation & Source
+
+- **GitHub Repository:** [brainfish-ai/webcrawl-anything](https://github.com/brainfish-ai/webcrawl-anything)
+- **Cloud Platform:** [https://webcrawl.app](https://webcrawl.app)
+- **License:** MIT

package/dist/client.d.ts

@@ -0,0 +1,80 @@
+import type { ApiResponse, CrawlParams, ExtractParams, MapParams, ParseParams, ParseResponse, RerankParams, ScrapeParams, SearchParams, SearchResponse } from "./types.js";
+export interface ClientOptions {
+    apiKey?: string;
+    baseUrl?: string;
+    /** Per-request timeout in ms. Default 60s. */
+    timeoutMs?: number;
+    fetch?: typeof fetch;
+    userAgent?: string;
+    /** IANA timezone sent as `X-User-Timezone` so results are local. Defaults to this machine's. */
+    timezone?: string;
+}
+export declare class WebcrawlClient {
+    readonly baseUrl: string;
+    private readonly apiKey?;
+    private readonly timeoutMs;
+    private readonly fetchImpl;
+    private readonly hasCustomFetch;
+    private readonly userAgent;
+    private readonly timezone?;
+    constructor(opts?: ClientOptions);
+    request<T = ApiResponse>(method: "GET" | "POST", path: string, body?: unknown): Promise<T>;
+    search(p: SearchParams): Promise<SearchResponse>;
+    scrape(p: ScrapeParams): Promise<ApiResponse>;
+    /** PDF, DOCX/XLSX/PPTX, image or audio/video to Markdown. Long files: pass `async: true` and poll `run()`. */
+    parse(p: ParseParams): Promise<ParseResponse>;
+    /** Status, trace and (when finished) result of an async run. */
+    run(id: string): Promise<ApiResponse>;
+    map(p: MapParams): Promise<ApiResponse>;
+    crawl(p: CrawlParams): Promise<ApiResponse>;
+    extract(p: ExtractParams): Promise<ApiResponse>;
+    rerank(p: RerankParams): Promise<ApiResponse>;
+    categories(): Promise<ApiResponse>;
+    targets(): Promise<ApiResponse>;
+    usage(): Promise<ApiResponse>;
+    /** Send a 6-digit OTP verification code to an email address to sign up or log in (provisions 1,000 free monthly credits). */
+    sendOtp(email: string): Promise<{
+        success: boolean;
+        email: string;
+        expiresInSeconds: number;
+        transport?: string;
+        devCode?: string;
+        error?: string;
+    }>;
+    /** Verify a 6-digit OTP code, creating the user account (with 1,000 monthly credits) if new, and return a `wc_live_...` API key. */
+    verifyOtp(params: {
+        email: string;
+        code: string;
+        name?: string;
+    }): Promise<{
+        success: boolean;
+        isNewUser?: boolean;
+        apiKey?: string;
+        sessionToken?: string;
+        user?: {
+            id: string;
+            email: string;
+            name?: string | null;
+            plan: string;
+            creditsLimit: number;
+            creditsUsed: number;
+            creditsRemaining: number;
+        };
+        error?: string;
+    }>;
+    /** Fetch authenticated user profile & remaining credit balance. */
+    me(): Promise<{
+        success: boolean;
+        authenticated: boolean;
+        user?: {
+            id: string;
+            email: string;
+            name?: string | null;
+            plan: string;
+            creditsLimit: number;
+            creditsUsed: number;
+            creditsRemaining: number;
+        };
+    }>;
+}
+export declare function createClient(opts?: ClientOptions): WebcrawlClient;

package/dist/client.js

@@ -0,0 +1,420 @@
+import { DEFAULT_BASE_URL } from "./config.js";
+import { WebcrawlError, kindForStatus } from "./errors.js";
+function localTimezone() {
+    try {
+        return Intl.DateTimeFormat().resolvedOptions().timeZone;
+    }
+    catch {
+        return undefined;
+    }
+}
+function bytesToBase64(bytes) {
+    if (typeof Buffer !== "undefined") {
+        return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength).toString("base64");
+    }
+    let s = "";
+    for (let i = 0; i < bytes.length; i += 0x8000) {
+        s += String.fromCharCode(...bytes.subarray(i, i + 0x8000));
+    }
+    return btoa(s);
+}
+const compact = (o) => Object.fromEntries(Object.entries(o).filter(([, v]) => v !== undefined));
+export class WebcrawlClient {
+    baseUrl;
+    apiKey;
+    timeoutMs;
+    fetchImpl;
+    hasCustomFetch;
+    userAgent;
+    timezone;
+    constructor(opts = {}) {
+        this.baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
+        this.apiKey = opts.apiKey;
+        this.timeoutMs = opts.timeoutMs ?? 60_000;
+        this.fetchImpl = opts.fetch ?? globalThis.fetch;
+        this.hasCustomFetch = Boolean(opts.fetch);
+        this.userAgent = opts.userAgent ?? "webcrawl-client/0.1";
+        this.timezone = opts.timezone ?? localTimezone();
+    }
+    async request(method, path, body) {
+        const headers = {
+            accept: "application/json",
+            "user-agent": this.userAgent,
+        };
+        if (this.apiKey) {
+            headers.authorization = `Bearer ${this.apiKey}`;
+        }
+        if (this.timezone) {
+            headers["x-user-timezone"] = this.timezone;
+        }
+        if (body !== undefined) {
+            headers["content-type"] = "application/json";
+        }
+        let res;
+        try {
+            res = await this.fetchImpl(`${this.baseUrl}${path}`, {
+                method,
+                headers,
+                body: body === undefined ? undefined : JSON.stringify(body),
+                signal: AbortSignal.timeout(this.timeoutMs),
+            });
+        }
+        catch (e) {
+            const name = e instanceof Error ? e.name : "";
+            if (name === "TimeoutError" || name === "AbortError") {
+                throw new WebcrawlError("timeout", `Request to ${path} timed out after ${this.timeoutMs}ms`);
+            }
+            throw new WebcrawlError("network", e instanceof Error ? e.message : String(e));
+        }
+        const text = await res.text();
+        let json;
+        try {
+            json = text ? JSON.parse(text) : {};
+        }
+        catch {
+            json = { error: text.slice(0, 300) };
+        }
+        if (!res.ok) {
+            const obj = (json && typeof json === "object" ? json : {});
+            const retry = Number(res.headers.get("retry-after"));
+            throw new WebcrawlError(kindForStatus(res.status), typeof obj.error === "string" ? obj.error : `HTTP ${res.status}`, { status: res.status, retryAfter: Number.isFinite(retry) && retry > 0 ? retry : undefined, body: json });
+        }
+        return json;
+    }
+    async search(p) {
+        const { sites, scrapeContent, ...rest } = p;
+        const rawCat = (p.category || "").trim().toLowerCase();
+        const rawSub = (p.subcategory || "").trim().toLowerCase();
+        const isAcademic = rawCat === "academic" ||
+            rawCat === "paper" ||
+            rawCat === "papers" ||
+            rawCat === "research" ||
+            rawCat === "arxiv" ||
+            rawCat === "scholar" ||
+            rawCat === "scholarly" ||
+            rawSub === "arxiv" ||
+            rawSub === "papers" ||
+            rawSub === "ml" ||
+            rawSub === "pubmed" ||
+            rawSub === "math";
+        const proxyPromise = this.request("POST", "/v1/search", {
+            ...compact({ ...rest, ...(isAcademic ? { category: "academic" } : {}) }),
+            ...(sites?.length ? { filters: { sites } } : {}),
+            ...(scrapeContent ? { scrapeOptions: { formats: ["markdown"] } } : {}),
+        });
+        if (!isAcademic || this.hasCustomFetch) {
+            return proxyPromise;
+        }
+        const limit = p.limit ?? 5;
+        if (!this.apiKey) {
+            const scholarlyItems = await fetchAcademicPapersForClient(this.fetchImpl, p.query, limit);
+            return {
+                success: true,
+                category: "academic",
+                results: scholarlyItems.slice(0, limit),
+            };
+        }
+        const [proxyRes, scholarlyItems] = await Promise.all([
+            proxyPromise.catch(() => ({ success: true, results: [] })),
+            fetchAcademicPapersForClient(this.fetchImpl, p.query, limit),
+        ]);
+        if (!scholarlyItems.length) {
+            return proxyRes;
+        }
+        const existing = Array.isArray(proxyRes.results) ? proxyRes.results : [];
+        const merged = [];
+        const seen = new Set();
+        for (const item of [...scholarlyItems, ...existing]) {
+            const url = typeof item.url === "string" ? item.url.toLowerCase().replace(/v\d+$/, "") : "";
+            if (!url || seen.has(url)) {
+                continue;
+            }
+            seen.add(url);
+            merged.push(item);
+            if (merged.length >= limit) {
+                break;
+            }
+        }
+        return {
+            ...proxyRes,
+            success: true,
+            category: "academic",
+            results: merged,
+        };
+    }
+    scrape(p) {
+        return this.request("POST", "/v1/scrape", compact({ url: p.url, formats: p.formats, noCache: p.noCache }));
+    }
+    /** PDF, DOCX/XLSX/PPTX, image or audio/video to Markdown. Long files: pass `async: true` and poll `run()`. */
+    parse(p) {
+        const { bytes, noCache, ...rest } = p;
+        if (!p.url && !bytes && !p.base64) {
+            throw new WebcrawlError("bad_request", "parse needs `url`, `bytes` or `base64`");
+        }
+        const base64 = bytes ? bytesToBase64(bytes) : p.base64;
+        return this.request("POST", "/v1/parse", compact({ ...rest, base64, ...(noCache ? { noCache: true } : {}) }));
+    }
+    /** Status, trace and (when finished) result of an async run. */
+    run(id) {
+        return this.request("GET", `/v1/runs/${encodeURIComponent(id)}`);
+    }
+    map(p) {
+        return this.request("POST", "/v1/map", compact({ ...p }));
+    }
+    crawl(p) {
+        return this.request("POST", "/v1/crawl", compact({ ...p, maxPages: p.maxPages === undefined ? undefined : Math.min(25, Math.max(1, p.maxPages)) }));
+    }
+    extract(p) {
+        return this.request("POST", "/v1/extract", compact({ ...p }));
+    }
+    rerank(p) {
+        return this.request("POST", "/v1/rerank", compact({ ...p }));
+    }
+    categories() {
+        return this.request("GET", "/v1/search/categories");
+    }
+    async targets() {
+        try {
+            return await this.request("GET", "/v1/targets");
+        }
+        catch (e) {
+            if (!this.hasCustomFetch && e instanceof WebcrawlError && (e.status === 404 || e.status === 405)) {
+                return { success: true, count: BUILTIN_CLIENT_TARGETS.length, targets: BUILTIN_CLIENT_TARGETS };
+            }
+            throw e;
+        }
+    }
+    usage() {
+        return this.request("GET", "/v1/account/usage");
+    }
+    /** Send a 6-digit OTP verification code to an email address to sign up or log in (provisions 1,000 free monthly credits). */
+    sendOtp(email) {
+        return this.request("POST", "/v1/auth/otp/send", { email: email.trim() });
+    }
+    /** Verify a 6-digit OTP code, creating the user account (with 1,000 monthly credits) if new, and return a `wc_live_...` API key. */
+    async verifyOtp(params) {
+        const res = await this.request("POST", "/v1/auth/otp/verify", {
+            email: params.email.trim(),
+            code: params.code.trim(),
+            ...(params.name ? { name: params.name.trim() } : {}),
+        });
+        if (res.success && !res.apiKey && res.sessionToken) {
+            try {
+                const sessionClient = new WebcrawlClient({
+                    baseUrl: this.baseUrl,
+                    apiKey: res.sessionToken,
+                    fetch: this.fetchImpl,
+                    userAgent: this.userAgent,
+                });
+                const keyRes = await sessionClient.request("POST", "/v1/account/keys", { name: "CLI API Key" });
+                if (keyRes.apiKey) {
+                    res.apiKey = keyRes.apiKey;
+                }
+            }
+            catch {
+                // Fallback to sessionToken if key creation fails
+                res.apiKey = res.sessionToken;
+            }
+        }
+        return res;
+    }
+    /** Fetch authenticated user profile & remaining credit balance. */
+    me() {
+        return this.request("GET", "/v1/auth/me");
+    }
+}
+const decodeXmlEntities = (s) => s
+    .replace(/&amp;/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'");
+async function fetchAcademicPapersForClient(fetchImpl, rawQuery, limit) {
+    const clean = rawQuery
+        .replace(/\([^)]*\)|-?site:\S+/g, " ")
+        .replace(/["'`]+/g, " ")
+        .replace(/\s+/g, " ")
+        .trim();
+    if (!clean) {
+        return [];
+    }
+    const arxivIdMatch = clean.match(/\b(\d{4}\.\d{4,5}(?:v\d+)?)\b/i);
+    const stop = new Set(["the", "and", "for", "with", "from", "paper", "papers", "arxiv", "research"]);
+    const words = clean
+        .split(/\s+/)
+        .map((w) => w.replace(/[^a-zA-Z0-9.-]/g, ""))
+        .filter((w) => w.length >= 2 && !stop.has(w.toLowerCase()))
+        .slice(0, 6);
+    const searchExpr = arxivIdMatch
+        ? `id:${encodeURIComponent(arxivIdMatch[1])}`
+        : words.length <= 1
+            ? `all:${encodeURIComponent(words[0] || clean)}`
+            : words.map((w) => `all:${encodeURIComponent(w)}`).join("+AND+");
+    const arxivPromise = (async () => {
+        try {
+            const url = `https://export.arxiv.org/api/query?search_query=${searchExpr}&start=0&max_results=${Math.max(limit, 6)}`;
+            const res = await fetchImpl(url, {
+                headers: { "user-agent": "webcrawl-anything/1.2 (https://webcrawl.app)" },
+                signal: AbortSignal.timeout(4000),
+            });
+            if (!res.ok) {
+                return [];
+            }
+            const xml = await res.text();
+            const entries = xml.match(/<entry>([\s\S]*?)<\/entry>/gi) || [];
+            const out = [];
+            for (const entry of entries) {
+                if (out.length >= limit) {
+                    break;
+                }
+                const title = decodeXmlEntities((entry.match(/<title>([\s\S]*?)<\/title>/i)?.[1] || "").replace(/\s+/g, " ").trim());
+                const rawIdUrl = (entry.match(/<id>([\s\S]*?)<\/id>/i)?.[1] || "").trim();
+                const idUrl = rawIdUrl.replace(/^http:\/\/arxiv\.org\//i, "https://arxiv.org/");
+                const summary = decodeXmlEntities((entry.match(/<summary>([\s\S]*?)<\/summary>/i)?.[1] || "").replace(/\s+/g, " ").trim());
+                const published = (entry.match(/<published>([\s\S]*?)<\/published>/i)?.[1] || "").trim();
+                const authors = Array.from(entry.matchAll(/<author>[\s\S]*?<name>([\s\S]*?)<\/name>[\s\S]*?<\/author>/gi))
+                    .map((m) => decodeXmlEntities(m[1].trim()))
+                    .filter(Boolean)
+                    .slice(0, 4);
+                const pdfUrl = (entry.match(/<link[^>]+title=["']pdf["'][^>]+href=["']([^"']+)["']/i)?.[1] ||
+                    idUrl.replace("/abs/", "/pdf/")).replace(/^http:\/\//i, "https://");
+                if (title && idUrl) {
+                    const metaPrefix = [
+                        authors.length ? `Authors: ${authors.join(", ")}` : "",
+                        pdfUrl ? `PDF: ${pdfUrl}` : "",
+                    ]
+                        .filter(Boolean)
+                        .join(" · ");
+                    out.push({
+                        title: `[arXiv] ${title}`,
+                        url: idUrl,
+                        description: (metaPrefix ? `${metaPrefix} — ${summary}` : summary).slice(0, 460),
+                        publishedDate: published || undefined,
+                        sources: ["arxiv"],
+                    });
+                }
+            }
+            return out;
+        }
+        catch {
+            return [];
+        }
+    })();
+    const openAlexPromise = (async () => {
+        try {
+            const url = `https://api.openalex.org/works?search=${encodeURIComponent(clean)}&per-page=${Math.min(limit, 6)}`;
+            const res = await fetchImpl(url, {
+                headers: { "user-agent": "webcrawl-anything/1.2 (mailto:support@webcrawl.app)" },
+                signal: AbortSignal.timeout(3200),
+            });
+            if (!res.ok) {
+                return [];
+            }
+            const data = (await res.json());
+            const out = [];
+            for (const w of data.results || []) {
+                if (out.length >= limit) {
+                    break;
+                }
+                const title = (w.title || "").replace(/\s+/g, " ").trim();
+                const targetUrl = w.primary_location?.landing_page_url || w.open_access?.oa_url || w.doi || "";
+                if (!title || !targetUrl) {
+                    continue;
+                }
+                const authors = (w.authorships || [])
+                    .map((a) => a.author?.display_name)
+                    .filter((n) => Boolean(n))
+                    .slice(0, 4)
+                    .join(", ");
+                const venue = w.primary_location?.source?.display_name;
+                const pdfUrl = w.primary_location?.pdf_url || w.open_access?.oa_url;
+                const desc = [
+                    authors ? `Authors: ${authors}` : "",
+                    venue ? `Venue: ${venue}` : "",
+                    typeof w.cited_by_count === "number" ? `Citations: ${w.cited_by_count}` : "",
+                    pdfUrl ? `PDF: ${pdfUrl}` : "",
+                ]
+                    .filter(Boolean)
+                    .join(" · ");
+                out.push({
+                    title: `[Paper] ${title}`,
+                    url: targetUrl,
+                    description: desc || title,
+                    publishedDate: w.publication_date,
+                    sources: ["arxiv"],
+                });
+            }
+            return out;
+        }
+        catch {
+            return [];
+        }
+    })();
+    const [arxivItems, openAlexItems] = await Promise.all([arxivPromise, openAlexPromise]);
+    return [...arxivItems, ...openAlexItems];
+}
+const BUILTIN_CLIENT_TARGETS = [
+    {
+        id: "llms-txt",
+        name: "llms.txt & AI Documentation Indexes",
+        returns: "Structured sections, doc links, summaries, and full markdown manifest for LLM context ingestion",
+        requirements: "Built-in zero-key markdown manifest parser",
+    },
+    {
+        id: "github",
+        name: "GitHub Repositories, Issues & Releases",
+        returns: "Repository metadata, stars/forks, topics, language, issue/PR threads, release notes, and README markdown",
+        requirements: "Built-in zero-key HTML/JSON inspector via impit TLS fetch",
+    },
+    {
+        id: "arxiv",
+        name: "ArXiv & Academic Research Papers",
+        returns: "Paper title, authors, abstract, primary subject, DOI, submission date, and direct PDF link",
+        requirements: "Built-in ArXiv + OpenAlex paper search (-c academic) and abstract/PDF parser",
+    },
+    {
+        id: "youtube-media",
+        name: "YouTube Videos, Audio & Media Files",
+        returns: "Timestamped transcript Markdown, video title/channel/duration metadata, and spoken audio transcription",
+        requirements: "Built-in zero-key YouTube multi-client caption extractor",
+    },
+    {
+        id: "pdf-office",
+        name: "PDFs & Office Documents (DOCX, XLSX, PPTX)",
+        returns: "Page-numbered Markdown, tables, native text extraction, automatic OCR on scanned pages",
+        requirements: "Built-in (@firecrawl/pdf-inspector, pdf-parse, mammoth, officeparser, tesseract.js)",
+    },
+    {
+        id: "cve",
+        name: "Security Advisories & CVEs (NVD / OSV / CISA KEV)",
+        returns: "Structured CVERecord with CVSS v3.1/v4.0 score, severity, CWEs, affected/fixed versions, and CISA KEV status",
+        requirements: "Built-in live NVD 2.0 + OSV.dev + CISA KEV enrichment (-c security --sub cve)",
+    },
+    {
+        id: "shopping",
+        name: "Shopping & Multi-Retailer Product Offers",
+        returns: "AggregatedProduct & ProductOffer records with price, currency, stock availability, retailer, rating, and reviews",
+        requirements: "Zero-key local HTML/JSON-LD product extractor (-c shopping)",
+    },
+    {
+        id: "travel",
+        name: "Flights & Hotels",
+        returns: "Structured FlightOffer & HotelListing items with price, currency, airline/stops or star/guest rating",
+        requirements: "Zero-key local search + schema.org extractor (-c flights / -c hotels)",
+    },
+    {
+        id: "real-estate",
+        name: "Real Estate & Local Business Listings",
+        returns: "PropertyListing (bedrooms, bathrooms, parking, suburb) & BusinessListing (phone, address, rating, hours)",
+        requirements: "Zero-key local extractor (-c real-estate --sub buy|rent|sold / -c business)",
+    },
+    {
+        id: "web",
+        name: "Websites, Docs & Single-Page Apps (SPAs)",
+        returns: "Clean Markdown, title, metadata, canonical links, content hash, and evidence source tier",
+        requirements: "Zero-key local (impit TLS impersonation + local Chrome) or webcrawl.app cloud stealth browsers",
+    },
+];
+export function createClient(opts = {}) {
+    return new WebcrawlClient(opts);
+}

package/dist/config.d.ts

@@ -0,0 +1,27 @@
+export declare const DEFAULT_BASE_URL = "https://webcrawl.app";
+export type ExecutionMode = "local" | "cloud" | "auto";
+export interface FileConfig {
+    apiKey?: string;
+    baseUrl?: string;
+    mode?: ExecutionMode;
+}
+export interface ResolvedConfig {
+    apiKey?: string;
+    baseUrl: string;
+    mode: ExecutionMode;
+    effectiveMode: "local" | "cloud";
+    source: {
+        apiKey: "flag" | "env" | "file" | "none";
+        baseUrl: "flag" | "env" | "file" | "default";
+        mode: "flag" | "env" | "file" | "default";
+    };
+}
+export declare function configPath(env?: NodeJS.ProcessEnv): string;
+export declare function readConfigFile(path?: string): FileConfig;
+export declare function writeConfigFile(cfg: FileConfig, path?: string): void;
+/** Precedence: explicit flags > environment > config file > defaults. */
+export declare function resolveConfig(flags?: {
+    apiKey?: string;
+    baseUrl?: string;
+    mode?: string;
+}, env?: NodeJS.ProcessEnv, file?: FileConfig): ResolvedConfig;

package/dist/config.js

@@ -0,0 +1,58 @@
+import { chmodSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+export const DEFAULT_BASE_URL = "https://webcrawl.app";
+export function configPath(env = process.env) {
+    const base = env.XDG_CONFIG_HOME || join(homedir(), ".config");
+    return join(base, "webcrawl", "config.json");
+}
+export function readConfigFile(path = configPath()) {
+    try {
+        const parsed = JSON.parse(readFileSync(path, "utf8"));
+        if (parsed && typeof parsed === "object") {
+            const o = parsed;
+            return {
+                apiKey: typeof o.apiKey === "string" ? o.apiKey : undefined,
+                baseUrl: typeof o.baseUrl === "string" ? o.baseUrl : undefined,
+                mode: o.mode === "local" || o.mode === "cloud" || o.mode === "auto" ? o.mode : undefined,
+            };
+        }
+    }
+    catch {
+        // missing or unreadable config is fine
+    }
+    return {};
+}
+export function writeConfigFile(cfg, path = configPath()) {
+    mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+    writeFileSync(path, `${JSON.stringify(cfg, null, 2)}\n`, { mode: 0o600 });
+    chmodSync(path, 0o600);
+}
+const clean = (v) => (v && v.trim() ? v.trim() : undefined);
+/** Precedence: explicit flags > environment > config file > defaults. */
+export function resolveConfig(flags = {}, env = process.env, file = readConfigFile(configPath(env))) {
+    const pick = (f, e, fl, none) => {
+        if (clean(f)) {
+            return [clean(f), "flag"];
+        }
+        if (clean(e)) {
+            return [clean(e), "env"];
+        }
+        if (clean(fl)) {
+            return [clean(fl), "file"];
+        }
+        return [undefined, none];
+    };
+    const [apiKey, keySrc] = pick(flags.apiKey, env.WEBCRAWL_API_KEY, file.apiKey, "none");
+    const [baseUrl, urlSrc] = pick(flags.baseUrl, env.WEBCRAWL_URL, file.baseUrl, "default");
+    const [rawMode, modeSrc] = pick(flags.mode, env.WEBCRAWL_MODE, file.mode, "default");
+    const mode = rawMode === "local" || rawMode === "cloud" || rawMode === "auto" ? rawMode : "auto";
+    const effectiveMode = mode === "local" ? "local" : mode === "cloud" ? "cloud" : apiKey ? "cloud" : "local";
+    return {
+        apiKey,
+        baseUrl: (baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, ""),
+        mode,
+        effectiveMode,
+        source: { apiKey: keySrc, baseUrl: urlSrc, mode: modeSrc },
+    };
+}

package/dist/errors.d.ts

@@ -0,0 +1,16 @@
+export type WebcrawlErrorKind = "auth" | "credits" | "rate_limit" | "bad_request" | "not_found" | "server" | "network" | "timeout";
+export declare class WebcrawlError extends Error {
+    readonly kind: WebcrawlErrorKind;
+    readonly status?: number;
+    /** Seconds to wait before retrying, from `Retry-After` on 429. */
+    readonly retryAfter?: number;
+    readonly body?: unknown;
+    constructor(kind: WebcrawlErrorKind, message: string, opts?: {
+        status?: number;
+        retryAfter?: number;
+        body?: unknown;
+    });
+}
+export declare function kindForStatus(status: number): WebcrawlErrorKind;
+/** One actionable sentence for people (and models) reading the error. */
+export declare function describeError(err: unknown, baseUrl?: string): string;

package/dist/errors.js

@@ -0,0 +1,55 @@
+export class WebcrawlError extends Error {
+    kind;
+    status;
+    /** Seconds to wait before retrying, from `Retry-After` on 429. */
+    retryAfter;
+    body;
+    constructor(kind, message, opts = {}) {
+        super(message);
+        this.name = "WebcrawlError";
+        this.kind = kind;
+        this.status = opts.status;
+        this.retryAfter = opts.retryAfter;
+        this.body = opts.body;
+    }
+}
+export function kindForStatus(status) {
+    if (status === 401 || status === 403) {
+        return "auth";
+    }
+    if (status === 402) {
+        return "credits";
+    }
+    if (status === 429) {
+        return "rate_limit";
+    }
+    if (status === 404) {
+        return "not_found";
+    }
+    if (status >= 500) {
+        return "server";
+    }
+    return "bad_request";
+}
+/** One actionable sentence for people (and models) reading the error. */
+export function describeError(err, baseUrl = "https://webcrawl.app") {
+    if (!(err instanceof WebcrawlError)) {
+        return err instanceof Error ? err.message : String(err);
+    }
+    switch (err.kind) {
+        case "auth":
+            return "Your API key is missing or invalid. Set WEBCRAWL_API_KEY or run `webcrawl login`.";
+        case "credits":
+            return `You are out of credits. Add more at ${baseUrl}/account.`;
+        case "rate_limit":
+            return err.retryAfter
+                ? `Rate limited. Retry in ${err.retryAfter}s.`
+                : "Rate limited. Wait a moment and retry.";
+        case "timeout":
+            return "The request timed out. Try again, or narrow the request.";
+        case "network":
+            return `Could not reach ${baseUrl}. Check your connection or WEBCRAWL_URL.`;
+        default:
+            return err.message;
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export { WebcrawlClient, createClient, type ClientOptions } from "./client.js";
+export { DEFAULT_BASE_URL, configPath, readConfigFile, resolveConfig, writeConfigFile, type ExecutionMode, type FileConfig, type ResolvedConfig, } from "./config.js";
+export { WebcrawlError, describeError, kindForStatus, type WebcrawlErrorKind } from "./errors.js";
+export type * from "./types.js";

package/dist/index.js

@@ -0,0 +1,3 @@
+export { WebcrawlClient, createClient } from "./client.js";
+export { DEFAULT_BASE_URL, configPath, readConfigFile, resolveConfig, writeConfigFile, } from "./config.js";
+export { WebcrawlError, describeError, kindForStatus } from "./errors.js";

package/dist/types.d.ts

@@ -0,0 +1,126 @@
+/** Fields every API response may carry. Responses are intentionally loose: the API grows faster than this client. */
+export interface CreditInfo {
+    charged?: number;
+    used?: number;
+    remaining?: number;
+    limit?: number;
+}
+export interface ApiResponse {
+    success?: boolean;
+    credits?: CreditInfo;
+    [key: string]: unknown;
+}
+export interface Location {
+    country?: string;
+    region?: string;
+    city?: string;
+    timezone?: string;
+}
+export interface SearchParams extends Location {
+    query: string;
+    category?: string;
+    subcategory?: string;
+    limit?: number;
+    /** Restrict to these domains. */
+    sites?: string[];
+    includeDomains?: string[];
+    excludeDomains?: string[];
+    /** Time filter, e.g. `qdr:w`. */
+    tbs?: string;
+    scrapeContent?: boolean;
+}
+export interface SearchResponse extends ApiResponse {
+    results?: Array<{
+        title?: string;
+        url?: string;
+        description?: string;
+        markdown?: string;
+    }>;
+    data?: unknown;
+    offers?: unknown[];
+    products?: unknown[];
+    listings?: unknown[];
+    flights?: unknown[];
+    hotels?: unknown[];
+    businesses?: unknown[];
+    /** Structured CVE records (category "security"): id, cvssScore, severity, affected, fixed versions, kev, references. */
+    cves?: unknown[];
+}
+export interface ScrapeParams {
+    url: string;
+    formats?: Array<"markdown" | "links" | "html">;
+    noCache?: boolean;
+}
+/** Parse a file (PDF, Office, image, audio/video) from a URL, raw bytes or base64. */
+export interface ParseParams {
+    /** File URL. Provide one of `url`, `bytes` or `base64`. */
+    url?: string;
+    /** Raw file bytes (sent as base64 JSON). */
+    bytes?: Uint8Array;
+    base64?: string;
+    fileName?: string;
+    contentType?: string;
+    /** `auto` OCRs only scanned pages; `force` OCRs every page; `off` never OCRs. */
+    ocr?: "auto" | "force" | "off";
+    /** 1-indexed PDF pages to read. */
+    pages?: number[];
+    maxOcrPages?: number;
+    /** ISO-639-1 hint for transcription. */
+    language?: string;
+    /** WebVTT/SRT captions to use instead of transcribing. */
+    captions?: string;
+    describeVideo?: boolean;
+    /** Run as a durable job (returns a run id), for big PDFs and long media. */
+    async?: boolean;
+    noCache?: boolean;
+}
+export interface ParseMetadata {
+    kind?: string;
+    method?: "native" | "ocr" | "vision" | "transcript";
+    pages?: number;
+    pagesOcr?: number[];
+    pagesSkipped?: Array<{
+        page: number;
+        reason: string;
+    }>;
+    durationSeconds?: number;
+    warnings?: string[];
+}
+export interface ParseResponse extends ApiResponse {
+    title?: string;
+    markdown?: string;
+    /** Present when the parse runs as a durable job. */
+    id?: string;
+    status?: string;
+    statusUrl?: string;
+    metadata?: {
+        parse?: ParseMetadata;
+        [key: string]: unknown;
+    };
+}
+export interface MapParams {
+    url: string;
+    limit?: number;
+    search?: string;
+}
+export interface CrawlParams {
+    url: string;
+    maxPages?: number;
+    extractStructured?: boolean;
+}
+export interface ExtractParams {
+    url: string;
+    prompt?: string;
+    schema?: Record<string, unknown>;
+}
+export type RerankDocument = string | {
+    title?: string;
+    url?: string;
+    text?: string;
+    description?: string;
+};
+export interface RerankParams {
+    query: string;
+    documents: RerankDocument[];
+    topK?: number;
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,52 @@
 {
   "name": "@webcrawl/client",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "1.2.2",
+  "description": "Zero-dependency TypeScript & Node.js SDK for Webcrawl Anything (search, scrape, parse, map, crawl, extract, rerank, targets)",
+  "type": "module",
+  "license": "MIT",
+  "author": "Brainfish AI <support@webcrawl.app> (https://webcrawl.app)",
+  "homepage": "https://webcrawl.app",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/brainfish-ai/webcrawl-anything.git",
+    "directory": "packages/client"
+  },
+  "bugs": {
+    "url": "https://github.com/brainfish-ai/webcrawl-anything/issues"
+  },
+  "keywords": [
+    "webcrawl",
+    "web-crawler",
+    "web-scraper",
+    "search-api",
+    "markdown",
+    "pdf-parser",
+    "cve",
+    "mcp",
+    "ai-agents"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "development": "./src/index.ts",
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "prepack": "npm run build",
+    "test": "node --conditions=development --import tsx --test test/*.test.ts"
+  },
+  "engines": {
+    "node": ">=20"
+  }
+}