@webcrawl/client 1.2.0 → 1.2.2

package/dist/client.d.ts

@@ -14,6 +14,7 @@ export declare class WebcrawlClient {
     private readonly apiKey?;
     private readonly timeoutMs;
     private readonly fetchImpl;
+    private readonly hasCustomFetch;
     private readonly userAgent;
     private readonly timezone?;
     constructor(opts?: ClientOptions);
@@ -31,5 +32,49 @@ export declare class WebcrawlClient {
     categories(): Promise<ApiResponse>;
     targets(): Promise<ApiResponse>;
     usage(): Promise<ApiResponse>;
+    /** Send a 6-digit OTP verification code to an email address to sign up or log in (provisions 1,000 free monthly credits). */
+    sendOtp(email: string): Promise<{
+        success: boolean;
+        email: string;
+        expiresInSeconds: number;
+        transport?: string;
+        devCode?: string;
+        error?: string;
+    }>;
+    /** Verify a 6-digit OTP code, creating the user account (with 1,000 monthly credits) if new, and return a `wc_live_...` API key. */
+    verifyOtp(params: {
+        email: string;
+        code: string;
+        name?: string;
+    }): Promise<{
+        success: boolean;
+        isNewUser?: boolean;
+        apiKey?: string;
+        sessionToken?: string;
+        user?: {
+            id: string;
+            email: string;
+            name?: string | null;
+            plan: string;
+            creditsLimit: number;
+            creditsUsed: number;
+            creditsRemaining: number;
+        };
+        error?: string;
+    }>;
+    /** Fetch authenticated user profile & remaining credit balance. */
+    me(): Promise<{
+        success: boolean;
+        authenticated: boolean;
+        user?: {
+            id: string;
+            email: string;
+            name?: string | null;
+            plan: string;
+            creditsLimit: number;
+            creditsUsed: number;
+            creditsRemaining: number;
+        };
+    }>;
 }
 export declare function createClient(opts?: ClientOptions): WebcrawlClient;

package/dist/client.js

@@ -24,6 +24,7 @@ export class WebcrawlClient {
     apiKey;
     timeoutMs;
     fetchImpl;
+    hasCustomFetch;
     userAgent;
     timezone;
     constructor(opts = {}) {
@@ -31,6 +32,7 @@ export class WebcrawlClient {
         this.apiKey = opts.apiKey;
         this.timeoutMs = opts.timeoutMs ?? 60_000;
         this.fetchImpl = opts.fetch ?? globalThis.fetch;
+        this.hasCustomFetch = Boolean(opts.fetch);
         this.userAgent = opts.userAgent ?? "webcrawl-client/0.1";
         this.timezone = opts.timezone ?? localTimezone();
     }
@@ -79,13 +81,66 @@ export class WebcrawlClient {
         }
         return json;
     }
-    search(p) {
+    async search(p) {
         const { sites, scrapeContent, ...rest } = p;
-        return this.request("POST", "/v1/search", {
-            ...compact(rest),
+        const rawCat = (p.category || "").trim().toLowerCase();
+        const rawSub = (p.subcategory || "").trim().toLowerCase();
+        const isAcademic = rawCat === "academic" ||
+            rawCat === "paper" ||
+            rawCat === "papers" ||
+            rawCat === "research" ||
+            rawCat === "arxiv" ||
+            rawCat === "scholar" ||
+            rawCat === "scholarly" ||
+            rawSub === "arxiv" ||
+            rawSub === "papers" ||
+            rawSub === "ml" ||
+            rawSub === "pubmed" ||
+            rawSub === "math";
+        const proxyPromise = this.request("POST", "/v1/search", {
+            ...compact({ ...rest, ...(isAcademic ? { category: "academic" } : {}) }),
             ...(sites?.length ? { filters: { sites } } : {}),
             ...(scrapeContent ? { scrapeOptions: { formats: ["markdown"] } } : {}),
         });
+        if (!isAcademic || this.hasCustomFetch) {
+            return proxyPromise;
+        }
+        const limit = p.limit ?? 5;
+        if (!this.apiKey) {
+            const scholarlyItems = await fetchAcademicPapersForClient(this.fetchImpl, p.query, limit);
+            return {
+                success: true,
+                category: "academic",
+                results: scholarlyItems.slice(0, limit),
+            };
+        }
+        const [proxyRes, scholarlyItems] = await Promise.all([
+            proxyPromise.catch(() => ({ success: true, results: [] })),
+            fetchAcademicPapersForClient(this.fetchImpl, p.query, limit),
+        ]);
+        if (!scholarlyItems.length) {
+            return proxyRes;
+        }
+        const existing = Array.isArray(proxyRes.results) ? proxyRes.results : [];
+        const merged = [];
+        const seen = new Set();
+        for (const item of [...scholarlyItems, ...existing]) {
+            const url = typeof item.url === "string" ? item.url.toLowerCase().replace(/v\d+$/, "") : "";
+            if (!url || seen.has(url)) {
+                continue;
+            }
+            seen.add(url);
+            merged.push(item);
+            if (merged.length >= limit) {
+                break;
+            }
+        }
+        return {
+            ...proxyRes,
+            success: true,
+            category: "academic",
+            results: merged,
+        };
     }
     scrape(p) {
         return this.request("POST", "/v1/scrape", compact({ url: p.url, formats: p.formats, noCache: p.noCache }));
@@ -118,13 +173,248 @@ export class WebcrawlClient {
     categories() {
         return this.request("GET", "/v1/search/categories");
     }
-    targets() {
-        return this.request("GET", "/v1/targets");
+    async targets() {
+        try {
+            return await this.request("GET", "/v1/targets");
+        }
+        catch (e) {
+            if (!this.hasCustomFetch && e instanceof WebcrawlError && (e.status === 404 || e.status === 405)) {
+                return { success: true, count: BUILTIN_CLIENT_TARGETS.length, targets: BUILTIN_CLIENT_TARGETS };
+            }
+            throw e;
+        }
     }
     usage() {
         return this.request("GET", "/v1/account/usage");
     }
+    /** Send a 6-digit OTP verification code to an email address to sign up or log in (provisions 1,000 free monthly credits). */
+    sendOtp(email) {
+        return this.request("POST", "/v1/auth/otp/send", { email: email.trim() });
+    }
+    /** Verify a 6-digit OTP code, creating the user account (with 1,000 monthly credits) if new, and return a `wc_live_...` API key. */
+    async verifyOtp(params) {
+        const res = await this.request("POST", "/v1/auth/otp/verify", {
+            email: params.email.trim(),
+            code: params.code.trim(),
+            ...(params.name ? { name: params.name.trim() } : {}),
+        });
+        if (res.success && !res.apiKey && res.sessionToken) {
+            try {
+                const sessionClient = new WebcrawlClient({
+                    baseUrl: this.baseUrl,
+                    apiKey: res.sessionToken,
+                    fetch: this.fetchImpl,
+                    userAgent: this.userAgent,
+                });
+                const keyRes = await sessionClient.request("POST", "/v1/account/keys", { name: "CLI API Key" });
+                if (keyRes.apiKey) {
+                    res.apiKey = keyRes.apiKey;
+                }
+            }
+            catch {
+                // Fallback to sessionToken if key creation fails
+                res.apiKey = res.sessionToken;
+            }
+        }
+        return res;
+    }
+    /** Fetch authenticated user profile & remaining credit balance. */
+    me() {
+        return this.request("GET", "/v1/auth/me");
+    }
+}
+const decodeXmlEntities = (s) => s
+    .replace(/&/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'");
+async function fetchAcademicPapersForClient(fetchImpl, rawQuery, limit) {
+    const clean = rawQuery
+        .replace(/\([^)]*\)|-?site:\S+/g, " ")
+        .replace(/["'`]+/g, " ")
+        .replace(/\s+/g, " ")
+        .trim();
+    if (!clean) {
+        return [];
+    }
+    const arxivIdMatch = clean.match(/\b(\d{4}\.\d{4,5}(?:v\d+)?)\b/i);
+    const stop = new Set(["the", "and", "for", "with", "from", "paper", "papers", "arxiv", "research"]);
+    const words = clean
+        .split(/\s+/)
+        .map((w) => w.replace(/[^a-zA-Z0-9.-]/g, ""))
+        .filter((w) => w.length >= 2 && !stop.has(w.toLowerCase()))
+        .slice(0, 6);
+    const searchExpr = arxivIdMatch
+        ? `id:${encodeURIComponent(arxivIdMatch[1])}`
+        : words.length <= 1
+            ? `all:${encodeURIComponent(words[0] || clean)}`
+            : words.map((w) => `all:${encodeURIComponent(w)}`).join("+AND+");
+    const arxivPromise = (async () => {
+        try {
+            const url = `https://export.arxiv.org/api/query?search_query=${searchExpr}&start=0&max_results=${Math.max(limit, 6)}`;
+            const res = await fetchImpl(url, {
+                headers: { "user-agent": "webcrawl-anything/1.2 (https://webcrawl.app)" },
+                signal: AbortSignal.timeout(4000),
+            });
+            if (!res.ok) {
+                return [];
+            }
+            const xml = await res.text();
+            const entries = xml.match(/<entry>([\s\S]*?)<\/entry>/gi) || [];
+            const out = [];
+            for (const entry of entries) {
+                if (out.length >= limit) {
+                    break;
+                }
+                const title = decodeXmlEntities((entry.match(/<title>([\s\S]*?)<\/title>/i)?.[1] || "").replace(/\s+/g, " ").trim());
+                const rawIdUrl = (entry.match(/<id>([\s\S]*?)<\/id>/i)?.[1] || "").trim();
+                const idUrl = rawIdUrl.replace(/^http:\/\/arxiv\.org\//i, "https://arxiv.org/");
+                const summary = decodeXmlEntities((entry.match(/<summary>([\s\S]*?)<\/summary>/i)?.[1] || "").replace(/\s+/g, " ").trim());
+                const published = (entry.match(/<published>([\s\S]*?)<\/published>/i)?.[1] || "").trim();
+                const authors = Array.from(entry.matchAll(/<author>[\s\S]*?<name>([\s\S]*?)<\/name>[\s\S]*?<\/author>/gi))
+                    .map((m) => decodeXmlEntities(m[1].trim()))
+                    .filter(Boolean)
+                    .slice(0, 4);
+                const pdfUrl = (entry.match(/<link[^>]+title=["']pdf["'][^>]+href=["']([^"']+)["']/i)?.[1] ||
+                    idUrl.replace("/abs/", "/pdf/")).replace(/^http:\/\//i, "https://");
+                if (title && idUrl) {
+                    const metaPrefix = [
+                        authors.length ? `Authors: ${authors.join(", ")}` : "",
+                        pdfUrl ? `PDF: ${pdfUrl}` : "",
+                    ]
+                        .filter(Boolean)
+                        .join(" · ");
+                    out.push({
+                        title: `[arXiv] ${title}`,
+                        url: idUrl,
+                        description: (metaPrefix ? `${metaPrefix} — ${summary}` : summary).slice(0, 460),
+                        publishedDate: published || undefined,
+                        sources: ["arxiv"],
+                    });
+                }
+            }
+            return out;
+        }
+        catch {
+            return [];
+        }
+    })();
+    const openAlexPromise = (async () => {
+        try {
+            const url = `https://api.openalex.org/works?search=${encodeURIComponent(clean)}&per-page=${Math.min(limit, 6)}`;
+            const res = await fetchImpl(url, {
+                headers: { "user-agent": "webcrawl-anything/1.2 (mailto:support@webcrawl.app)" },
+                signal: AbortSignal.timeout(3200),
+            });
+            if (!res.ok) {
+                return [];
+            }
+            const data = (await res.json());
+            const out = [];
+            for (const w of data.results || []) {
+                if (out.length >= limit) {
+                    break;
+                }
+                const title = (w.title || "").replace(/\s+/g, " ").trim();
+                const targetUrl = w.primary_location?.landing_page_url || w.open_access?.oa_url || w.doi || "";
+                if (!title || !targetUrl) {
+                    continue;
+                }
+                const authors = (w.authorships || [])
+                    .map((a) => a.author?.display_name)
+                    .filter((n) => Boolean(n))
+                    .slice(0, 4)
+                    .join(", ");
+                const venue = w.primary_location?.source?.display_name;
+                const pdfUrl = w.primary_location?.pdf_url || w.open_access?.oa_url;
+                const desc = [
+                    authors ? `Authors: ${authors}` : "",
+                    venue ? `Venue: ${venue}` : "",
+                    typeof w.cited_by_count === "number" ? `Citations: ${w.cited_by_count}` : "",
+                    pdfUrl ? `PDF: ${pdfUrl}` : "",
+                ]
+                    .filter(Boolean)
+                    .join(" · ");
+                out.push({
+                    title: `[Paper] ${title}`,
+                    url: targetUrl,
+                    description: desc || title,
+                    publishedDate: w.publication_date,
+                    sources: ["arxiv"],
+                });
+            }
+            return out;
+        }
+        catch {
+            return [];
+        }
+    })();
+    const [arxivItems, openAlexItems] = await Promise.all([arxivPromise, openAlexPromise]);
+    return [...arxivItems, ...openAlexItems];
 }
+const BUILTIN_CLIENT_TARGETS = [
+    {
+        id: "llms-txt",
+        name: "llms.txt & AI Documentation Indexes",
+        returns: "Structured sections, doc links, summaries, and full markdown manifest for LLM context ingestion",
+        requirements: "Built-in zero-key markdown manifest parser",
+    },
+    {
+        id: "github",
+        name: "GitHub Repositories, Issues & Releases",
+        returns: "Repository metadata, stars/forks, topics, language, issue/PR threads, release notes, and README markdown",
+        requirements: "Built-in zero-key HTML/JSON inspector via impit TLS fetch",
+    },
+    {
+        id: "arxiv",
+        name: "ArXiv & Academic Research Papers",
+        returns: "Paper title, authors, abstract, primary subject, DOI, submission date, and direct PDF link",
+        requirements: "Built-in ArXiv + OpenAlex paper search (-c academic) and abstract/PDF parser",
+    },
+    {
+        id: "youtube-media",
+        name: "YouTube Videos, Audio & Media Files",
+        returns: "Timestamped transcript Markdown, video title/channel/duration metadata, and spoken audio transcription",
+        requirements: "Built-in zero-key YouTube multi-client caption extractor",
+    },
+    {
+        id: "pdf-office",
+        name: "PDFs & Office Documents (DOCX, XLSX, PPTX)",
+        returns: "Page-numbered Markdown, tables, native text extraction, automatic OCR on scanned pages",
+        requirements: "Built-in (@firecrawl/pdf-inspector, pdf-parse, mammoth, officeparser, tesseract.js)",
+    },
+    {
+        id: "cve",
+        name: "Security Advisories & CVEs (NVD / OSV / CISA KEV)",
+        returns: "Structured CVERecord with CVSS v3.1/v4.0 score, severity, CWEs, affected/fixed versions, and CISA KEV status",
+        requirements: "Built-in live NVD 2.0 + OSV.dev + CISA KEV enrichment (-c security --sub cve)",
+    },
+    {
+        id: "shopping",
+        name: "Shopping & Multi-Retailer Product Offers",
+        returns: "AggregatedProduct & ProductOffer records with price, currency, stock availability, retailer, rating, and reviews",
+        requirements: "Zero-key local HTML/JSON-LD product extractor (-c shopping)",
+    },
+    {
+        id: "travel",
+        name: "Flights & Hotels",
+        returns: "Structured FlightOffer & HotelListing items with price, currency, airline/stops or star/guest rating",
+        requirements: "Zero-key local search + schema.org extractor (-c flights / -c hotels)",
+    },
+    {
+        id: "real-estate",
+        name: "Real Estate & Local Business Listings",
+        returns: "PropertyListing (bedrooms, bathrooms, parking, suburb) & BusinessListing (phone, address, rating, hours)",
+        requirements: "Zero-key local extractor (-c real-estate --sub buy|rent|sold / -c business)",
+    },
+    {
+        id: "web",
+        name: "Websites, Docs & Single-Page Apps (SPAs)",
+        returns: "Clean Markdown, title, metadata, canonical links, content hash, and evidence source tier",
+        requirements: "Zero-key local (impit TLS impersonation + local Chrome) or webcrawl.app cloud stealth browsers",
+    },
+];
 export function createClient(opts = {}) {
     return new WebcrawlClient(opts);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@webcrawl/client",
-  "version": "1.2.0",
+  "version": "1.2.2",
   "description": "Zero-dependency TypeScript & Node.js SDK for Webcrawl Anything (search, scrape, parse, map, crawl, extract, rerank, targets)",
   "type": "module",
   "license": "MIT",