novada-proxy-core 0.0.1

package/build/index.js

@@ -0,0 +1,22 @@
+// ─── Config ──────────────────────────────────────────────────────────────────
+export { VERSION, NPM_PACKAGE } from "./config.js";
+// ─── Errors ──────────────────────────────────────────────────────────────────
+export { classifyError } from "./errors.js";
+export { resolveAdapter, listAdapters } from "./adapters/index.js";
+// ─── Tools ───────────────────────────────────────────────────────────────────
+export { novadaProxyFetch, validateFetchParams, getCacheTtl, makeCacheKey, clearResponseCache, } from "./tools/fetch.js";
+export { novadaProxyBatchFetch, validateBatchFetchParams } from "./tools/batch.js";
+export { novadaProxySearch, validateSearchParams } from "./tools/search.js";
+export { novadaProxySession, validateSessionParams } from "./tools/session.js";
+export { novadaProxyStatus } from "./tools/status.js";
+export { novadaProxyRender, validateRenderParams } from "./tools/render.js";
+export { novadaProxyExtract, validateExtractParams, extractField, deepFind, shouldEscalateToRender, } from "./tools/extract.js";
+export { novadaProxyMap, validateMapParams } from "./tools/map.js";
+export { novadaProxyCrawl, validateCrawlParams } from "./tools/crawl.js";
+export { novadaProxyResearch, validateResearchParams } from "./tools/research.js";
+// ─── Utils ───────────────────────────────────────────────────────────────────
+export { unicodeSafeTruncate, decodeHtmlEntities, htmlToMarkdown, htmlToText, stripNoiseElements, countHtmlTags, contentDensity, } from "./utils.js";
+// ─── Validation constants ─────────────────────────────────────────────────────
+export { SAFE_COUNTRY, SAFE_CITY, SAFE_SESSION_ID, QUOTA_NOTE } from "./validation.js";
+// ─── Redaction ───────────────────────────────────────────────────────────────
+export { redactCredentials } from "./redact.js";

package/build/redact.d.ts

@@ -0,0 +1,2 @@
+import type { ProxyAdapter, ProxyCredentials } from "./adapters/index.js";
+export declare function redactCredentials(message: string, adapter: ProxyAdapter, credentials: ProxyCredentials): string;

package/build/redact.js

@@ -0,0 +1,24 @@
+export function redactCredentials(message, adapter, credentials) {
+    let redacted = message;
+    for (const field of adapter.sensitiveFields) {
+        const val = credentials[field];
+        if (val) {
+            redacted = redacted.replaceAll(val, "***");
+            redacted = redacted.replaceAll(encodeURIComponent(val), "***");
+        }
+    }
+    const user = credentials["user"];
+    if (user) {
+        redacted = redacted.replaceAll(user, "***");
+        redacted = redacted.replaceAll(encodeURIComponent(user), "***");
+    }
+    // Also redact API keys that may appear in URLs
+    for (const envKey of ["NOVADA_API_KEY", "NOVADA_BROWSER_WS"]) {
+        const val = process.env[envKey];
+        if (val) {
+            redacted = redacted.replaceAll(val, "***");
+            redacted = redacted.replaceAll(encodeURIComponent(val), "***");
+        }
+    }
+    return redacted;
+}

package/build/tools/batch.d.ts

@@ -0,0 +1,24 @@
+import type { ProxyAdapter, ProxyCredentials } from "../adapters/index.js";
+export interface BatchFetchParams {
+    urls: string[];
+    country?: string;
+    session_id?: string;
+    format?: "markdown" | "raw";
+    timeout?: number;
+    concurrency?: number;
+}
+export interface BatchFetchResult {
+    url: string;
+    ok: boolean;
+    status_code?: number;
+    content?: string;
+    size_bytes?: number;
+    cache_hit?: boolean;
+    error?: {
+        code: string;
+        message: string;
+    };
+    latency_ms: number;
+}
+export declare function novadaProxyBatchFetch(params: BatchFetchParams, adapter: ProxyAdapter, credentials: ProxyCredentials): Promise<string>;
+export declare function validateBatchFetchParams(raw: Record<string, unknown>): BatchFetchParams;

package/build/tools/batch.js

@@ -0,0 +1,156 @@
+import { novadaProxyFetch } from "./fetch.js";
+import { SAFE_COUNTRY, SAFE_SESSION_ID, QUOTA_NOTE } from "../validation.js";
+export async function novadaProxyBatchFetch(params, adapter, credentials) {
+    const { urls, country, session_id, format = "markdown", timeout = 60, concurrency = 3, } = params;
+    const wallStart = Date.now();
+    // Semaphore-based concurrency control
+    let activeCount = 0;
+    const queue = [];
+    function acquire() {
+        return new Promise((resolve) => {
+            if (activeCount < concurrency) {
+                activeCount++;
+                resolve();
+            }
+            else {
+                queue.push(() => {
+                    activeCount++;
+                    resolve();
+                });
+            }
+        });
+    }
+    function release() {
+        activeCount--;
+        const next = queue.shift();
+        if (next)
+            next();
+    }
+    const fetchOne = async (url) => {
+        const start = Date.now();
+        await acquire();
+        try {
+            const rawResult = await novadaProxyFetch({ url, country, session_id, format, timeout }, adapter, credentials);
+            const parsed = JSON.parse(rawResult);
+            const latency_ms = Date.now() - start;
+            return {
+                url,
+                ok: true,
+                status_code: parsed.data.status_code,
+                content: parsed.data.content,
+                size_bytes: parsed.data.size_bytes,
+                cache_hit: parsed.meta.cache_hit,
+                latency_ms,
+            };
+        }
+        catch (err) {
+            const latency_ms = Date.now() - start;
+            let msg = err instanceof Error ? err.message : String(err);
+            // Redact credentials from per-URL error messages (proxy URLs may leak in TLS errors)
+            for (const field of adapter.sensitiveFields) {
+                const val = credentials[field];
+                if (val) {
+                    msg = msg.replaceAll(val, "***");
+                    msg = msg.replaceAll(encodeURIComponent(val), "***");
+                }
+            }
+            const user = credentials["user"];
+            if (user) {
+                msg = msg.replaceAll(user, "***");
+                msg = msg.replaceAll(encodeURIComponent(user), "***");
+            }
+            // Infer error code from message
+            let code = "UNKNOWN_ERROR";
+            if (msg.includes("429") || msg.includes("rate limit"))
+                code = "RATE_LIMITED";
+            else if (msg.includes("timeout") || msg.includes("ECONNABORTED"))
+                code = "TIMEOUT";
+            else if (msg.includes("TLS") || msg.includes("SSL"))
+                code = "TLS_ERROR";
+            else if (msg.includes("blocked") || msg.includes("403") || msg.includes("401"))
+                code = "BOT_DETECTION_SUSPECTED";
+            else if (msg.includes("must start with"))
+                code = "INVALID_INPUT";
+            return {
+                url,
+                ok: false,
+                error: { code, message: msg },
+                latency_ms,
+            };
+        }
+        finally {
+            release();
+        }
+    };
+    const results = await Promise.all(urls.map(fetchOne));
+    const wallLatency = Date.now() - wallStart;
+    const succeeded = results.filter(r => r.ok).length;
+    const failed = results.filter(r => !r.ok).length;
+    const cachedCount = results.filter(r => r.ok && r.cache_hit).length;
+    const result = {
+        ok: true,
+        tool: "novada_proxy_batch_fetch",
+        data: {
+            requested: urls.length,
+            succeeded,
+            failed,
+            results: results,
+        },
+        meta: {
+            latency_ms: wallLatency,
+            concurrency,
+            country,
+            quota: {
+                credits_estimated: urls.length - cachedCount,
+                note: QUOTA_NOTE,
+            },
+        },
+    };
+    if (!result.meta.country)
+        delete result.meta.country;
+    return JSON.stringify(result);
+}
+export function validateBatchFetchParams(raw) {
+    if (!raw.urls || !Array.isArray(raw.urls)) {
+        throw new Error("urls is required and must be an array");
+    }
+    if (raw.urls.length < 2 || raw.urls.length > 20) {
+        throw new Error("urls must contain between 2 and 20 URLs");
+    }
+    for (const u of raw.urls) {
+        if (typeof u !== "string")
+            throw new Error("each url must be a string");
+        // Note: URL format is intentionally NOT validated here — invalid URLs are captured
+        // as per-item errors in data.results rather than failing the whole batch.
+        // This allows agents to pass mixed lists and inspect failures per-URL.
+    }
+    if (raw.country !== undefined) {
+        if (typeof raw.country !== "string" || raw.country.length > 10 || !SAFE_COUNTRY.test(raw.country)) {
+            throw new Error("country must be a 2-letter ISO code with no hyphens (e.g. US, DE, GB)");
+        }
+    }
+    if (raw.session_id !== undefined) {
+        if (typeof raw.session_id !== "string" || raw.session_id.length > 64 || !SAFE_SESSION_ID.test(raw.session_id)) {
+            throw new Error("session_id must contain only letters, numbers, and underscores, max 64 chars (no hyphens)");
+        }
+    }
+    if (raw.format && raw.format !== "raw" && raw.format !== "markdown") {
+        throw new Error("format must be 'raw' or 'markdown'");
+    }
+    const timeout = raw.timeout !== undefined ? Number(raw.timeout) : 60;
+    if (!Number.isFinite(timeout) || timeout < 1 || timeout > 120) {
+        throw new Error("timeout must be between 1 and 120 seconds");
+    }
+    const concurrency = raw.concurrency !== undefined ? Number(raw.concurrency) : 3;
+    if (!Number.isFinite(concurrency) || concurrency < 1 || concurrency > 5) {
+        throw new Error("concurrency must be between 1 and 5");
+    }
+    return {
+        urls: raw.urls,
+        country: raw.country,
+        session_id: raw.session_id,
+        format: raw.format || "markdown",
+        timeout,
+        concurrency,
+    };
+}

package/build/tools/crawl.d.ts

@@ -0,0 +1,33 @@
+import type { ProxyAdapter, ProxyCredentials } from "../adapters/index.js";
+export interface CrawlParams {
+    url: string;
+    depth?: number;
+    limit?: number;
+    max_pages?: number;
+    max_depth?: number;
+    include_content?: boolean;
+    include_patterns?: string[];
+    exclude_patterns?: string[];
+    render?: "none" | "render" | "browser";
+    output_format?: "markdown" | "html" | "text";
+    extract_fields?: string[];
+    country?: string;
+    timeout?: number;
+    format?: "markdown" | "raw";
+    rate_limit?: number;
+    browser_ws?: string;
+}
+export interface CrawlPageResult {
+    url: string;
+    depth: number;
+    title?: string;
+    status_code?: number;
+    total_links: number;
+    new_links: number;
+    content?: string;
+    extracted_fields?: Record<string, string | null>;
+    links_found?: string[];
+    error?: string;
+}
+export declare function novadaProxyCrawl(params: CrawlParams, adapter: ProxyAdapter, credentials: ProxyCredentials): Promise<string>;
+export declare function validateCrawlParams(raw: Record<string, unknown>): CrawlParams;