novada-proxy-core 0.0.1

package/build/tools/search.d.ts

@@ -0,0 +1,9 @@
+export interface SearchParams {
+    query: string;
+    engine?: "google";
+    num?: number;
+    country?: string;
+    language?: string;
+}
+export declare function novadaProxySearch(params: SearchParams, novadaApiKey: string): Promise<string>;
+export declare function validateSearchParams(raw: Record<string, unknown>): SearchParams;

package/build/tools/search.js

@@ -0,0 +1,104 @@
+import axios from "axios";
+import { NOVADA_SEARCH_URL, DEFAULT_USER_AGENT } from "../config.js";
+import { QUOTA_NOTE } from "../validation.js";
+const SAFE_LOCALE = /^[a-zA-Z0-9_-]{1,10}$/;
+export async function novadaProxySearch(params, novadaApiKey) {
+    const { query, engine = "google", num = 10, country = "", language = "" } = params;
+    // Guard: validate locale params even when called directly (not via validateSearchParams)
+    if (country && !SAFE_LOCALE.test(country))
+        throw new Error("country contains invalid characters");
+    if (language && !SAFE_LOCALE.test(language))
+        throw new Error("language contains invalid characters");
+    // Note: Novada Scraper API authenticates via query param (api_key), not header.
+    // The key is therefore visible in server-side access logs — this is an API design
+    // constraint of the current Novada endpoint. We mitigate by never including the
+    // key in error messages surfaced to the agent (see sanitizeMessage below).
+    const searchParams = new URLSearchParams({
+        q: query,
+        api_key: novadaApiKey,
+        engine,
+        num: String(num),
+    });
+    if (country)
+        searchParams.set("country", country);
+    if (language)
+        searchParams.set("language", language);
+    const requestUrl = `${NOVADA_SEARCH_URL}?${searchParams.toString()}`;
+    const startTime = Date.now();
+    let response;
+    try {
+        response = await axios.get(requestUrl, {
+            headers: {
+                "User-Agent": DEFAULT_USER_AGENT,
+                Origin: "https://www.novada.com",
+                Referer: "https://www.novada.com/",
+            },
+            timeout: 30000,
+        });
+    }
+    catch (err) {
+        // Sanitize: never surface the request URL (contains api_key) in error messages
+        // Sanitize api_key from all error paths — it's embedded in the request URL
+        const sanitize = (s) => s.replaceAll(novadaApiKey, "***")
+            .replaceAll(encodeURIComponent(novadaApiKey), "***");
+        if (axios.isAxiosError(err)) {
+            const status = err.response?.status;
+            const msg = sanitize(String(err.response?.data?.msg || err.message));
+            throw new Error(status ? `Search API HTTP ${status}: ${msg}` : `Search API error: ${msg}`);
+        }
+        throw new Error(sanitize(String(err instanceof Error ? err.message : err)));
+    }
+    const latency_ms = Date.now() - startTime;
+    const data = response.data;
+    if (data.code && data.code !== 200 && data.code !== 0) {
+        throw new Error(`Novada search error (${data.code}): ${String(data.msg || "unknown")}`);
+    }
+    const rawResults = data.data?.organic_results || data.organic_results || data.data?.results || data.results || [];
+    const results = rawResults.slice(0, num).map(r => ({
+        title: r.title || "Untitled",
+        url: r.redirection_link || r.url || r.link || "",
+        snippet: r.description || r.snippet || "",
+    }));
+    const result = {
+        ok: true,
+        tool: "novada_proxy_search",
+        data: {
+            query,
+            engine,
+            count: results.length,
+            results,
+        },
+        meta: {
+            latency_ms,
+            quota: { credits_estimated: 1, note: QUOTA_NOTE },
+        },
+    };
+    return JSON.stringify(result);
+}
+export function validateSearchParams(raw) {
+    if (!raw.query || typeof raw.query !== "string") {
+        throw new Error("query is required");
+    }
+    if (raw.query.length > 500) {
+        throw new Error("query must be 500 characters or less");
+    }
+    if (raw.engine && raw.engine !== "google") {
+        throw new Error("engine must be 'google' — other engines have known quality issues");
+    }
+    const num = raw.num !== undefined ? Number(raw.num) : 10;
+    if (!Number.isFinite(num) || num < 1 || num > 20)
+        throw new Error("num must be between 1 and 20");
+    if (raw.country && (typeof raw.country !== "string" || !SAFE_LOCALE.test(raw.country))) {
+        throw new Error("country must be a short locale code (e.g. us, uk, de)");
+    }
+    if (raw.language && (typeof raw.language !== "string" || !SAFE_LOCALE.test(raw.language))) {
+        throw new Error("language must be a short language code (e.g. en, zh, de)");
+    }
+    return {
+        query: raw.query,
+        engine: raw.engine || "google",
+        num,
+        country: raw.country || "",
+        language: raw.language || "",
+    };
+}

package/build/tools/session.d.ts

@@ -0,0 +1,12 @@
+import type { ProxyAdapter, ProxyCredentials } from "../adapters/index.js";
+export interface SessionParams {
+    session_id: string;
+    url: string;
+    country?: string;
+    city?: string;
+    format?: "raw" | "markdown";
+    timeout?: number;
+    verify_sticky?: boolean;
+}
+export declare function novadaProxySession(params: SessionParams, adapter: ProxyAdapter, credentials: ProxyCredentials): Promise<string>;
+export declare function validateSessionParams(raw: Record<string, unknown>): SessionParams;

package/build/tools/session.js

@@ -0,0 +1,108 @@
+import axios from "axios";
+import { HttpsProxyAgent } from "https-proxy-agent";
+import { novadaProxyFetch } from "./fetch.js";
+import { SAFE_COUNTRY, SAFE_CITY, SAFE_SESSION_ID, QUOTA_NOTE } from "../validation.js";
+export async function novadaProxySession(params, adapter, credentials) {
+    const { verify_sticky = false } = params;
+    // Make the main fetch call
+    const fetchResultStr = await novadaProxyFetch({
+        url: params.url,
+        session_id: params.session_id,
+        country: params.country,
+        city: params.city,
+        format: params.format || "markdown",
+        timeout: params.timeout,
+    }, adapter, credentials);
+    // Parse the fetch result JSON
+    let fetchResult;
+    try {
+        fetchResult = JSON.parse(fetchResultStr);
+    }
+    catch {
+        // Fallback: return raw fetch result if JSON parsing fails
+        return fetchResultStr;
+    }
+    // If verify_sticky requested, make a second call to httpbin.org/ip with same session
+    let session_verified;
+    if (verify_sticky && adapter.capabilities.sticky) {
+        try {
+            const proxyUrl = adapter.buildProxyUrl(credentials, {
+                session_id: params.session_id,
+                country: params.country,
+            });
+            const httpsAgent = new HttpsProxyAgent(proxyUrl);
+            // First IP check via httpbin
+            const ip1Resp = await axios.get("https://httpbin.org/ip", {
+                httpsAgent,
+                proxy: false,
+                timeout: 15000,
+            });
+            const ip1 = ip1Resp.data.origin?.split(",")[0]?.trim();
+            // Second IP check — same session, should return same IP
+            const ip2Resp = await axios.get("https://httpbin.org/ip", {
+                httpsAgent,
+                proxy: false,
+                timeout: 15000,
+            });
+            const ip2 = ip2Resp.data.origin?.split(",")[0]?.trim();
+            session_verified = ip1 !== undefined && ip2 !== undefined && ip1 === ip2;
+        }
+        catch {
+            // Verification call failed — leave session_verified undefined
+            session_verified = false;
+        }
+    }
+    // credits: 1 base + 2 for verify_sticky (2 httpbin calls)
+    const creditsEstimated = verify_sticky ? 3 : 1;
+    // Rebuild response with session_verified in meta
+    const result = {
+        ...fetchResult,
+        tool: "novada_proxy_session",
+        meta: {
+            ...fetchResult.meta,
+            session_id: params.session_id,
+            session_verified,
+            quota: { credits_estimated: creditsEstimated, note: QUOTA_NOTE },
+        },
+    };
+    if (result.meta.session_verified === undefined)
+        delete result.meta.session_verified;
+    return JSON.stringify(result);
+}
+export function validateSessionParams(raw) {
+    if (!raw.session_id || typeof raw.session_id !== "string" || raw.session_id.length > 64 || !SAFE_SESSION_ID.test(raw.session_id)) {
+        throw new Error("session_id is required — letters, numbers, underscores only, max 64 chars (no hyphens)");
+    }
+    if (!raw.url || typeof raw.url !== "string") {
+        throw new Error("url is required");
+    }
+    if (!raw.url.startsWith("http://") && !raw.url.startsWith("https://")) {
+        throw new Error("url must start with http:// or https://");
+    }
+    if (raw.country !== undefined) {
+        if (typeof raw.country !== "string" || raw.country.length > 10 || !SAFE_COUNTRY.test(raw.country)) {
+            throw new Error("country must be a 2-letter ISO code with no hyphens (e.g. US, DE, GB)");
+        }
+    }
+    if (raw.city !== undefined) {
+        if (typeof raw.city !== "string" || raw.city.length > 50 || !SAFE_CITY.test(raw.city)) {
+            throw new Error("city must contain only letters, numbers, underscores, max 50 chars (e.g. newyork, london)");
+        }
+    }
+    if (raw.format && raw.format !== "raw" && raw.format !== "markdown") {
+        throw new Error("format must be 'raw' or 'markdown'");
+    }
+    const timeout = raw.timeout !== undefined ? Number(raw.timeout) : 60;
+    if (!Number.isFinite(timeout) || timeout < 1 || timeout > 120) {
+        throw new Error("timeout must be between 1 and 120 seconds");
+    }
+    return {
+        session_id: raw.session_id,
+        url: raw.url,
+        country: raw.country,
+        city: raw.city,
+        format: raw.format || "markdown",
+        timeout,
+        verify_sticky: raw.verify_sticky === true,
+    };
+}

package/build/tools/status.d.ts

@@ -0,0 +1,2 @@
+import type { ProxyAdapter, ProxyCredentials } from "../adapters/index.js";
+export declare function novadaProxyStatus(adapter?: ProxyAdapter, credentials?: ProxyCredentials): Promise<string>;

package/build/tools/status.js

@@ -0,0 +1,66 @@
+import axios from "axios";
+import { HttpsProxyAgent } from "https-proxy-agent";
+import { VERSION } from "../config.js";
+import { QUOTA_NOTE } from "../validation.js";
+export async function novadaProxyStatus(adapter, credentials) {
+    const startTime = Date.now();
+    let connectivity_status = "UNAVAILABLE";
+    let proxy_ip;
+    const verified_via = "https://httpbin.org/ip";
+    if (adapter && credentials) {
+        try {
+            const proxyUrl = adapter.buildProxyUrl(credentials, {});
+            const httpsAgent = new HttpsProxyAgent(proxyUrl);
+            const response = await axios.get(verified_via, {
+                httpsAgent,
+                proxy: false,
+                timeout: 10000,
+            });
+            const ip = response.data.origin?.split(",")[0]?.trim();
+            if (ip) {
+                proxy_ip = ip;
+                connectivity_status = "HEALTHY";
+            }
+            else {
+                connectivity_status = "DEGRADED";
+            }
+        }
+        catch {
+            connectivity_status = "UNAVAILABLE";
+        }
+    }
+    const latency_ms = Date.now() - startTime;
+    const capabilities = [];
+    if (adapter) {
+        if (adapter.capabilities.country)
+            capabilities.push("country_targeting");
+        if (adapter.capabilities.city)
+            capabilities.push("city_targeting");
+        if (adapter.capabilities.sticky)
+            capabilities.push("sticky_sessions");
+    }
+    const result = {
+        ok: true,
+        tool: "novada_proxy_status",
+        data: {
+            provider: adapter?.displayName || "none configured",
+            version: VERSION,
+            capabilities,
+            connectivity: {
+                status: connectivity_status,
+                verified_via,
+                proxy_ip,
+                latency_ms,
+            },
+        },
+        meta: {
+            latency_ms,
+            quota: { credits_estimated: 1, note: QUOTA_NOTE },
+        },
+    };
+    // Remove undefined proxy_ip
+    if (!proxy_ip) {
+        delete result.data.connectivity.proxy_ip;
+    }
+    return JSON.stringify(result);
+}

package/build/types.d.ts

@@ -0,0 +1,34 @@
+export type ProxyErrorCode = "BOT_DETECTION_SUSPECTED" | "SESSION_STICKINESS_FAILED" | "RATE_LIMITED" | "INVALID_INPUT" | "TIMEOUT" | "TLS_ERROR" | "NETWORK_ERROR" | "PROVIDER_NOT_CONFIGURED" | "UNKNOWN_ERROR";
+export interface QuotaMeta {
+    credits_estimated: number;
+    note: string;
+}
+export interface ProxySuccessResponse {
+    ok: true;
+    tool: string;
+    data: Record<string, unknown>;
+    meta: {
+        latency_ms: number;
+        proxy_ip?: string;
+        country?: string;
+        session_id?: string;
+        session_verified?: boolean;
+        truncated?: boolean;
+        content_density?: number;
+        concurrency?: number;
+        quota?: QuotaMeta;
+        cache_hit?: boolean;
+        cache_age_seconds?: number;
+    };
+}
+export interface ProxyErrorResponse {
+    ok: false;
+    error: {
+        code: ProxyErrorCode;
+        message: string;
+        recoverable: boolean;
+        agent_instruction: string;
+        retry_after_seconds?: number;
+    };
+}
+export type ProxyResponse = ProxySuccessResponse | ProxyErrorResponse;

package/build/types.js

@@ -0,0 +1 @@
+export {};

package/build/utils.d.ts

@@ -0,0 +1,18 @@
+export declare function unicodeSafeTruncate(s: string, maxChars: number): string;
+export declare function decodeHtmlEntities(s: string): string;
+/**
+ * Remove noise elements from HTML BEFORE markdown conversion.
+ * Conservative — only strips elements where the tag name or class/id strongly indicates noise.
+ */
+export declare function stripNoiseElements(html: string): string;
+export declare function htmlToMarkdown(html: string): string;
+/**
+ * Count rough number of HTML tags in a string.
+ */
+export declare function countHtmlTags(html: string): number;
+/**
+ * Compute content density score: ratio of text content to total content + tag overhead.
+ * Higher = cleaner content. Range: 0.0 to 1.0.
+ */
+export declare function contentDensity(markdownLength: number, tagCount: number): number;
+export declare function htmlToText(html: string): string;

package/build/utils.js

@@ -0,0 +1,151 @@
+export function unicodeSafeTruncate(s, maxChars) {
+    if (s.length <= maxChars)
+        return s;
+    let end = maxChars;
+    // Don't split a surrogate pair at the boundary
+    const code = s.charCodeAt(end - 1);
+    if (code >= 0xD800 && code <= 0xDBFF)
+        end--; // high surrogate stranded → drop it
+    else if (code >= 0xDC00 && code <= 0xDFFF)
+        end -= 2; // low surrogate → drop the whole pair
+    return s.slice(0, end);
+}
+export function decodeHtmlEntities(s) {
+    return s
+        .replace(/&amp;/g, "&")
+        .replace(/&lt;/g, "<")
+        .replace(/&gt;/g, ">")
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'")
+        .replace(/&nbsp;/g, " ");
+}
+/**
+ * Noise class/id patterns that strongly indicate non-content elements.
+ * Conservative: only match when the pattern is a clear indicator of noise.
+ */
+const NOISE_ATTR_PATTERN = /\b(cookie[-_]?banner|cookie[-_]?consent|cookie[-_]?notice|popup|modal|overlay|sidebar|nav[-_]?bar|navigation|footer|header|advertisement|ad[-_]?banner|social[-_]?share|share[-_]?buttons|comments?[-_]?section|menu[-_]?toggle|skip[-_]?nav|breadcrumb)\b/i;
+/**
+ * Remove noise elements from HTML BEFORE markdown conversion.
+ * Conservative — only strips elements where the tag name or class/id strongly indicates noise.
+ */
+export function stripNoiseElements(html) {
+    let result = html;
+    // 1. Strip structural noise tags and their content: nav, header, footer, aside, form
+    result = result.replace(/<nav[\s>][\s\S]*?<\/nav>/gi, "");
+    result = result.replace(/<header[\s>][\s\S]*?<\/header>/gi, "");
+    result = result.replace(/<footer[\s>][\s\S]*?<\/footer>/gi, "");
+    result = result.replace(/<aside[\s>][\s\S]*?<\/aside>/gi, "");
+    result = result.replace(/<form[\s>][\s\S]*?<\/form>/gi, "");
+    // 2. Strip elements with noise class/id patterns
+    //    Match opening tags with class="..." or id="..." containing noise keywords,
+    //    then remove through the matching closing tag.
+    //    We handle <div>, <section>, <span>, <ul>, <ol> with noise attributes.
+    const noiseTagNames = ["div", "section", "span", "ul", "ol", "p"];
+    for (const tag of noiseTagNames) {
+        // Match opening tag with class or id containing noise pattern
+        const openTagRe = new RegExp(`<${tag}\\s[^>]*(?:class|id)\\s*=\\s*["'][^"']*${NOISE_ATTR_PATTERN.source}[^"']*["'][^>]*>`, "gi");
+        // For each match, find the corresponding closing tag and remove everything
+        let match;
+        while ((match = openTagRe.exec(result)) !== null) {
+            const startIdx = match.index;
+            // Simple depth-based closing tag finder
+            const closeTag = `</${tag}>`;
+            let depth = 1;
+            let searchPos = startIdx + match[0].length;
+            const openRe = new RegExp(`<${tag}[\\s>]`, "gi");
+            const closeRe = new RegExp(`</${tag}>`, "gi");
+            let endIdx = -1;
+            while (depth > 0 && searchPos < result.length) {
+                openRe.lastIndex = searchPos;
+                closeRe.lastIndex = searchPos;
+                const nextOpen = openRe.exec(result);
+                const nextClose = closeRe.exec(result);
+                if (!nextClose)
+                    break; // malformed HTML, bail
+                if (nextOpen && nextOpen.index < nextClose.index) {
+                    depth++;
+                    searchPos = nextOpen.index + nextOpen[0].length;
+                }
+                else {
+                    depth--;
+                    if (depth === 0) {
+                        endIdx = nextClose.index + closeTag.length;
+                    }
+                    searchPos = nextClose.index + nextClose[0].length;
+                }
+            }
+            if (endIdx !== -1) {
+                result = result.slice(0, startIdx) + result.slice(endIdx);
+                openTagRe.lastIndex = startIdx; // re-scan from same position
+            }
+        }
+    }
+    // 3. Strip hidden elements
+    result = result.replace(/<[^>]+style\s*=\s*["'][^"']*display\s*:\s*none[^"']*["'][^>]*>[\s\S]*?<\/[^>]+>/gi, "");
+    result = result.replace(/<[^>]+style\s*=\s*["'][^"']*visibility\s*:\s*hidden[^"']*["'][^>]*>[\s\S]*?<\/[^>]+>/gi, "");
+    result = result.replace(/<[^>]+aria-hidden\s*=\s*["']true["'][^>]*>[\s\S]*?<\/[^>]+>/gi, "");
+    // 4. Strip empty divs and spans (only whitespace content)
+    result = result.replace(/<div[^>]*>\s*<\/div>/gi, "");
+    result = result.replace(/<span[^>]*>\s*<\/span>/gi, "");
+    return result;
+}
+export function htmlToMarkdown(html) {
+    // Step 1: Strip noise elements before conversion
+    const cleaned = stripNoiseElements(html);
+    let md = cleaned
+        .replace(/<script[\s\S]*?<\/script>/gi, "")
+        .replace(/<style[\s\S]*?<\/style>/gi, "")
+        .replace(/<noscript[\s\S]*?<\/noscript>/gi, "")
+        .replace(/<br\s*\/?>/gi, "\n")
+        .replace(/<\/p>/gi, "\n\n")
+        .replace(/<\/h[1-6]>/gi, "\n\n")
+        .replace(/<\/li>/gi, "\n")
+        .replace(/<li[^>]*>/gi, "- ")
+        .replace(/<h([1-6])[^>]*>/gi, (_, n) => "#".repeat(Number(n)) + " ")
+        .replace(/<a[^>]+href=["']([^"']+)["'][^>]*>([^<]*)<\/a>/gi, (_, href, text) => {
+        const decoded = decodeHtmlEntities(href);
+        if (decoded.startsWith("data:") || decoded.startsWith("javascript:"))
+            return text;
+        return `[${text}](${decoded})`;
+    })
+        .replace(/<[^>]+>/g, "")
+        .replace(/&amp;/g, "&")
+        .replace(/&lt;/g, "<")
+        .replace(/&gt;/g, ">")
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'")
+        .replace(/&nbsp;/g, " ");
+    // Step 2: Post-conversion cleanup
+    // Collapse 3+ consecutive newlines to 2
+    md = md.replace(/\n{3,}/g, "\n\n");
+    // Remove lines that are only dashes or underscores (visual separators)
+    md = md.replace(/^\s*[-_]{3,}\s*$/gm, "");
+    // Trim trailing whitespace per line
+    md = md.replace(/[^\S\n]+$/gm, "");
+    // Final collapse after separator removal
+    md = md.replace(/\n{3,}/g, "\n\n");
+    return md.trim();
+}
+/**
+ * Count rough number of HTML tags in a string.
+ */
+export function countHtmlTags(html) {
+    const matches = html.match(/<[a-zA-Z][^>]*>/g);
+    return matches ? matches.length : 0;
+}
+/**
+ * Compute content density score: ratio of text content to total content + tag overhead.
+ * Higher = cleaner content. Range: 0.0 to 1.0.
+ */
+export function contentDensity(markdownLength, tagCount) {
+    if (markdownLength === 0 && tagCount === 0)
+        return 0;
+    return parseFloat((markdownLength / (markdownLength + tagCount * 10)).toFixed(2));
+}
+export function htmlToText(html) {
+    return htmlToMarkdown(html)
+        .replace(/\[([^\]]+)\]\([^)]+\)/g, "$1") // strip link URLs, keep text
+        .replace(/#+\s/g, "") // strip heading markers
+        .replace(/^-\s/gm, "") // strip list bullets
+        .trim();
+}

package/build/validation.d.ts

@@ -0,0 +1,4 @@
+export declare const SAFE_COUNTRY: RegExp;
+export declare const SAFE_CITY: RegExp;
+export declare const SAFE_SESSION_ID: RegExp;
+export declare const QUOTA_NOTE = "Check dashboard.novada.com for real-time balance";

package/build/validation.js

@@ -0,0 +1,6 @@
+// Proxy username injection prevention — no hyphens allowed
+// (providers use `-` as segment delimiter in auth strings)
+export const SAFE_COUNTRY = /^[a-zA-Z0-9_]+$/;
+export const SAFE_CITY = /^[a-zA-Z0-9_]+$/;
+export const SAFE_SESSION_ID = /^[a-zA-Z0-9_]+$/;
+export const QUOTA_NOTE = "Check dashboard.novada.com for real-time balance";

package/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "novada-proxy-core",
+  "version": "0.0.1",
+  "description": "Core proxy engine \u2014 adapters, tools, types for Novada Proxy",
+  "type": "module",
+  "main": "build/index.js",
+  "types": "build/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./build/index.d.ts",
+      "import": "./build/index.js"
+    },
+    "./tools": {
+      "types": "./build/tools/index.d.ts",
+      "import": "./build/tools/index.js"
+    },
+    "./adapters": {
+      "types": "./build/adapters/index.d.ts",
+      "import": "./build/adapters/index.js"
+    },
+    "./errors": {
+      "types": "./build/errors.d.ts",
+      "import": "./build/errors.js"
+    }
+  },
+  "files": [
+    "build/**/*.js",
+    "build/**/*.d.ts",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run"
+  },
+  "dependencies": {
+    "axios": "^1.7.0",
+    "http-proxy-agent": "^7.0.0",
+    "https-proxy-agent": "^9.0.0",
+    "puppeteer-core": "^22.15.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.11.24",
+    "typescript": "^5.3.3",
+    "vitest": "^4.1.4"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "license": "MIT"
+}