neuron-inspector 0.3.1 → 0.4.0

package/dist/compound-tools.js

@@ -162,6 +162,106 @@ export const COMPOUND_TOOLS = [
             required: ["tabId", "action"],
         },
     },
+    {
+        name: "neuron_vision_act",
+        description: "Take a screenshot of the page and describe what's visible, then perform an action based on " +
+            "visual understanding — no CSS selectors needed. The extension screenshots the viewport, the " +
+            "agent analyzes the image description, and issues click/type commands using element coordinates " +
+            "or best-match selectors. Use when you don't know the page structure or selectors keep breaking. " +
+            "Describe what you want to interact with in natural language.",
+        inputSchema: {
+            type: "object",
+            properties: {
+                tabId: { type: "number", description: "Chrome tab ID" },
+                instruction: {
+                    type: "string",
+                    description: "What to do, described visually (e.g. 'click the blue Send button', 'type in the search box at the top', 'scroll to the comments section')",
+                },
+                screenshot: { type: "boolean", description: "Return the screenshot for the AI to analyze (default: true)" },
+            },
+            required: ["tabId", "instruction"],
+        },
+    },
+    {
+        name: "neuron_approve_via_whatsapp",
+        description: "Send an approval request to WhatsApp and wait for the response. Takes a screenshot of the " +
+            "current state, sends it with a prompt to a WhatsApp number, waits for approve/reject reply. " +
+            "Use instead of pausing for keyboard approval — the user approves from their phone. Requires " +
+            "a Neuron bot API key and the recipient's phone number.",
+        inputSchema: {
+            type: "object",
+            properties: {
+                tabId: { type: "number", description: "Chrome tab ID to screenshot (optional)" },
+                prompt: { type: "string", description: "The approval question (e.g. 'Send this message to John?')" },
+                phone: { type: "string", description: "WhatsApp phone number to send the approval request to (E.164 format)" },
+                api_key: { type: "string", description: "Neuron bot API key with 'nrn_' prefix" },
+                context: { type: "string", description: "Additional context (e.g. the full message text, the form data)" },
+                timeout_seconds: { type: "number", description: "How long to wait for a response (default: 120)" },
+            },
+            required: ["prompt", "phone", "api_key"],
+        },
+    },
+    {
+        name: "neuron_extract_to_json",
+        description: "Extract structured data from a page and return it as clean JSON ready for piping to an API, " +
+            "spreadsheet, or file. Navigates to the URL, extracts data matching a schema you define, " +
+            "and returns normalized rows. Use for building data pipelines — scrape a page and push the " +
+            "data to a webhook, save as CSV, or append to a collection.",
+        inputSchema: {
+            type: "object",
+            properties: {
+                url: { type: "string", description: "Page URL to extract from" },
+                tabId: { type: "number", description: "Use existing tab (optional)" },
+                schema: {
+                    type: "object",
+                    description: "Expected output schema — keys are field names, values describe what to extract (e.g. {title: 'the post title', price: 'the price as a number', url: 'link to the item'})",
+                },
+                selector: { type: "string", description: "CSS selector for repeating items (optional — auto-detects)" },
+                pages: { type: "number", description: "Number of pages to paginate through (default: 1)" },
+                webhook_url: { type: "string", description: "POST extracted data to this URL as JSON (optional)" },
+                format: { type: "string", description: "Output format: 'json' (default), 'csv', 'yaml'" },
+            },
+            required: ["url"],
+        },
+    },
+    {
+        name: "neuron_grab_media",
+        description: "Extract and download video/audio from any page. Navigates to the URL, plays the media to " +
+            "trigger network requests, searches captured traffic for video/audio streams (mp4, m3u8, " +
+            "webm, mp3, blob), extracts the CDN URLs, and initiates a browser download. Works on " +
+            "Instagram reels, TikTok videos, X/Twitter videos, Facebook videos, LinkedIn videos, YouTube, " +
+            "and most sites with embedded video. Returns the download URLs found and download status.",
+        inputSchema: {
+            type: "object",
+            properties: {
+                url: { type: "string", description: "Page URL containing the video (e.g. an Instagram reel URL, TikTok video URL)" },
+                filename: { type: "string", description: "Filename to save as (optional — auto-generates from URL)" },
+                tabId: { type: "number", description: "Use an existing tab (optional)" },
+                waitSeconds: { type: "number", description: "How long to wait for video to start loading (default: 5)" },
+                preferQuality: { type: "string", description: "Preferred quality: 'highest', 'lowest', 'auto' (default: highest)" },
+            },
+            required: ["url"],
+        },
+    },
+    {
+        name: "neuron_grab_media_batch",
+        description: "Download videos from multiple URLs. Opens each in a tab, extracts video streams, " +
+            "downloads all. Up to 5 URLs per call. Returns results for each URL with download status " +
+            "and any failures. Use for batch downloading from feeds, trending pages, or saved lists.",
+        inputSchema: {
+            type: "object",
+            properties: {
+                urls: {
+                    type: "array",
+                    items: { type: "string" },
+                    description: "Video page URLs (max 5)",
+                },
+                outputDir: { type: "string", description: "Subdirectory name for downloads (optional)" },
+                delayMs: { type: "number", description: "Delay between processing each URL in ms (default: 3000)" },
+            },
+            required: ["urls"],
+        },
+    },
 ];
 // ── Handlers ────────────────────────────────────────────────
 export async function handleCompoundTool(name, args, ctx) {
@@ -178,6 +278,16 @@ export async function handleCompoundTool(name, args, ctx) {
             return auditPage(args, ctx);
         case "neuron_monitor_action":
             return monitorAction(args, ctx);
+        case "neuron_vision_act":
+            return visionAct(args, ctx);
+        case "neuron_approve_via_whatsapp":
+            return approveViaWhatsapp(args, ctx);
+        case "neuron_extract_to_json":
+            return extractToJson(args, ctx);
+        case "neuron_grab_media":
+            return grabMedia(args, ctx);
+        case "neuron_grab_media_batch":
+            return grabMediaBatch(args, ctx);
         default:
             throw new Error(`Unknown compound tool: ${name}`);
     }
@@ -520,4 +630,553 @@ async function monitorAction(args, ctx) {
         errors,
     };
 }
+// ── Vision-based action ─────────────────────────────────────
+async function visionAct(args, ctx) {
+    const tabId = args.tabId;
+    const instruction = args.instruction;
+    const wantScreenshot = args.screenshot ?? true;
+    // Screenshot the page
+    let screenshot = null;
+    if (wantScreenshot) {
+        try {
+            screenshot = await call(ctx, "captureScreenshot", { tabId });
+        }
+        catch { /* non-fatal */ }
+    }
+    // Get page structure for element mapping
+    const elements = await call(ctx, "findElements", {
+        tabId,
+        selectors: [
+            "button", "a", "input", "textarea", "select",
+            "[role='button']", "[role='link']", "[role='textbox']",
+            "[contenteditable='true']", "[onclick]",
+        ],
+        limit: 50,
+    });
+    // Get visible text blocks for context
+    const pageData = await call(ctx, "extractData", { tabId });
+    return {
+        tabId,
+        instruction,
+        screenshot,
+        interactive_elements: elements,
+        page_content: pageData,
+        hint: "Analyze the screenshot and interactive_elements to find what matches the instruction. " +
+            "Use the element selectors or text from interactive_elements to call neuron_click or neuron_type. " +
+            "If no exact match, try neuron_find_elements with descriptive text from the instruction.",
+    };
+}
+// ── WhatsApp approval ───────────────────────────────────────
+async function approveViaWhatsapp(args, ctx) {
+    const tabId = args.tabId;
+    const prompt = args.prompt;
+    const phone = args.phone;
+    const apiKey = args.api_key;
+    const context = args.context;
+    const timeoutSec = args.timeout_seconds ?? 120;
+    // Screenshot if tabId provided
+    let screenshotData = null;
+    if (tabId) {
+        try {
+            const ss = (await call(ctx, "captureScreenshot", { tabId }));
+            screenshotData = ss?.dataUrl ?? null;
+        }
+        catch { /* non-fatal */ }
+    }
+    // Build the approval message
+    const fullPrompt = context
+        ? `${prompt}\n\n---\nContext:\n${context}`
+        : prompt;
+    // Send approval request via Neuron's approval API (HTTP from the bridge process)
+    const approvalPayload = {
+        apiKey,
+        to: phone,
+        prompt: fullPrompt,
+        context: context || undefined,
+        expiresInSeconds: timeoutSec,
+        callback: {
+            response: { schemes: ["keyword"] },
+        },
+    };
+    // Use the extension to make the HTTP call (it has the network context)
+    // Alternatively, make a direct fetch from the bridge process
+    try {
+        const response = await fetch("https://api.neuron.ng/api/v1/approvals", {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "Authorization": `Bearer ${apiKey}`,
+            },
+            body: JSON.stringify(approvalPayload),
+        });
+        if (!response.ok) {
+            const errBody = await response.text();
+            return {
+                status: "approval_send_failed",
+                error: `HTTP ${response.status}: ${errBody}`,
+            };
+        }
+        const result = await response.json();
+        const approvalId = result?.data?.id;
+        if (!approvalId) {
+            return { status: "approval_send_failed", error: "No approval ID returned" };
+        }
+        // Poll for decision
+        const startTime = Date.now();
+        const pollInterval = 3000;
+        while (Date.now() - startTime < timeoutSec * 1000) {
+            await sleep(pollInterval);
+            try {
+                const pollResponse = await fetch(`https://api.neuron.ng/api/v1/approvals/${approvalId}`, {
+                    headers: { "Authorization": `Bearer ${apiKey}` },
+                });
+                if (pollResponse.ok) {
+                    const pollResult = await pollResponse.json();
+                    const status = pollResult?.data?.status;
+                    if (status === "approved" || status === "rejected") {
+                        return {
+                            status: status,
+                            approval_id: approvalId,
+                            decision: pollResult?.data?.decision,
+                            reason: pollResult?.data?.reason,
+                        };
+                    }
+                    if (status === "expired" || status === "cancelled") {
+                        return { status: status, approval_id: approvalId };
+                    }
+                }
+            }
+            catch {
+                // Poll failure — retry
+            }
+        }
+        return {
+            status: "timeout",
+            approval_id: approvalId,
+            message: `No response within ${timeoutSec} seconds`,
+        };
+    }
+    catch (err) {
+        return {
+            status: "error",
+            error: err.message,
+        };
+    }
+}
+// ── Data pipeline extraction ────────────────────────────────
+async function extractToJson(args, ctx) {
+    const url = args.url;
+    const schema = args.schema;
+    const extractSel = args.selector;
+    const maxPages = args.pages ?? 1;
+    const webhookUrl = args.webhook_url;
+    const format = args.format ?? "json";
+    // Navigate and extract using search_and_collect for multi-page
+    let tabId = args.tabId;
+    if (!tabId) {
+        const tab = (await call(ctx, "openTab", { url }));
+        tabId = tab?.tabId;
+        await sleep(2500);
+    }
+    else {
+        await call(ctx, "navigateTo", { tabId, url });
+        await sleep(2500);
+    }
+    if (!tabId)
+        throw new Error("Failed to open tab");
+    // Collect data across pages
+    const allItems = [];
+    for (let page = 0; page < maxPages; page++) {
+        // Scroll to load lazy content
+        for (let i = 0; i < 3; i++) {
+            await call(ctx, "scrollPage", { tabId, deltaY: 600, smooth: true });
+            await sleep(400);
+        }
+        const extractArgs = { tabId };
+        if (extractSel)
+            extractArgs.selector = extractSel;
+        const pageData = await call(ctx, "extractData", extractArgs);
+        const items = Array.isArray(pageData)
+            ? pageData
+            : pageData?.items ?? [pageData];
+        allItems.push(...items);
+        // Paginate if needed
+        if (page < maxPages - 1) {
+            try {
+                await call(ctx, "clickElement", {
+                    tabId,
+                    selectors: [
+                        "button[aria-label='Next']", "a[aria-label='Next']",
+                        ".pagination-next", "[data-test='pagination-next']",
+                        "a:has-text('Next')", "button:has-text('Next')",
+                    ],
+                });
+                await sleep(2000);
+            }
+            catch {
+                break;
+            }
+        }
+    }
+    // Format output
+    let output;
+    if (format === "csv" && allItems.length > 0) {
+        const firstItem = allItems[0];
+        const headers = Object.keys(firstItem);
+        const csvRows = [headers.join(",")];
+        for (const item of allItems) {
+            const row = headers.map((h) => {
+                const val = String(item[h] ?? "");
+                return val.includes(",") || val.includes('"') ? `"${val.replace(/"/g, '""')}"` : val;
+            });
+            csvRows.push(row.join(","));
+        }
+        output = csvRows.join("\n");
+    }
+    else if (format === "yaml") {
+        // Simple YAML-ish output
+        output = allItems.map((item, i) => {
+            const entries = Object.entries(item);
+            return `- # item ${i + 1}\n` + entries.map(([k, v]) => `  ${k}: ${JSON.stringify(v)}`).join("\n");
+        }).join("\n");
+    }
+    else {
+        output = allItems;
+    }
+    // Send to webhook if specified
+    let webhookResult = null;
+    if (webhookUrl) {
+        try {
+            const res = await fetch(webhookUrl, {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify({ source: url, extracted_at: new Date().toISOString(), items: allItems }),
+            });
+            webhookResult = { status: res.status, ok: res.ok };
+        }
+        catch (err) {
+            webhookResult = { error: err.message };
+        }
+    }
+    return {
+        url,
+        tabId,
+        format,
+        total_items: allItems.length,
+        pages_collected: Math.min(maxPages, 1),
+        data: output,
+        schema_hint: schema ?? null,
+        webhook: webhookResult,
+    };
+}
+// ── Media patterns ──────────────────────────────────────────
+const VIDEO_PATTERNS = [
+    ".mp4", ".m3u8", ".webm", ".m4v", ".mov",
+    "video/mp4", "video/webm", "video/quicktime",
+    "application/x-mpegURL", "application/vnd.apple.mpegurl",
+];
+const VIDEO_CDN_PATTERNS = [
+    /cdninstagram\.com.*\.mp4/,
+    /scontent.*\.cdninstagram\.com/,
+    /video\.twimg\.com/,
+    /pbs\.twimg\.com.*\/vid\//,
+    /video\.xx\.fbcdn\.net/,
+    /scontent.*\.xx\.fbcdn\.net.*video/,
+    /v\d+-webapp.*\.tiktok.*\.com/,
+    /pull-.*\.tiktokcdn.*\.com/,
+    /googlevideo\.com\/videoplayback/,
+    /\.googlevideo\.com/,
+];
+function scoreMediaUrl(url) {
+    // Higher = better quality
+    let score = 0;
+    if (/1080|hd|high/i.test(url))
+        score += 10;
+    if (/720/i.test(url))
+        score += 7;
+    if (/480/i.test(url))
+        score += 4;
+    if (/360|240|low/i.test(url))
+        score += 1;
+    if (/\.mp4/i.test(url))
+        score += 3;
+    if (/\.m3u8/i.test(url))
+        score += 1; // HLS needs extra handling
+    // Prefer longer URLs (usually contain more path segments = direct CDN)
+    if (url.length > 200)
+        score += 2;
+    return score;
+}
+// ── Media grab ──────────────────────────────────────────────
+async function grabMedia(args, ctx) {
+    const url = args.url;
+    const filename = args.filename;
+    const waitSec = args.waitSeconds ?? 5;
+    const preferQuality = args.preferQuality ?? "highest";
+    // Open the page
+    let tabId = args.tabId;
+    if (!tabId) {
+        const tab = (await call(ctx, "openTab", { url }));
+        tabId = tab?.tabId;
+        await sleep(3000);
+    }
+    else {
+        await call(ctx, "navigateTo", { tabId, url });
+        await sleep(3000);
+    }
+    if (!tabId)
+        throw new Error("Failed to open tab");
+    // Try to click play / interact to start video loading
+    try {
+        // Common play button selectors across platforms
+        await call(ctx, "evaluateJS", {
+            tabId,
+            expression: `
+        // Try clicking play buttons or video elements to trigger load
+        const playBtns = document.querySelectorAll(
+          'video, [aria-label*="Play"], [aria-label*="play"], [data-testid*="play"], ' +
+          'button[class*="play"], div[class*="play"], svg[aria-label*="Play"]'
+        );
+        for (const el of playBtns) {
+          if (el instanceof HTMLElement) { el.click(); break; }
+        }
+        // Also try to play any video element directly
+        const video = document.querySelector('video');
+        if (video) { video.play().catch(() => {}); }
+        'triggered'
+      `,
+        });
+    }
+    catch {
+        // Non-fatal — the page might auto-play
+    }
+    // Wait for video to load and network requests to be captured
+    await sleep(waitSec * 1000);
+    // Search captured traffic for video URLs
+    const candidates = [];
+    // Method 1: Search request log for video content types and patterns
+    for (const pattern of [".mp4", ".m3u8", ".webm", "video/", "mpegURL"]) {
+        try {
+            const results = (await call(ctx, "searchTraffic", {
+                query: pattern,
+                limit: 20,
+            }));
+            const matches = results?.matches;
+            if (Array.isArray(matches)) {
+                for (const m of matches) {
+                    const match = m;
+                    const mUrl = String(match.url || "");
+                    if (mUrl && isVideoUrl(mUrl)) {
+                        candidates.push({
+                            url: mUrl,
+                            type: String(match.contentType || guessType(mUrl)),
+                            source: "traffic_search",
+                            size: match.size,
+                        });
+                    }
+                }
+            }
+        }
+        catch {
+            // Continue with other patterns
+        }
+    }
+    // Method 2: Check request log filtered by URL patterns
+    try {
+        const requests = (await call(ctx, "getRecentRequests", {
+            tabId,
+            limit: 200,
+        }));
+        const reqList = requests?.requests;
+        if (Array.isArray(reqList)) {
+            for (const r of reqList) {
+                const req = r;
+                const rUrl = String(req.url || "");
+                if (isVideoUrl(rUrl)) {
+                    candidates.push({
+                        url: rUrl,
+                        type: guessType(rUrl),
+                        source: "request_log",
+                    });
+                }
+            }
+        }
+    }
+    catch {
+        // Non-fatal
+    }
+    // Method 3: Extract video src from DOM
+    try {
+        const domResult = await call(ctx, "evaluateJS", {
+            tabId,
+            expression: `
+        const sources = [];
+        // Direct video elements
+        document.querySelectorAll('video source, video').forEach(el => {
+          const src = el.src || el.getAttribute('src');
+          if (src && !src.startsWith('blob:')) sources.push({ url: src, source: 'video_element' });
+          // Check source children
+          el.querySelectorAll?.('source')?.forEach(s => {
+            const sSrc = s.src || s.getAttribute('src');
+            if (sSrc && !sSrc.startsWith('blob:')) sources.push({ url: sSrc, source: 'source_element' });
+          });
+        });
+        // OG video meta
+        const ogVideo = document.querySelector('meta[property="og:video"]');
+        if (ogVideo?.content) sources.push({ url: ogVideo.content, source: 'og_meta' });
+        const ogVideoUrl = document.querySelector('meta[property="og:video:url"]');
+        if (ogVideoUrl?.content) sources.push({ url: ogVideoUrl.content, source: 'og_meta' });
+        JSON.stringify(sources);
+      `,
+        });
+        const domSources = JSON.parse(String(domResult ?? "[]"));
+        for (const s of domSources) {
+            if (s.url) {
+                candidates.push({ url: s.url, type: guessType(s.url), source: s.source });
+            }
+        }
+    }
+    catch {
+        // Non-fatal
+    }
+    // Deduplicate by URL
+    const seen = new Set();
+    const unique = candidates.filter((c) => {
+        if (seen.has(c.url))
+            return false;
+        seen.add(c.url);
+        return true;
+    });
+    if (unique.length === 0) {
+        return {
+            url,
+            tabId,
+            status: "no_video_found",
+            message: "No downloadable video URL found in network traffic or DOM. The video may use DRM, blob URLs, or require authentication.",
+            candidates_checked: candidates.length,
+        };
+    }
+    // Sort by quality preference
+    unique.sort((a, b) => {
+        const scoreA = scoreMediaUrl(a.url);
+        const scoreB = scoreMediaUrl(b.url);
+        return preferQuality === "lowest" ? scoreA - scoreB : scoreB - scoreA;
+    });
+    const best = unique[0];
+    // Initiate download via the extension
+    let downloadStatus = "url_found";
+    try {
+        const dlFilename = filename || generateFilename(url, best.type);
+        await call(ctx, "evaluateJS", {
+            tabId,
+            expression: `
+        // Use fetch to download, then create a download link
+        fetch("${best.url.replace(/"/g, '\\"')}", { credentials: 'include' })
+          .then(r => r.blob())
+          .then(blob => {
+            const a = document.createElement('a');
+            a.href = URL.createObjectURL(blob);
+            a.download = "${dlFilename.replace(/"/g, '\\"')}";
+            document.body.appendChild(a);
+            a.click();
+            document.body.removeChild(a);
+            URL.revokeObjectURL(a.href);
+          })
+          .catch(() => {
+            // Fallback: open the URL directly
+            window.open("${best.url.replace(/"/g, '\\"')}", '_blank');
+          });
+        'download_initiated'
+      `,
+        }, 30000);
+        downloadStatus = "download_initiated";
+    }
+    catch {
+        downloadStatus = "url_found_download_failed";
+    }
+    return {
+        url,
+        tabId,
+        status: downloadStatus,
+        best_url: best.url,
+        type: best.type,
+        source: best.source,
+        all_candidates: unique.length,
+        candidates: unique.slice(0, 5).map((c) => ({
+            url: c.url,
+            type: c.type,
+            source: c.source,
+            quality_score: scoreMediaUrl(c.url),
+        })),
+    };
+}
+function isVideoUrl(url) {
+    const lower = url.toLowerCase();
+    if (VIDEO_PATTERNS.some((p) => lower.includes(p)))
+        return true;
+    if (VIDEO_CDN_PATTERNS.some((p) => p.test(url)))
+        return true;
+    return false;
+}
+function guessType(url) {
+    const lower = url.toLowerCase();
+    if (lower.includes(".mp4") || lower.includes("video/mp4"))
+        return "video/mp4";
+    if (lower.includes(".m3u8") || lower.includes("mpegurl"))
+        return "application/x-mpegURL";
+    if (lower.includes(".webm"))
+        return "video/webm";
+    if (lower.includes(".mp3") || lower.includes("audio/mp"))
+        return "audio/mpeg";
+    return "video/unknown";
+}
+function generateFilename(pageUrl, type) {
+    const ts = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19);
+    let platform = "video";
+    if (pageUrl.includes("instagram"))
+        platform = "ig";
+    else if (pageUrl.includes("tiktok"))
+        platform = "tk";
+    else if (pageUrl.includes("x.com") || pageUrl.includes("twitter"))
+        platform = "x";
+    else if (pageUrl.includes("facebook"))
+        platform = "fb";
+    else if (pageUrl.includes("linkedin"))
+        platform = "li";
+    else if (pageUrl.includes("youtube") || pageUrl.includes("youtu.be"))
+        platform = "yt";
+    const ext = type.includes("mp4") ? ".mp4" : type.includes("webm") ? ".webm" : ".mp4";
+    return `${platform}-${ts}${ext}`;
+}
+async function grabMediaBatch(args, ctx) {
+    const urls = args.urls.slice(0, 5);
+    const delayMs = args.delayMs ?? 3000;
+    const results = [];
+    for (const url of urls) {
+        try {
+            const result = (await grabMedia({ url, preferQuality: "highest" }, ctx));
+            results.push({
+                url,
+                status: String(result.status || "unknown"),
+                best_url: result.best_url,
+                type: result.type,
+            });
+        }
+        catch (err) {
+            results.push({
+                url,
+                status: "error",
+                error: err.message,
+            });
+        }
+        if (delayMs > 0)
+            await sleep(delayMs);
+    }
+    return {
+        total: results.length,
+        downloaded: results.filter((r) => r.status === "download_initiated").length,
+        failed: results.filter((r) => r.status === "error" || r.status === "no_video_found").length,
+        results,
+    };
+}
 //# sourceMappingURL=compound-tools.js.map