imperium-crawl 1.5.3 → 2.0.0

package/dist/tools/snapshot.js

@@ -0,0 +1,160 @@
+import { z } from "zod";
+import { isPlaywrightAvailable } from "../stealth/browser.js";
+import { acquirePage } from "../stealth/chrome-profile.js";
+import { resolveProxy } from "../stealth/proxy.js";
+import { normalizeUrl } from "../utils/url.js";
+import { getSessionManager } from "../sessions/index.js";
+import { getEnhancedSnapshot, getSnapshotStore, annotateScreenshot } from "../snapshot/index.js";
+import { installDomainFilter } from "../security/domain-filter.js";
+import { MAX_URL_LENGTH, MAX_TIMEOUT_MS } from "../constants.js";
+export const name = "snapshot";
+export const description = "Take an ARIA-based accessibility snapshot of a web page. Returns a structured tree with interactive element refs (e.g. [ref=e1]) that can be used in the interact tool for precise element targeting. Workflow: snapshot → analyze refs → interact with ref targeting → snapshot again to verify.";
+export const schema = z.object({
+    url: z.string().max(MAX_URL_LENGTH).describe("URL to snapshot"),
+    session_id: z
+        .string()
+        .max(200)
+        .optional()
+        .describe("Session ID to restore cookies and store refs. Also used as snapshot key for ref resolution."),
+    interactive: z
+        .boolean()
+        .default(true)
+        .describe("Only include interactive elements (buttons, links, inputs). Set false for full page content. (default: true)"),
+    cursor: z
+        .boolean()
+        .default(false)
+        .describe("Detect cursor:pointer/onclick elements without ARIA roles (default: false)"),
+    compact: z
+        .boolean()
+        .default(true)
+        .describe("Filter structural elements without refs for a cleaner tree (default: true)"),
+    scope_selector: z
+        .string()
+        .max(500)
+        .optional()
+        .describe("CSS selector to scope snapshot to a subtree (e.g. '#main-content', '.sidebar')"),
+    return_screenshot: z
+        .boolean()
+        .default(false)
+        .describe("Include a screenshot of the page"),
+    annotate: z
+        .boolean()
+        .default(false)
+        .describe("Overlay numbered badges on interactive elements in the screenshot. Requires return_screenshot: true."),
+    chrome_profile: z
+        .string()
+        .max(1000)
+        .optional()
+        .describe("Path to Chrome user data directory. Overrides CHROME_PROFILE_PATH env var."),
+    proxy: z
+        .string()
+        .max(MAX_URL_LENGTH)
+        .optional()
+        .describe("Proxy URL. Overrides PROXY_URL env var."),
+    timeout: z
+        .number()
+        .min(1000)
+        .max(MAX_TIMEOUT_MS)
+        .default(30000)
+        .describe("Navigation timeout in ms (default: 30000)"),
+    allowed_domains: z
+        .array(z.string().max(500))
+        .max(100)
+        .optional()
+        .describe("Domain whitelist. Blocks requests to non-allowed domains. Supports wildcards (e.g. '*.example.com')."),
+});
+export async function execute(input) {
+    if (!(await isPlaywrightAvailable())) {
+        return {
+            content: [{
+                    type: "text",
+                    text: JSON.stringify({
+                        error: "rebrowser-playwright is required for the snapshot tool. Install with: npm i rebrowser-playwright",
+                    }, null, 2),
+                }],
+        };
+    }
+    const url = normalizeUrl(input.url);
+    const proxyUrl = resolveProxy(input.proxy);
+    const snapshotId = input.session_id ?? `snap_${Date.now()}`;
+    const handle = await acquirePage({
+        chromeProfile: input.chrome_profile,
+        proxyUrl,
+    });
+    try {
+        const { page } = handle;
+        // Install domain filter
+        if (input.allowed_domains?.length) {
+            await installDomainFilter(page.context(), input.allowed_domains);
+        }
+        // Restore session cookies if session_id provided
+        if (input.session_id) {
+            const session = await getSessionManager().load(input.session_id);
+            if (session?.cookies.length) {
+                await page.context().addCookies(session.cookies);
+            }
+        }
+        // Navigate
+        await page.goto(url, { waitUntil: "load", timeout: input.timeout });
+        // Take snapshot
+        const snapshot = await getEnhancedSnapshot(page, {
+            interactive: input.interactive,
+            cursor: input.cursor,
+            compact: input.compact,
+            selector: input.scope_selector,
+        });
+        // Store refs for later use by interact tool
+        getSnapshotStore().save(snapshotId, snapshot.refs, page.url());
+        // Build output
+        const output = {
+            snapshot_id: snapshotId,
+            url: page.url(),
+            tree: snapshot.tree,
+            stats: {
+                ...snapshot.stats,
+                refCount: Object.keys(snapshot.refs).length,
+                treeLines: snapshot.tree.split("\n").length,
+                treeChars: snapshot.tree.length,
+            },
+        };
+        const content = [
+            { type: "text", text: JSON.stringify(output, null, 2) },
+        ];
+        // Optional screenshot (plain or annotated)
+        if (input.return_screenshot) {
+            const buf = input.annotate
+                ? await annotateScreenshot(page, snapshot.refs)
+                : await page.screenshot({ fullPage: false });
+            content.push({
+                type: "image",
+                data: buf.toString("base64"),
+                mimeType: "image/png",
+            });
+        }
+        // Save session if session_id provided
+        if (input.session_id) {
+            try {
+                const cookies = await page.context().cookies();
+                const stored = cookies.map((c) => ({
+                    name: c.name,
+                    value: c.value,
+                    domain: c.domain,
+                    path: c.path,
+                    expires: c.expires,
+                    httpOnly: c.httpOnly,
+                    secure: c.secure,
+                    sameSite: c.sameSite,
+                }));
+                await getSessionManager().save(input.session_id, stored, page.url());
+            }
+            catch {
+                // Non-critical — snapshot was still taken successfully
+            }
+        }
+        return { content };
+    }
+    finally {
+        await handle.cleanup();
+    }
+}
+//# sourceMappingURL=snapshot.js.map

package/dist/tools/snapshot.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"snapshot.js","sourceRoot":"","sources":["../../src/tools/snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AACjG,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAEjE,MAAM,CAAC,MAAM,IAAI,GAAG,UAAU,CAAC;AAE/B,MAAM,CAAC,MAAM,WAAW,GACtB,mSAAmS,CAAC;AAEtS,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7B,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IAC/D,UAAU,EAAE,CAAC;SACV,MAAM,EAAE;SACR,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,6FAA6F,CAAC;IAC1G,WAAW,EAAE,CAAC;SACX,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,8GAA8G,CAAC;IAC3H,MAAM,EAAE,CAAC;SACN,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,4EAA4E,CAAC;IACzF,OAAO,EAAE,CAAC;SACP,OAAO,EAAE;SACT,OAAO,CAAC,IAAI,CAAC;SACb,QAAQ,CAAC,4EAA4E,CAAC;IACzF,cAAc,EAAE,CAAC;SACd,MAAM,EAAE;SACR,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,gFAAgF,CAAC;IAC7F,iBAAiB,EAAE,CAAC;SACjB,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,kCAAkC,CAAC;IAC/C,QAAQ,EAAE,CAAC;SACR,OAAO,EAAE;SACT,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,sGAAsG,CAAC;IACnH,cAAc,EAAE,CAAC;SACd,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,QAAQ,EAAE;SACV,QAAQ,CAAC,4EAA4E,CAAC;IACzF,KAAK,EAAE,CAAC;SACL,MAAM,EAAE;SACR,GAAG,CAAC,cAAc,CAAC;SACnB,QAAQ,EAAE;SACV,QAAQ,CAAC,yCAAyC,CAAC;IACtD,OAAO,EAAE,CAAC;SACP,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC;SACT,GAAG,CAAC,cAAc,CAAC;SACnB,OAAO,CAAC,KAAK,CAAC;SACd,QAAQ,CAAC,2CAA2C,CAAC;IACxD,eAAe,EAAE,CAAC;SACf,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;SAC1B,GAAG,CAAC,GAAG,CAAC;SACR,QAAQ,EAAE;SACV,QAAQ,CAAC,sGAAsG,CAAC;CACpH,CAAC,CAAC;AAIH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,KAAoB;IAChD,IAAI,CAAC,CAAC,MAAM,qBAAqB,EAAE,CAAC,EAAE,CAAC;QACrC,OAAO;YACL,OAAO,EAAE,CAAC;oBACR,IAAI,EAAE,MAAe;oBACrB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK,EAAE,kGAAkG;qBAC1G,EAAE,IAAI,EAAE,CAAC,CAAC;iBACZ,CAAC;SACH,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC3C,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,IAAI,QAAQ,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAE5D,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;QAC/B,aAAa,EAAE,KAAK,CAAC,cAAc;QACnC,QAAQ;KACT,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC;QAExB,wBAAwB;QACxB,IAAI,KAAK,CAAC,eAAe,EAAE,MAAM,EAAE,CAAC;YAClC,MAAM,mBAAmB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;QACnE,CAAC;QAED,iDAAiD;QACjD,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACrB,MAAM,OAAO,GAAG,MAAM,iBAAiB,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACjE,IAAI,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC5B,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;QAED,WAAW;QACX,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAEpE,gBAAgB;QAChB,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CAAC,IAAI,EAAE;YAC/C,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ,EAAE,KAAK,CAAC,cAAc;SAC/B,CAAC,CAAC;QAEH,4CAA4C;QAC5C,gBAAgB,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAE/D,eAAe;QACf,MAAM,MAAM,GAA4B;YACtC,WAAW,EAAE,UAAU;YACvB,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE;YACf,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,KAAK,EAAE;gBACL,GAAG,QAAQ,CAAC,KAAK;gBACjB,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM;gBAC3C,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM;gBAC3C,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM;aAChC;SACF,CAAC;QAEF,MAAM,OAAO,GAA6E;YACxF,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE;SACxD,CAAC;QAEF,2CAA2C;QAC3C,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ;gBACxB,CAAC,CAAC,MAAM,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC;gBAC/C,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/C,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC5B,QAAQ,EAAE,WAAW;aACtB,CAAC,CAAC;QACL,CAAC;QAED,sCAAsC;QACtC,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACrB,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,CAAC;gBAC/C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACjC,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,OAAO,EAAE,CAAC,CAAC,OAAO;oBAClB,QAAQ,EAAE,CAAC,CAAC,QAAQ;oBACpB,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,QAAQ,EAAE,CAAC,CAAC,QAAqC;iBAClD,CAAC,CAAC,CAAC;gBACJ,MAAM,iBAAiB,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YACvE,CAAC;YAAC,MAAM,CAAC;gBACP,uDAAuD;YACzD,CAAC;QACH,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,CAAC;IACrB,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IACzB,CAAC;AACH,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "imperium-crawl",
-  "version": "1.5.3",
+  "version": "2.0.0",
   "description": "Open-source MCP server with Firecrawl-like scraping, crawling, search, and custom skills",
   "type": "module",
   "bin": {