@imenam/simple-scraper 1.0.14 → 1.0.16

package/README.md

@@ -37,11 +37,15 @@ simple-scraper
 |----------|----------|---------|-------------|
 | `PUPPETEER_HEADLESS` | No | `true` | Run Chromium in headless mode. Set to `false` to display the browser window. |
 | `PUPPETEER_TIMEOUT` | No | `30000` | Default timeout in milliseconds for page navigation and waits. |
+| `PUPPETEER_VIEWPORT_WIDTH` | No | `1280` | Viewport width in CSS pixels, applied to every page. Affects responsive layouts and lazy-loading, not just screenshots. |
+| `PUPPETEER_VIEWPORT_HEIGHT` | No | `800` | Viewport height in CSS pixels, applied to every page. |
+| `PUPPETEER_DEVICE_SCALE_FACTOR` | No | `1` | Physical pixels per CSS pixel. Set to `2` for retina-density screenshots (same framing, twice the detail, ~4× the file size). |
 | `COOKIES_DIR` | No | - | Absolute path to a folder containing Netscape-format `.txt` cookie files. All files are loaded and merged automatically before each request. |
 | `MCP_LOG_DIR` | No | `.mcp-gui/logs` | Absolute path to the directory where log files are written. |
 | `PROXY_URL` | No | - | Base URL of the [MCP HTTP Gateway](https://www.npmjs.com/package/@imenam/mcp-http-gateway). Required to enable the GUI. |
 | `PROXY_APP_PATH` | No | `/simple-scraper-mcp` | URL path under which the GUI is registered on the proxy. |
 | `PROXY_APP_NAME` | No | `Simple Scraper MCP` | Display name shown in the proxy's app list. |
+| `APP_GROUP` | No | - | Groups the app under a collapsible section (folded by default) in the proxy dashboard. Apps sharing the same value appear together; if unset, the app lands in the default "ungrouped" section. |
 | `SCRAPER_MAX_SESSIONS` | No | `5` | Maximum number of concurrent interactive sessions. |
 | `SCRAPER_SESSION_TTL_MS` | No | `600000` | Inactivity TTL for sessions in milliseconds (default: 10 minutes). Sessions unused beyond this duration are closed automatically. |
 
@@ -53,6 +57,9 @@ Copy `.env.example` to `.env` and configure the variables:
 # Puppeteer options (optional)
 PUPPETEER_HEADLESS=true
 PUPPETEER_TIMEOUT=30000
+PUPPETEER_VIEWPORT_WIDTH=1280
+PUPPETEER_VIEWPORT_HEIGHT=800
+PUPPETEER_DEVICE_SCALE_FACTOR=1
 
 # Optional: path to a folder containing Netscape-format cookie files (.txt)
 # All files in this folder will be loaded automatically before each request.
@@ -62,6 +69,8 @@ PUPPETEER_TIMEOUT=30000
 # PROXY_URL=http://localhost:3000
 # PROXY_APP_PATH=/simple-scraper-mcp
 # PROXY_APP_NAME=Simple Scraper MCP
+# Optional: groups this app under a collapsible section of the proxy dashboard
+# APP_GROUP=Scraping
 ```
 
 ## Usage with Claude Desktop
@@ -229,6 +238,7 @@ Open a persistent browser session. Returns a `session_id` to use with all `sessi
 | `url` | string | ✅ | URL to navigate to |
 | `wait_for` | string | | CSS selector to wait for before the session is considered ready |
 | `timeout` | number | | Timeout in ms (default: 30000) |
+| `viewport` | object | | Viewport override for this session, applied before navigation: `{ width, height, device_scale_factor }`. Omitted fields fall back to the server defaults. |
 
 ### `close_session`
 
@@ -308,6 +318,8 @@ Return the current full rendered HTML of the session page.
 
 Capture a screenshot of a page. Use `session_id` to capture an active session in its current state, or `url` for a one-shot capture.
 
+The capture area is resolved by precedence: `selector` → `clip` → `full_page` → the visible viewport.
+
 | Parameter | Type | Required | Description |
 |-----------|------|----------|-------------|
 | `session_id` | string | | Session ID. If provided, `url` is ignored and the current page state is captured. |
@@ -315,10 +327,12 @@ Capture a screenshot of a page. Use `session_id` to capture an active session in
 | `wait_for` | string | | CSS selector to wait for (one-shot mode only) |
 | `timeout` | number | | Timeout in ms (default: 30000) |
 | `selector` | string | | CSS selector of a specific element to capture |
-| `full_page` | boolean | | Capture the full scrollable page height (default: false, ignored when `selector` is provided) |
+| `clip` | object | | Arbitrary pixel region `{ x, y, width, height }` in CSS pixels, measured from the top-left of the page. May extend below the visible viewport. Ignored when `selector` is set; cannot be combined with `full_page`. |
+| `viewport` | object | | Viewport override for this capture: `{ width, height, device_scale_factor }`. For a session, the previous viewport is restored afterwards. |
+| `full_page` | boolean | | Capture the full scrollable page height (default: false, ignored when `selector` or `clip` is provided) |
 | `format` | `png` \| `jpeg` | | Image format (default: `png`) |
 | `output` | `inline` \| `file` \| `both` | ✅ | `inline` embeds the image in the response, `file` saves to disk and returns the path, `both` does both |
-| `path` | string | | Absolute or relative path for the saved file. Defaults to `./screenshots/screenshot-<timestamp>.<format>` |
+| `path` | string | | **Absolute** path for the saved file — relative paths are rejected, since they would resolve against the server's working directory, which the client sets unpredictably. Missing parent directories are created. Defaults to `<server_root>/screenshots/screenshot-<timestamp>.<format>` |
 
 ---
 

package/dist/browser.js

@@ -4,6 +4,29 @@ export function getDefaultTimeout() {
     const val = process.env.PUPPETEER_TIMEOUT;
     return val ? parseInt(val, 10) : 30_000;
 }
+function envNumber(name, fallback) {
+    const raw = process.env[name];
+    if (!raw)
+        return fallback;
+    const parsed = Number(raw);
+    if (!Number.isFinite(parsed) || parsed <= 0) {
+        console.error(`[Browser] Ignoring invalid ${name}="${raw}", using ${fallback}`);
+        return fallback;
+    }
+    return parsed;
+}
+/**
+ * Default viewport applied to every page. Puppeteer's own default is 800x600,
+ * which is narrow enough to trigger mobile/tablet layouts on many sites — that
+ * affects scraping (lazy-load, responsive hiding), not just screenshots.
+ */
+export function getDefaultViewport() {
+    return {
+        width: envNumber('PUPPETEER_VIEWPORT_WIDTH', 1280),
+        height: envNumber('PUPPETEER_VIEWPORT_HEIGHT', 800),
+        deviceScaleFactor: envNumber('PUPPETEER_DEVICE_SCALE_FACTOR', 1),
+    };
+}
 export async function getBrowser() {
     if (browser && browser.connected) {
         return browser;
@@ -11,6 +34,7 @@ export async function getBrowser() {
     const headless = process.env.PUPPETEER_HEADLESS !== 'false';
     browser = await puppeteer.launch({
         headless,
+        defaultViewport: getDefaultViewport(),
         args: ['--no-sandbox', '--disable-setuid-sandbox'],
     });
     console.error('[Browser] Launched Puppeteer instance');

package/dist/index.js

@@ -10,7 +10,7 @@ import { setupLogging } from './logger.js';
 import { getBrowser, closeBrowser, getDefaultTimeout } from './browser.js';
 import { loadAllCookies } from './cookies.js';
 import { GuiLauncher, reconnectGuiToolDefinition } from '@imenam/mcp-gui-interface';
-import { createSession, getSession, closeSession, closeAllSessions, listSessions, getSessionLogs, clearSessionLogs } from './sessions.js';
+import { createSession, getSession, closeSession, closeAllSessions, listSessions, getSessionLogs, clearSessionLogs, resolveViewport } from './sessions.js';
 import { registerSessionRpc } from './session-rpc.js';
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const rootDir = path.resolve(__dirname, '..');
@@ -33,6 +33,24 @@ const server = new McpServer({
     name: 'simple-scraper-mcp',
     version: '1.0.0',
 });
+// Shared viewport override. Any omitted field falls back to the env-configured
+// default (PUPPETEER_VIEWPORT_WIDTH / _HEIGHT / _DEVICE_SCALE_FACTOR).
+const viewportSchema = z
+    .object({
+    width: z.number().int().positive().optional().describe('Viewport width in CSS pixels'),
+    height: z.number().int().positive().optional().describe('Viewport height in CSS pixels'),
+    device_scale_factor: z
+        .number()
+        .positive()
+        .optional()
+        .describe('Physical pixels per CSS pixel, e.g. 2 for retina-density output (default: 1)'),
+})
+    .optional();
+function toViewportOverride(v) {
+    if (!v)
+        return undefined;
+    return { width: v.width, height: v.height, deviceScaleFactor: v.device_scale_factor };
+}
 // ─── One-shot helpers ─────────────────────────────────────────────────────────
 async function setCookies(page, cookies, pageUrl) {
     const prepared = cookies.map(c => {
@@ -329,9 +347,14 @@ server.tool('open_session', 'Open a persistent browser session on a URL. The pag
     url: z.string().url().describe('URL to navigate to when opening the session'),
     wait_for: z.string().optional().describe('CSS selector to wait for before the session is considered ready'),
     timeout: z.number().optional().describe('Timeout in milliseconds (default: 30000)'),
-}, async ({ url, wait_for, timeout }) => {
+    viewport: viewportSchema.describe('Viewport override for this session, applied before navigation. Omitted fields fall back to the server default.'),
+}, async ({ url, wait_for, timeout, viewport }) => {
     try {
-        const sessionId = await createSession(url, { waitFor: wait_for, timeout });
+        const sessionId = await createSession(url, {
+            waitFor: wait_for,
+            timeout,
+            viewport: toViewportOverride(viewport),
+        });
         return { content: [{ type: 'text', text: JSON.stringify({ session_id: sessionId }) }] };
     }
     catch (error) {
@@ -550,25 +573,50 @@ server.tool('session_scroll', 'Scroll the active session page. Use selector to s
     }
 });
 // ─── screenshot ───────────────────────────────────────────────────────────────
-server.tool('screenshot', 'Take a screenshot of a page. Provide either session_id (captures the page in its current interactive state) or url (one-shot: navigates, captures, closes). The selector parameter restricts the capture to a specific element. output controls the return format: "inline" returns a base64 image directly in the response, "file" writes the image to disk and returns the path, "both" does both.', {
+server.tool('screenshot', 'Take a screenshot of a page. Provide either session_id (captures the page in its current interactive state) or url (one-shot: navigates, captures, closes). Capture area, by precedence: selector (a specific element), clip (an arbitrary pixel region), full_page (the whole scrollable height), otherwise the visible viewport. output controls the return format: "inline" returns a base64 image directly in the response, "file" writes the image to disk and returns the path, "both" does both.', {
     session_id: z.string().optional().describe('Session ID of an active browser session. If provided, url is ignored and the current page state is captured.'),
     url: z.string().url().optional().describe('URL to navigate to for a one-shot screenshot (required if session_id is not provided)'),
     wait_for: z.string().optional().describe('CSS selector to wait for before capturing (one-shot mode only)'),
     timeout: z.number().optional().describe('Timeout in milliseconds (default: 30000)'),
     selector: z.string().optional().describe('CSS selector of a specific element to capture instead of the full page'),
-    full_page: z.boolean().optional().describe('Capture the full scrollable page height (default: false, ignored when selector is provided)'),
+    clip: z
+        .object({
+        x: z.number().describe('Left offset in CSS pixels, from the top-left of the page'),
+        y: z.number().describe('Top offset in CSS pixels, from the top-left of the page'),
+        width: z.number().positive().describe('Region width in CSS pixels'),
+        height: z.number().positive().describe('Region height in CSS pixels'),
+    })
+        .optional()
+        .describe('Arbitrary pixel region to capture. Ignored when selector is provided, and cannot be combined with full_page.'),
+    viewport: viewportSchema.describe('Viewport override for this capture. For a session, the previous viewport is restored afterwards.'),
+    full_page: z.boolean().optional().describe('Capture the full scrollable page height (default: false, ignored when selector or clip is provided)'),
     format: z.enum(['png', 'jpeg']).optional().describe('Image format (default: png)'),
     output: z.enum(['inline', 'file', 'both']).describe('Return format: "inline" embeds the image in the response, "file" saves to disk and returns the path, "both" does both'),
-    path: z.string().optional().describe('Absolute or relative file path for the saved image (used when output is "file" or "both"). Relative paths are resolved from the MCP server\'s working directory. Defaults to <server_root>/screenshots/<timestamp>.<format>'),
-}, async ({ session_id, url, wait_for, timeout, selector, full_page, format, output, path: filePath }) => {
+    path: z
+        .string()
+        .refine(p => path.isAbsolute(p), {
+        message: 'path must be absolute — a relative path would resolve against the MCP server\'s working directory, which is set by the client and not predictable.',
+    })
+        .optional()
+        .describe('Absolute file path for the saved image (used when output is "file" or "both"). Must be absolute; missing parent directories are created. Defaults to <server_root>/screenshots/screenshot-<timestamp>.<format>'),
+}, async ({ session_id, url, wait_for, timeout, selector, clip, viewport, full_page, format, output, path: filePath }) => {
     const effectiveTimeout = timeout ?? getDefaultTimeout();
     const fmt = format ?? 'png';
     const mimeType = fmt === 'jpeg' ? 'image/jpeg' : 'image/png';
+    if (clip && full_page) {
+        return { content: [{ type: 'text', text: 'Error: clip and full_page cannot be combined — clip already defines the region to capture.' }], isError: true };
+    }
     let page = null;
     let ownedPage = false;
+    // Only set for session pages, whose viewport must survive this call unchanged.
+    let previousViewport;
     try {
         if (session_id) {
             page = getSession(session_id);
+            if (viewport) {
+                previousViewport = page.viewport();
+                await page.setViewport(resolveViewport(toViewportOverride(viewport)));
+            }
         }
         else {
             if (!url) {
@@ -577,6 +625,9 @@ server.tool('screenshot', 'Take a screenshot of a page. Provide either session_i
             const browser = await getBrowser();
             page = await browser.newPage();
             ownedPage = true;
+            if (viewport) {
+                await page.setViewport(resolveViewport(toViewportOverride(viewport)));
+            }
             await applyCookies(page, url);
             await page.goto(url, { waitUntil: 'networkidle2', timeout: effectiveTimeout });
             if (wait_for) {
@@ -592,11 +643,17 @@ server.tool('screenshot', 'Take a screenshot of a page. Provide either session_i
             }
             imageBuffer = Buffer.from(await element.screenshot({ type: fmt }));
         }
+        else if (clip) {
+            // captureBeyondViewport lets the region extend past the visible area
+            // without having to scroll the page first.
+            imageBuffer = Buffer.from(await page.screenshot({ type: fmt, clip, captureBeyondViewport: true }));
+        }
         else {
             imageBuffer = Buffer.from(await page.screenshot({ type: fmt, fullPage: full_page ?? false }));
         }
         const base64 = imageBuffer.toString('base64');
-        // Resolve file path
+        // Resolve file path. filePath is guaranteed absolute by the schema; resolve()
+        // is kept only to normalise separators and any ".." segments.
         const resolvedPath = (() => {
             if (filePath)
                 return path.resolve(filePath);
@@ -624,6 +681,9 @@ server.tool('screenshot', 'Take a screenshot of a page. Provide either session_i
         if (ownedPage && page) {
             await page.close();
         }
+        else if (page && previousViewport) {
+            await page.setViewport(previousViewport).catch(() => undefined);
+        }
     }
 });
 function launchGUI() {

package/dist/sessions.js

@@ -1,4 +1,4 @@
-import { getBrowser } from './browser.js';
+import { getBrowser, getDefaultViewport } from './browser.js';
 import { loadAllCookies } from './cookies.js';
 const MAX_SESSIONS = parseInt(process.env.SCRAPER_MAX_SESSIONS ?? '5', 10);
 const TTL_MS = parseInt(process.env.SCRAPER_SESSION_TTL_MS ?? String(10 * 60 * 1000), 10);
@@ -13,6 +13,17 @@ async function applyCookies(page, pageUrl) {
     const prepared = cookies.map(c => (c.domain ? c : { ...c, url: pageUrl }));
     await page.setCookie(...prepared);
 }
+/** Merge a partial override onto the env-configured default viewport. */
+export function resolveViewport(override) {
+    const base = getDefaultViewport();
+    if (!override)
+        return base;
+    return {
+        width: override.width ?? base.width,
+        height: override.height ?? base.height,
+        deviceScaleFactor: override.deviceScaleFactor ?? base.deviceScaleFactor,
+    };
+}
 export async function createSession(url, opts = {}) {
     if (sessions.size >= MAX_SESSIONS) {
         throw new Error(`Session limit reached (max ${MAX_SESSIONS}). Close an existing session before opening a new one.`);
@@ -25,6 +36,9 @@ export async function createSession(url, opts = {}) {
         entry.consoleLogs.push({ type: msg.type(), text: msg.text(), timestamp: Date.now() });
     });
     try {
+        if (opts.viewport) {
+            await page.setViewport(resolveViewport(opts.viewport));
+        }
         await applyCookies(page, url);
         await page.goto(url, { waitUntil: 'networkidle2', timeout: opts.timeout ?? 30_000 });
         if (opts.waitFor) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@imenam/simple-scraper",
-  "version": "1.0.14",
+  "version": "1.0.16",
   "description": "MCP server for web scraping and JavaScript execution using Puppeteer",
   "type": "module",
   "main": "dist/index.js",