@yusufffararatt/dombridge-mcp 2.7.5

package/src/tools/manage-site-profile.js

@@ -0,0 +1,153 @@
+/**
+ * manage_site_profile — unified profile management tool
+ *
+ * Replaces: load_site_profile, save_site_profile, check_site_changes
+ * Actions: load | save | check | delete | list | update
+ */
+import { ProfileRepo } from '../profiles/profile-repo.js';
+import { join } from 'path';
+import { homedir } from 'os';
+import { mkdirSync } from 'fs';
+
+const DB_PATH = process.env.PROFILE_DB_PATH
+    || join(homedir(), '.dombridge', 'profiles.db');
+
+let _repo = null;
+let _dbPath = null;
+export function _setDbPath(path) {
+    if (_repo) _repo.close();
+    _repo = null;
+    _dbPath = path;
+}
+function getRepo() {
+    if (!_repo) {
+        const p = _dbPath || DB_PATH;
+        mkdirSync(join(p, '..'), { recursive: true });
+        _repo = new ProfileRepo(p);
+    }
+    return _repo;
+}
+
+const ACTIONS = ['load', 'save', 'check', 'delete', 'list', 'update'];
+
+export const manageSiteProfileTool = {
+    name: 'manage_site_profile',
+    description: 'Unified site-profile management. Replaces load_site_profile / save_site_profile / check_site_changes. Actions: load, save, check, delete, list, update.',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            action: {
+                type: 'string',
+                enum: ACTIONS,
+                description: 'Which operation to perform.'
+            },
+            domain: {
+                type: 'string',
+                description: 'Target domain (required for: save, check, delete, update).'
+            },
+            notes: { type: 'string', description: 'For save/update: human-readable notes.' },
+            framework: { type: 'string', description: 'For save/update: detected framework.' },
+            pageType: { type: 'string', description: 'For save/update: SSR/SPA/etc.' },
+            endpoints: { type: 'array', description: 'For update: incremental endpoint upsert (Bug #7 fix). Each entry: {method, url, status?}.' },
+            limit: { type: 'number', description: 'For list: max profiles to return (default 20).' }
+        },
+        required: ['action']
+    },
+    handler: manageSiteProfileHandler
+    // Actual withMetrics wrapping happens in tools/index.js
+};
+
+export async function manageSiteProfileHandler(args, bridgeClient) {
+    const { action, domain } = args || {};
+
+    // Connection guard — required for save/check which need captured endpoints
+    if (['save', 'check'].includes(action) && (!bridgeClient || !bridgeClient.isConnected)) {
+        return {
+            content: [{
+                type: 'text',
+                text: `❌ Extension not connected. Required for action: ${action}.\nOpen the target page in Chrome and retry.`
+            }],
+            isError: true
+        };
+    }
+
+    const repo = getRepo();
+
+    if (!ACTIONS.includes(action)) {
+        return {
+            content: [{ type: 'text', text: `❌ Unknown action: ${action}. Allowed: ${ACTIONS.join(', ')}` }],
+            isError: true
+        };
+    }
+
+    if (['save', 'check', 'delete', 'update'].includes(action) && !domain) {
+        return {
+            content: [{ type: 'text', text: `❌ domain_required for action: ${action}` }],
+            isError: true
+        };
+    }
+
+    switch (action) {
+        case 'load': {
+            if (!domain) {
+                const profiles = repo.list({ limit: args.limit || 20 });
+                return { content: [{ type: 'text', text: JSON.stringify({ profiles, count: profiles.length }, null, 2) }] };
+            }
+            const loaded = repo.load(domain);
+            if (!loaded) {
+                return { content: [{ type: 'text', text: `❌ Profile not found for domain: ${domain}` }], isError: true };
+            }
+            return { content: [{ type: 'text', text: JSON.stringify(loaded, null, 2) }] };
+        }
+
+        case 'save': {
+            // Pull captured endpoints from bridge (Phase 2.4 pattern)
+            const captured = bridgeClient.getCapturedEndpoints
+                ? bridgeClient.getCapturedEndpoints(domain)
+                : [];
+            const result = repo.save(domain, {
+                endpoints: captured,
+                notes: args.notes,
+                framework: args.framework,
+                pageType: args.pageType
+            });
+            const lines = [
+                `✅ Profile saved: ${domain}`,
+                `Endpoints inserted: ${result.inserted}`,
+                `Endpoints dropped (cross-domain): ${result.dropped.length}`,
+                ''
+            ];
+            if (result.dropped.length > 0) {
+                lines.push('Dropped URLs:');
+                for (const d of result.dropped.slice(0, 5)) lines.push(`  - ${d.url}`);
+            }
+            return { content: [{ type: 'text', text: lines.join('\n') }] };
+        }
+
+        case 'check': {
+            const captured = bridgeClient.getCapturedEndpoints
+                ? bridgeClient.getCapturedEndpoints(domain)
+                : [];
+            return { content: [{ type: 'text', text: JSON.stringify(repo.check(domain, captured), null, 2) }] };
+        }
+
+        case 'delete': {
+            return { content: [{ type: 'text', text: JSON.stringify({ domain, ...repo.delete(domain) }, null, 2) }] };
+        }
+
+        case 'list': {
+            const profiles = repo.list({ limit: args.limit || 20 });
+            return { content: [{ type: 'text', text: JSON.stringify({ profiles, count: profiles.length }, null, 2) }] };
+        }
+
+        case 'update': {
+            const patch = {};
+            if (args.notes !== undefined) patch.notes = args.notes;
+            if (args.framework !== undefined) patch.framework = args.framework;
+            if (args.pageType !== undefined) patch.pageType = args.pageType;
+            // Bug #7 fix: support incremental endpoint upsert
+            if (Array.isArray(args.endpoints)) patch.endpoints = args.endpoints;
+            return { content: [{ type: 'text', text: JSON.stringify({ domain, ...repo.update(domain, patch) }, null, 2) }] };
+        }
+    }
+}

package/src/tools/paginate.js

@@ -0,0 +1,444 @@
+/**
+ * Tool: paginate
+ * Multi-page and infinite scroll navigation for list/archive pages
+ */
+
+import { StateValidator } from '../utils/state-validator.js';
+import { detectPaginationStrategy } from '../utils/paginate-detector.js';
+import { runScript } from '../utils/run-script.js';
+
+export const paginateTool = {
+  name: 'paginate',
+  description: `Navigate through multi-page content or infinite scroll on a Chrome tab.
+
+WORKFLOW POSITION: 🟠 RPA — Use after identifying a list/archive page
+
+PREREQUISITES:
+- ✅ Extension must be connected
+- A page with paginated or scrollable content must be open
+
+STRATEGIES:
+- auto: Detect automatically (URL param → DOM next button → scroll behavior)
+- button: Click a "next page" button by CSS selector
+- scroll: Scroll down and wait for new content
+- url_increment: Increment page number in URL (e.g. ?page=1 → ?page=2)
+
+STOP CONDITIONS (any one triggers stop):
+- Same content appears twice in a row
+- Next button not found or disabled
+- No new elements loaded after scroll
+- Page limit reached
+- HTTP or navigation failure
+
+PARAMETERS:
+- strategy: Pagination strategy (default: 'auto')
+- selector: CSS selector for next button (required for strategy:'button')
+- limit: Max pages to navigate (default: 5, max: 50)
+- delayMs: Wait time between pages in ms (default: 1500)
+- verbose: Return per-page detail (default: false)
+- force: Skip soft guards and run even if recent scan exists (default: false)
+
+MULTI-TAB: Call debug_mcp_state() first to get tab IDs, then pass tabId to paginate a specific tab.
+
+NOTE: db_write and data pipeline features are intentionally excluded from v1.
+      Those belong in a separate roadmap item.
+
+EXAMPLE:
+paginate({ strategy: 'auto', limit: 3 })
+paginate({ strategy: 'button', selector: 'a.next-page', limit: 10 })
+paginate({ strategy: 'scroll', limit: 5, delayMs: 2000 })`,
+
+  inputSchema: {
+    type: 'object',
+    properties: {
+      strategy: {
+        type: 'string',
+        enum: ['auto', 'button', 'scroll', 'url_increment'],
+        description: "Pagination strategy. 'auto' detects automatically.",
+        default: 'auto'
+      },
+      selector: {
+        type: 'string',
+        description: "CSS selector for next-page button. Required for strategy:'button', used as hint for 'auto'."
+      },
+      limit: {
+        type: 'number',
+        description: 'Maximum pages to navigate (default: 5, max: 50)',
+        default: 5
+      },
+      delayMs: {
+        type: 'number',
+        description: 'Wait time between page navigations in ms (default: 1500)',
+        default: 1500
+      },
+      verbose: {
+        type: 'boolean',
+        description: 'Return per-page detail including element counts (default: false)',
+        default: false
+      },
+      force: {
+        type: 'boolean',
+        description: 'Skip soft guards (default: false)',
+        default: false
+      },
+      tabId: {
+        type: 'number',
+        description: 'Target tab ID (optional). Omit to use active tab. Get IDs from debug_mcp_state().'
+      }
+    },
+    required: []
+  },
+
+  handler: async (args, bridgeClient) => {
+    // Hard guard: extension must be connected
+    const connValidation = StateValidator.validateConnection(bridgeClient);
+    if (!connValidation.valid) {
+      return StateValidator.formatValidationError(connValidation);
+    }
+
+    const {
+      strategy = 'auto',
+      selector,
+      limit = 5,
+      delayMs = 1500,
+      verbose = false,
+      force: _force = false,
+      tabId
+    } = args || {};
+
+    // Validate limit
+    const safeLimit = Math.min(Math.max(1, limit), 50);
+
+    // strategy:'button' requires a selector
+    if (strategy === 'button' && !selector) {
+      return {
+        content: [{
+          type: 'text',
+          text: `❌ strategy:'button' requires a selector parameter.\n\nExample: paginate({ strategy: 'button', selector: 'a.next-page' })`
+        }]
+      };
+    }
+
+    const pages = [];
+    let currentPage = 1;
+    let stopReason = null;
+    let resolvedStrategy = strategy;
+
+    try {
+      // Step 1: If auto, detect strategy
+      if (strategy === 'auto') {
+        // Step 1a: Probe page state for pagination metadata (__PROPS, __NEXT_DATA__, __NUXT__, __INITIAL_STATE__)
+        // This handles the Trendyol-style case where widgetList.totalPages=N makes url_increment correct
+        let pageState = null;
+        let hasNextButton = false;
+        let hasInfiniteScroll = false;
+        try {
+          const probeResult = await runScript(
+            `(function() {
+              var ps = {};
+              try { if (window.__widget_list_v2__PROPS) ps.widgetList = window.__widget_list_v2__PROPS.widgetList; } catch(e) {}
+              try { if (window.__NEXT_DATA__ && window.__NEXT_DATA__.props && window.__NEXT_DATA__.props.pageProps) ps.__NEXT_DATA__ = { props: { pageProps: window.__NEXT_DATA__.props.pageProps } }; } catch(e) {}
+              try { if (window.__NUXT__ && window.__NUXT__.data) ps.pagination = window.__NUXT__.data.pagination; } catch(e) {}
+              try { if (window.__INITIAL_STATE__ && window.__INITIAL_STATE__.pagination) ps.pagination = window.__INITIAL_STATE__.pagination; } catch(e) {}
+              var nb = !!document.querySelector('a.next, [class*="next"], [aria-label*="next" i]');
+              var scrollH = document.documentElement.scrollHeight;
+              var clientH = document.documentElement.clientHeight;
+              var isc = scrollH > clientH * 2;
+              return JSON.stringify({ pageState: ps, hasNextButton: nb, hasInfiniteScroll: isc });
+            })()`,
+            bridgeClient,
+            5000
+          );
+          if (probeResult) {
+            const parsed = JSON.parse(probeResult);
+            pageState = parsed.pageState;
+            hasNextButton = parsed.hasNextButton;
+            hasInfiniteScroll = parsed.hasInfiniteScroll;
+          }
+        } catch (_e) {
+          // ignore — fall through to default detection
+        }
+
+        // Step 1b: Use the detector to choose a strategy from probed signals
+        const detectedFromMeta = detectPaginationStrategy({
+          pageState,
+          nextButtonSelector: hasNextButton ? 'a.next, [class*="next"], [aria-label*="next" i]' : null,
+          hasInfiniteScroll
+        });
+
+        if (detectedFromMeta !== 'auto') {
+          resolvedStrategy = detectedFromMeta;
+        } else {
+          // Step 1c: Fallback to the existing URL/button probe chain
+          const detected = await detectStrategy(bridgeClient, selector, tabId);
+          resolvedStrategy = detected.strategy;
+          if (!selector && detected.selector) {
+            args = { ...args, selector: detected.selector };
+          }
+        }
+      }
+
+      // Step 2: Get initial element count as baseline
+      const initialCount = await getElementCount(bridgeClient, tabId);
+
+      while (currentPage <= safeLimit) {
+        const pageResult = { page: currentPage };
+
+        // Execute one pagination step
+        const stepResult = await executePaginationStep({
+          strategy: resolvedStrategy,
+          selector: args?.selector || selector,
+          delayMs,
+          bridgeClient,
+          tabId
+        });
+
+        if (!stepResult.success) {
+          stopReason = stepResult.reason;
+          break;
+        }
+
+        // Wait for content to load
+        // Scroll strategy: poll until element count changes (early exit) or timeout
+        // Other strategies: fixed sleep is sufficient
+        const prevCount = pages.length > 0 ? pages[pages.length - 1].elementCount : initialCount;
+        const newCount = resolvedStrategy === 'scroll'
+          ? await pollUntilNewContent(prevCount, delayMs, bridgeClient, tabId)
+          : (await sleep(delayMs), await getElementCount(bridgeClient, tabId));
+        pageResult.elementCount = newCount;
+        pageResult.newElements = Math.max(0, newCount - initialCount);
+
+        // Stop condition: no new content. Distinguish between:
+        //   - page_end: scroll has reached document bottom (no more to load)
+        //   - no_scroll_progress: scrolled but new content didn't load (slow or stuck)
+        if (pages.length > 0) {
+          const lastCount = pages[pages.length - 1].elementCount;
+          if (newCount <= lastCount) {
+            const scrollState = await getScrollState(bridgeClient, tabId);
+            stopReason = (scrollState && scrollState.atBottom)
+              ? 'page_end'
+              : 'no_scroll_progress';
+            pageResult.scrollState = scrollState;
+            break;
+          }
+        }
+
+        pages.push(pageResult);
+        currentPage++;
+      }
+
+      if (!stopReason && currentPage > safeLimit) {
+        stopReason = 'limit_reached';
+      }
+
+      // Build response
+      const totalNewElements = pages.length > 0
+        ? Math.max(0, (pages[pages.length - 1].elementCount || 0) - initialCount)
+        : 0;
+
+      let output = `## Pagination Complete\n\n`;
+      output += `**Strategy:** ${resolvedStrategy}\n`;
+      output += `**Pages navigated:** ${pages.length}\n`;
+      output += `**New elements loaded:** ${totalNewElements}\n`;
+      output += `**Stop reason:** ${formatStopReason(stopReason)}\n`;
+
+      if (verbose && pages.length > 0) {
+        output += `\n### Per-page detail\n`;
+        for (const p of pages) {
+          output += `- Page ${p.page}: ${p.elementCount} elements total (+${p.newElements} new)\n`;
+        }
+      }
+
+      if (pages.length === 0) {
+        output += `\n💡 No pages were navigated. The page may not have pagination or the strategy did not match.\n`;
+        output += `Try: paginate({ strategy: 'auto', verbose: true }) or specify a selector manually.\n`;
+      }
+
+      return { content: [{ type: 'text', text: output }] };
+
+    } catch (error) {
+      return {
+        isError: true,
+        content: [{
+          type: 'text',
+          text: `❌ Pagination error: ${error.message}\n\nREQUIRED STEPS:\n1. Verify pagination selector and strategy parameters\n2. Make sure the tab is connected and active`
+        }]
+      };
+    }
+  }
+};
+
+// ── Helpers ────────────────────────────────────────────────────────────────────
+
+async function detectStrategy(bridgeClient, hintSelector, tabId) {
+  // 1. Check URL for page param
+  const urlCheck = await executeJs(
+    `(function() {
+      const url = window.location.href;
+      const params = new URLSearchParams(window.location.search);
+      for (const [k, v] of params) {
+        if (/page|p|offset/i.test(k) && /^\\d+$/.test(v)) return { type: 'url', key: k, value: v };
+      }
+      return null;
+    })()`,
+    bridgeClient, tabId
+  );
+  if (urlCheck?.type === 'url') {
+    return { strategy: 'url_increment' };
+  }
+
+  // 2. Check DOM for next button
+  const buttonCheck = await executeJs(
+    `(function() {
+      const candidates = [
+        'a[aria-label*="next" i]', 'a[class*="next" i]', 'button[class*="next" i]',
+        'a[class*="sonraki" i]', 'button[class*="sonraki" i]',
+        '[data-testid*="next" i]', '.pagination a:last-child', 'li.next a'
+      ];
+      for (const sel of candidates) {
+        const el = document.querySelector(sel);
+        if (el && !el.disabled && !el.classList.contains('disabled')) {
+          return { type: 'button', selector: sel };
+        }
+      }
+      return null;
+    })()`,
+    bridgeClient, tabId
+  );
+  if (buttonCheck?.type === 'button') {
+    return { strategy: 'button', selector: hintSelector || buttonCheck.selector };
+  }
+
+  // 3. Default to scroll
+  return { strategy: 'scroll' };
+}
+
+async function executePaginationStep({ strategy, selector, delayMs: _delayMs, bridgeClient, tabId }) {
+  switch (strategy) {
+    case 'button': {
+      const result = await executeJs(
+        `(function() {
+          const el = document.querySelector(${JSON.stringify(selector)});
+          if (!el) return { ok: false, reason: 'button_not_found' };
+          if (el.disabled || el.classList.contains('disabled') || el.getAttribute('aria-disabled') === 'true') {
+            return { ok: false, reason: 'button_disabled' };
+          }
+          el.click();
+          return { ok: true };
+        })()`,
+        bridgeClient, tabId
+      );
+      if (!result?.ok) return { success: false, reason: result?.reason || 'button_error' };
+      return { success: true };
+    }
+
+    case 'scroll': {
+      await executeJs(
+        `window.scrollTo({ top: document.body.scrollHeight, behavior: 'smooth' })`,
+        bridgeClient, tabId
+      );
+      return { success: true };
+    }
+
+    case 'url_increment': {
+      const result = await executeJs(
+        `(function() {
+          const url = new URL(window.location.href);
+          const params = url.searchParams;
+          for (const [k, v] of params) {
+            if (/page|p|offset/i.test(k) && /^\\d+$/.test(v)) {
+              params.set(k, String(parseInt(v) + 1));
+              window.location.href = url.toString();
+              return { ok: true };
+            }
+          }
+          return { ok: false, reason: 'no_page_param' };
+        })()`,
+        bridgeClient, tabId
+      );
+      if (!result?.ok) return { success: false, reason: result?.reason || 'url_increment_failed' };
+      return { success: true };
+    }
+
+    default:
+      return { success: false, reason: 'unknown_strategy' };
+  }
+}
+
+async function getElementCount(bridgeClient, tabId) {
+  const result = await executeJs(
+    `document.querySelectorAll('li, article, ytd-rich-item-renderer, ytd-video-renderer, ytd-grid-video-renderer, [class*="item"], [class*="card"], [class*="product"], [class*="result"], [class*="video"]').length`,
+    bridgeClient, tabId
+  );
+  return typeof result === 'number' ? result : 0;
+}
+
+// Detect whether the viewport has reached the bottom of the scrollable area.
+// Useful to distinguish "page ended" from "scrolled but content didn't load".
+async function getScrollState(bridgeClient, tabId) {
+  const result = await executeJs(
+    `JSON.stringify({
+      scrollY: window.scrollY,
+      scrollHeight: document.documentElement.scrollHeight,
+      clientHeight: document.documentElement.clientHeight,
+      atBottom: (window.scrollY + document.documentElement.clientHeight) >= (document.documentElement.scrollHeight - 4)
+    })`,
+    bridgeClient, tabId
+  );
+  if (typeof result === 'string') {
+    try { return JSON.parse(result); } catch { return null; }
+  }
+  return null;
+}
+
+async function executeJs(code, bridgeClient, tabId) {
+  try {
+    const requestId = `paginate-js-${Date.now()}-${Math.floor(Math.random() * 1000)}`;
+    await bridgeClient.queueRequest('execute-js', {
+      code,
+      timeout: 5000,
+      id: requestId,
+      context: 'page',
+      ...(tabId !== undefined ? { tabId } : {})
+    });
+
+    const record = await bridgeClient.waitForResult('js-execution', requestId, 10000);
+    if (record && record.result !== undefined) {
+      return record.result;
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+async function pollUntilNewContent(baseCount, maxWaitMs, bridgeClient, tabId) {
+  const deadline = Date.now() + maxWaitMs;
+  await sleep(400); // allow scroll animation to reach viewport trigger
+  while (Date.now() < deadline) {
+    const count = await getElementCount(bridgeClient, tabId);
+    if (count > baseCount) return count; // new content detected early
+    await sleep(400);
+  }
+  return await getElementCount(bridgeClient, tabId);
+}
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+function formatStopReason(reason) {
+  const map = {
+    no_new_content: 'No new content loaded',
+    page_end: 'Reached page end (scroll at bottom — list fully loaded)',
+    no_scroll_progress: 'Scrolled but no new content loaded (page may be slow or stuck — try longer delayMs or different selector)',
+    button_not_found: 'Next button not found',
+    button_disabled: 'Next button is disabled',
+    limit_reached: 'Page limit reached',
+    no_page_param: 'No page parameter in URL',
+    url_increment_failed: 'URL increment failed',
+    unknown_strategy: 'Unknown strategy',
+    null: 'Completed normally'
+  };
+  return map[reason] || reason || 'Completed normally';
+}