@imsovikde/source-grab 1.0.1

package/background/service-worker.js

@@ -0,0 +1,426 @@
+// background/service-worker.js
+// v1.1 — Persistent data, onSuspend flush, deeper scan, stop-scroll, no auto-clear
+
+import { CaptureRegistry } from './capture-registry.js';
+import { DebuggerCapture } from './debugger-capture.js';
+import { DownloadEngine } from './download-engine.js';
+
+const registry = new CaptureRegistry();
+const debuggerCapture = new DebuggerCapture(registry);
+const downloadEngine = new DownloadEngine(registry, debuggerCapture);
+
+// Track which tabs have active capture (in-memory, rehydrated from storage)
+const activeTabs = new Set();
+let keepAliveInterval = null;
+
+function startKeepAlive() {
+  if (keepAliveInterval) return;
+  keepAliveInterval = setInterval(() => chrome.runtime.getPlatformInfo(() => {}), 25000);
+}
+
+function stopKeepAlive() {
+  if (keepAliveInterval) {
+    clearInterval(keepAliveInterval);
+    keepAliveInterval = null;
+  }
+}
+
+// ─── Synchronous top-level event registration (CRITICAL for MV3) ───
+
+chrome.runtime.onInstalled.addListener(() => {
+  chrome.sidePanel.setPanelBehavior({ openPanelOnActionClick: true });
+});
+
+chrome.runtime.onStartup.addListener(async () => {
+  await registry.init();
+  // Rehydrate active tab set from session storage (capture state is session-scoped)
+  const allData = await chrome.storage.session.get(null);
+  for (const [key, value] of Object.entries(allData)) {
+    if (key.startsWith('capture_state_') && value?.active) {
+      const tabId = parseInt(key.replace('capture_state_', ''), 10);
+      activeTabs.add(tabId);
+    }
+  }
+  if (activeTabs.size > 0) startKeepAlive();
+});
+
+// Eagerly init on first module load too (not just onStartup)
+registry.init().then(() => {
+  chrome.storage.session.get(null).then(allData => {
+    for (const [key, value] of Object.entries(allData)) {
+      if (key.startsWith('capture_state_') && value?.active) {
+        const tabId = parseInt(key.replace('capture_state_', ''), 10);
+        activeTabs.add(tabId);
+      }
+    }
+    if (activeTabs.size > 0) startKeepAlive();
+  });
+});
+
+// ─── Service worker suspend — flush all pending data ───
+
+chrome.runtime.onSuspend.addListener(() => {
+  registry.flushAll();
+});
+
+// ─── Tab lifecycle ───
+
+chrome.tabs.onRemoved.addListener((tabId) => {
+  activeTabs.delete(tabId);
+  // DON'T clear registry on tab close — user must explicitly clear.
+  // Only clear the session-scoped capture state.
+  chrome.storage.session.remove(`capture_state_${tabId}`);
+  chrome.storage.session.remove(`download_progress_${tabId}`);
+  if (activeTabs.size === 0) stopKeepAlive();
+});
+
+chrome.tabs.onUpdated.addListener(async (tabId, changeInfo, tab) => {
+  if (changeInfo.url && activeTabs.has(tabId)) {
+    await registry.addNavigationMarker(tabId, changeInfo.url);
+
+    // Update capture state URL
+    const stateKey = `capture_state_${tabId}`;
+    const stateData = await chrome.storage.session.get(stateKey);
+    if (stateData[stateKey]) {
+      await chrome.storage.session.set({
+        [stateKey]: { ...stateData[stateKey], tabUrl: changeInfo.url }
+      });
+    }
+
+    // Re-inject content scripts on navigation
+    try {
+      await chrome.scripting.executeScript({ target: { tabId, allFrames: true }, files: ['content-scripts/main-world-hook.js'], world: 'MAIN', injectImmediately: true });
+      await chrome.scripting.executeScript({ target: { tabId, allFrames: true }, files: ['content-scripts/isolated-bridge.js'], world: 'ISOLATED', injectImmediately: true });
+    } catch(e) { /* Tab may not be ready yet */ }
+
+    // Re-scan resource tree after navigation if debugger is attached
+    const state = stateData[stateKey];
+    if (state?.deepScan) {
+      setTimeout(async () => {
+        try { await debuggerCapture.getResourceTree(tabId); } catch(e) {}
+      }, 1500); // Wait for page to load resources
+    }
+  }
+});
+
+// ─── webRequest listeners (fire only after host_permissions granted) ───
+
+chrome.webRequest.onBeforeRequest.addListener(
+  (details) => {
+    if (details.tabId >= 0 && activeTabs.has(details.tabId)) {
+      registry.upsert(details.tabId, {
+        url: details.url,
+        source: 'webRequest',
+        status: 'requested',
+        resourceType: details.type
+      });
+    }
+  },
+  { urls: ['<all_urls>'] }
+);
+
+chrome.webRequest.onResponseStarted.addListener(
+  (details) => {
+    if (details.tabId >= 0 && activeTabs.has(details.tabId)) {
+      let size = null;
+      const cl = details.responseHeaders?.find(h => h.name.toLowerCase() === 'content-length');
+      if (cl) size = parseInt(cl.value, 10);
+      const ct = details.responseHeaders?.find(h => h.name.toLowerCase() === 'content-type');
+
+      registry.upsert(details.tabId, {
+        url: details.url,
+        statusCode: details.statusCode,
+        mimeType: ct ? ct.value.split(';')[0].trim() : null,
+        size: isNaN(size) ? null : size,
+        resourceType: details.type
+      });
+    }
+  },
+  { urls: ['<all_urls>'] },
+  ['responseHeaders']
+);
+
+chrome.webRequest.onCompleted.addListener(
+  (details) => {
+    if (details.tabId >= 0 && activeTabs.has(details.tabId)) {
+      registry.upsert(details.tabId, {
+        url: details.url,
+        status: 'completed'
+      });
+    }
+  },
+  { urls: ['<all_urls>'] }
+);
+
+chrome.webRequest.onErrorOccurred.addListener(
+  (details) => {
+    if (details.tabId >= 0 && activeTabs.has(details.tabId)) {
+      registry.upsert(details.tabId, {
+        url: details.url,
+        status: 'error'
+      });
+    }
+  },
+  { urls: ['<all_urls>'] }
+);
+
+// ─── Message router ───
+
+chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
+  // Content script resource detection
+  if (msg.type === 'RESOURCE_DETECTED') {
+    if (sender.tab && sender.tab.id && activeTabs.has(sender.tab.id)) {
+      const payload = msg.payload;
+      const tabId = sender.tab.id;
+
+      if (payload.type === 'RESOURCE_BLOB_DATA') {
+        return;
+      }
+
+      const entry = { url: payload.url, source: payload.source || 'domHook' };
+      if (payload.statusCode) entry.statusCode = payload.statusCode;
+      if (payload.mimeType) entry.mimeType = payload.mimeType;
+      if (payload.size) entry.size = payload.size;
+      if (payload.filename) entry.filename = payload.filename;
+      if (payload.type === 'FETCH_RESPONSE' || payload.type === 'XHR_RESPONSE') {
+        entry.status = 'completed';
+      }
+      if (payload.type === 'INLINE_SVG') {
+        entry.mimeType = 'image/svg+xml';
+        entry.status = 'completed';
+        entry.domContext = 'inline-svg';
+      }
+      if (payload.type === 'CSS_BACKGROUND') {
+        entry.domContext = 'css-background';
+      }
+      if (payload.type === 'DOM_RESOURCE') {
+        entry.domContext = payload.domContext || 'mutation';
+      }
+
+      registry.upsert(tabId, entry);
+    }
+    return;
+  }
+
+  // ─── Side Panel messages ───
+
+  if (msg.type === 'START_CAPTURE') {
+    (async () => {
+      try {
+        const tabId = msg.tabId;
+        activeTabs.add(tabId);
+
+        // Inject content scripts into the current tab
+        try {
+          await chrome.scripting.executeScript({ target: { tabId, allFrames: true }, files: ['content-scripts/main-world-hook.js'], world: 'MAIN', injectImmediately: true });
+        } catch(e) { console.warn('Main hook inject:', e.message); }
+        try {
+          await chrome.scripting.executeScript({ target: { tabId, allFrames: true }, files: ['content-scripts/isolated-bridge.js'], world: 'ISOLATED', injectImmediately: true });
+        } catch(e) { console.warn('Bridge inject:', e.message); }
+
+        // Register for future navigations (may already be registered, so try-catch)
+        try {
+          await chrome.scripting.registerContentScripts([
+            { id: 'sg-main-hook', js: ['content-scripts/main-world-hook.js'], matches: ['<all_urls>'], runAt: 'document_start', world: 'MAIN', allFrames: true },
+            { id: 'sg-bridge', js: ['content-scripts/isolated-bridge.js'], matches: ['<all_urls>'], runAt: 'document_start', world: 'ISOLATED', allFrames: true }
+          ]);
+        } catch (regErr) {
+          console.log('Content scripts already registered:', regErr.message);
+        }
+
+        // Attempt debugger attach (debugger is now a required permission)
+        let deepScan = false;
+        try {
+          await debuggerCapture.attach(tabId);
+          deepScan = true;
+
+          // Schedule a second resource tree scan after 3s for late-loading resources
+          setTimeout(async () => {
+            try { await debuggerCapture.getResourceTree(tabId); } catch(e) {}
+          }, 3000);
+        } catch (e) {
+          console.warn('Debugger attach failed:', e);
+        }
+
+        const tab = await chrome.tabs.get(tabId);
+        await chrome.storage.session.set({
+          [`capture_state_${tabId}`]: { active: true, deepScan, tabUrl: tab.url, startTime: Date.now() }
+        });
+
+        startKeepAlive();
+        sendResponse({ success: true });
+      } catch (e) {
+        sendResponse({ success: false, error: e.message });
+      }
+    })();
+    return true;
+  }
+
+  if (msg.type === 'STOP_CAPTURE') {
+    (async () => {
+      try {
+        const tabId = msg.tabId;
+        activeTabs.delete(tabId);
+
+        // Detach debugger
+        await debuggerCapture.detach(tabId);
+
+        // Unregister content scripts
+        try {
+          await chrome.scripting.unregisterContentScripts({ ids: ['sg-main-hook', 'sg-bridge'] });
+        } catch (e) { /* may not be registered */ }
+
+        // Update state — mark as inactive but DON'T clear registry
+        const stateKey = `capture_state_${tabId}`;
+        const data = await chrome.storage.session.get(stateKey);
+        if (data[stateKey]) {
+          await chrome.storage.session.set({
+            [stateKey]: { ...data[stateKey], active: false }
+          });
+        }
+
+        // Flush any pending registry writes to ensure data is saved
+        await registry.flushAll();
+
+        if (activeTabs.size === 0) stopKeepAlive();
+        sendResponse({ success: true });
+      } catch (e) {
+        sendResponse({ success: false, error: e.message });
+      }
+    })();
+    return true;
+  }
+
+  if (msg.type === 'TOGGLE_DEEP_SCAN') {
+    (async () => {
+      try {
+        const tabId = msg.tabId;
+        const enabled = msg.enabled;
+
+        if (enabled) {
+          try {
+            await debuggerCapture.attach(tabId);
+            // Immediate + delayed resource tree scan for thorough coverage
+            setTimeout(async () => {
+              try { await debuggerCapture.getResourceTree(tabId); } catch(e) {}
+            }, 2000);
+          } catch(e) {
+            console.warn('Debugger attach for deep scan:', e);
+          }
+        } else {
+          await debuggerCapture.detach(tabId);
+        }
+
+        // Update capture state
+        const stateKey = `capture_state_${tabId}`;
+        const data = await chrome.storage.session.get(stateKey);
+        if (data[stateKey]) {
+          await chrome.storage.session.set({
+            [stateKey]: { ...data[stateKey], deepScan: enabled }
+          });
+        }
+
+        sendResponse({ success: true });
+      } catch (e) {
+        sendResponse({ success: false, error: e.message });
+      }
+    })();
+    return true;
+  }
+
+  if (msg.type === 'DOWNLOAD_SELECTED') {
+    (async () => {
+      try {
+        const batchId = await downloadEngine.downloadBatch(msg.tabId, msg.urls);
+        sendResponse({ batchId });
+      } catch (e) {
+        sendResponse({ error: e.message });
+      }
+    })();
+    return true;
+  }
+
+  if (msg.type === 'DOWNLOAD_ZIP') {
+    (async () => {
+      try {
+        const res = await downloadEngine.downloadZip(msg.tabId, msg.urls, msg.zipFilename);
+        sendResponse(res);
+      } catch (e) {
+        sendResponse({ success: false, error: e.message });
+      }
+    })();
+    return true;
+  }
+
+  if (msg.type === 'AUTO_SCROLL') {
+    (async () => {
+      try {
+        await chrome.scripting.executeScript({ target: { tabId: msg.tabId }, files: ['content-scripts/auto-scroll.js'], world: 'MAIN' });
+        sendResponse({ success: true });
+      } catch (e) {
+        sendResponse({ success: false, error: e.message });
+      }
+    })();
+    return true;
+  }
+
+  if (msg.type === 'STOP_AUTO_SCROLL') {
+    (async () => {
+      try {
+        await chrome.scripting.executeScript({
+          target: { tabId: msg.tabId },
+          func: () => { window.__sgStopScroll = true; },
+          world: 'MAIN'
+        });
+        sendResponse({ success: true });
+      } catch (e) {
+        sendResponse({ success: false, error: e.message });
+      }
+    })();
+    return true;
+  }
+
+  if (msg.type === 'CLEAR_CAPTURE') {
+    (async () => {
+      try {
+        await registry.clear(msg.tabId);
+        sendResponse({ success: true });
+      } catch (e) {
+        sendResponse({ success: false, error: e.message });
+      }
+    })();
+    return true;
+  }
+
+  if (msg.type === 'RETRY_DOWNLOAD') {
+    (async () => {
+      try {
+        await downloadEngine.downloadBatch(msg.tabId, [msg.url]);
+        sendResponse({ success: true });
+      } catch (e) {
+        sendResponse({ success: false, error: e.message });
+      }
+    })();
+    return true;
+  }
+
+  // GET_REGISTRY — side panel requests full registry data for a tab
+  if (msg.type === 'GET_REGISTRY') {
+    (async () => {
+      try {
+        const tabId = msg.tabId;
+        const data = await chrome.storage.local.get(`registry_${tabId}`);
+        sendResponse({ registry: data[`registry_${tabId}`] || null });
+      } catch (e) {
+        sendResponse({ registry: null });
+      }
+    })();
+    return true;
+  }
+});
+
+// ─── Downloads tracker ───
+
+chrome.downloads.onChanged.addListener((delta) => {
+  // DownloadEngine tracks its own downloads internally via chromeDownload()
+});

package/content-scripts/auto-scroll.js

@@ -0,0 +1,51 @@
+// content-scripts/auto-scroll.js
+// v1.1 — Stoppable auto-scroll via window.__sgStopScroll flag
+
+(function() {
+  // Reset stop flag
+  window.__sgStopScroll = false;
+
+  const SCROLL_STEP = 400;
+  const SCROLL_INTERVAL = 600;
+  const IDLE_THRESHOLD = 3;
+  const MAX_SCROLLS = 500;
+
+  let scrollCount = 0;
+  let lastHeight = document.documentElement.scrollHeight;
+  let idleCount = 0;
+
+  const interval = setInterval(() => {
+    // Check stop flag
+    if (window.__sgStopScroll) {
+      clearInterval(interval);
+      window.__sgStopScroll = false;
+      return;
+    }
+
+    const currentHeight = document.documentElement.scrollHeight;
+    const scrollTop = window.scrollY || document.documentElement.scrollTop;
+    const clientHeight = window.innerHeight;
+
+    // At the bottom?
+    if (scrollTop + clientHeight >= currentHeight - 10) {
+      if (currentHeight === lastHeight) {
+        idleCount++;
+      } else {
+        idleCount = 0;
+        lastHeight = currentHeight;
+      }
+
+      if (idleCount >= IDLE_THRESHOLD) {
+        clearInterval(interval);
+        return;
+      }
+    }
+
+    window.scrollBy({ top: SCROLL_STEP, behavior: 'smooth' });
+    scrollCount++;
+
+    if (scrollCount >= MAX_SCROLLS) {
+      clearInterval(interval);
+    }
+  }, SCROLL_INTERVAL);
+})();

package/content-scripts/isolated-bridge.js

@@ -0,0 +1,36 @@
+// content-scripts/isolated-bridge.js
+// Runs in ISOLATED world. Relays messages between MAIN world and service worker.
+
+(function() {
+  if (window.__SOURCEGRAB_BRIDGE__) return;
+  window.__SOURCEGRAB_BRIDGE__ = true;
+
+  const CAPTURE_CHANNEL = '__SOURCEGRAB_CAPTURE__';
+  const CMD_CHANNEL = '__SOURCEGRAB_COMMAND__';
+
+  // MAIN world → Service Worker
+  window.addEventListener('message', (event) => {
+    if (event.source !== window) return;
+    if (event.data?.channel !== CAPTURE_CHANNEL) return;
+
+    try {
+      chrome.runtime.sendMessage({
+        type: 'RESOURCE_DETECTED',
+        payload: event.data.payload
+      });
+    } catch (e) {
+      // Extension context may be invalidated (e.g., extension reloaded)
+    }
+  });
+
+  // Service Worker → MAIN world
+  chrome.runtime.onMessage.addListener((msg, sender, sendResponse) => {
+    if (msg.type === 'COMMAND_TO_PAGE') {
+      window.postMessage({
+        channel: CMD_CHANNEL,
+        command: msg.command,
+        payload: msg.payload
+      }, '*');
+    }
+  });
+})();