@imsovikde/source-grab 1.0.1

package/background/debugger-capture.js

@@ -0,0 +1,171 @@
+// background/debugger-capture.js
+// v1.1 — Deeper scan: periodic re-scan, stylesheet sub-resource extraction
+
+export class DebuggerCapture {
+  constructor(registry) {
+    this.registry = registry;
+    this.attachedTabs = new Set();
+    this.boundOnEvent = this.onDebuggerEvent.bind(this);
+    this.boundOnDetach = this.onDetach.bind(this);
+
+    if (typeof chrome !== 'undefined' && chrome.debugger) {
+      if (chrome.debugger.onEvent) {
+        chrome.debugger.onEvent.addListener(this.boundOnEvent);
+      }
+      if (chrome.debugger.onDetach) {
+        chrome.debugger.onDetach.addListener(this.boundOnDetach);
+      }
+    }
+  }
+
+  async attach(tabId) {
+    if (!chrome?.debugger) return;
+    if (this.attachedTabs.has(tabId)) return; // Already attached
+    try {
+      await chrome.debugger.attach({ tabId }, '1.3');
+      this.attachedTabs.add(tabId);
+      await chrome.debugger.sendCommand({ tabId }, 'Network.enable');
+      await chrome.debugger.sendCommand({ tabId }, 'Page.enable');
+      // Also enable CSS domain for stylesheet sub-resource tracking
+      try { await chrome.debugger.sendCommand({ tabId }, 'CSS.enable'); } catch(e) {}
+
+      // Immediate full resource tree scan
+      await this.getResourceTree(tabId);
+    } catch (e) {
+      console.error('Debugger attach failed:', e, chrome.runtime?.lastError);
+      this.attachedTabs.delete(tabId);
+    }
+  }
+
+  async getResourceTree(tabId) {
+    if (!chrome?.debugger || !this.attachedTabs.has(tabId)) return;
+    try {
+      const response = await chrome.debugger.sendCommand({ tabId }, 'Page.getResourceTree');
+      if (response && response.frameTree) {
+        this.walkFrameTree(tabId, response.frameTree);
+      }
+    } catch (e) {
+      console.error('Failed to get resource tree:', e);
+    }
+  }
+
+  walkFrameTree(tabId, frameTree) {
+    if (frameTree.resources) {
+      for (const res of frameTree.resources) {
+        // Skip chrome-extension resources
+        if (res.url && (res.url.startsWith('chrome-extension://') || res.url.startsWith('chrome://'))) continue;
+
+        this.registry.upsert(tabId, {
+          url: res.url,
+          mimeType: res.mimeType,
+          source: 'resourceTree',
+          frameId: frameTree.frame?.id,
+          status: 'completed'
+        });
+      }
+    }
+    if (frameTree.childFrames) {
+      for (const child of frameTree.childFrames) {
+        this.walkFrameTree(tabId, child);
+      }
+    }
+  }
+
+  onDebuggerEvent(source, method, params) {
+    const tabId = source.tabId;
+    if (!tabId || !this.attachedTabs.has(tabId)) return;
+
+    if (method === 'Network.requestWillBeSent') {
+      // Skip chrome-extension and data URLs from network capture
+      const reqUrl = params.request?.url;
+      if (!reqUrl || reqUrl.startsWith('chrome-extension://') || reqUrl.startsWith('data:')) return;
+
+      this.registry.upsert(tabId, {
+        url: reqUrl,
+        requestId: params.requestId,
+        timestamp: Date.now(),
+        source: 'cdp'
+      });
+    } else if (method === 'Network.responseReceived') {
+      const respUrl = params.response?.url;
+      if (!respUrl || respUrl.startsWith('chrome-extension://')) return;
+
+      let size = null;
+      if (params.response.headers && params.response.headers['content-length']) {
+        size = parseInt(params.response.headers['content-length'], 10);
+      }
+      this.registry.upsert(tabId, {
+        url: respUrl,
+        requestId: params.requestId,
+        statusCode: params.response.status,
+        mimeType: params.response.mimeType,
+        size: isNaN(size) ? null : size
+      });
+    } else if (method === 'Network.loadingFinished') {
+      this.updateByRequestId(tabId, params.requestId, {
+        status: 'completed',
+        size: params.encodedDataLength || null
+      });
+    } else if (method === 'Network.dataReceived') {
+      // track cumulative dataLength (informational)
+    } else if (method === 'CSS.styleSheetAdded') {
+      // Deeper scan: capture fonts/images referenced inside stylesheets
+      const header = params.header;
+      if (header?.sourceURL && !header.sourceURL.startsWith('chrome-extension://')) {
+        this.registry.upsert(tabId, {
+          url: header.sourceURL,
+          mimeType: 'text/css',
+          source: 'cdp',
+          status: 'completed'
+        });
+      }
+    }
+  }
+
+  async updateByRequestId(tabId, requestId, partial) {
+    const entries = await this.registry.getAll(tabId);
+    const entry = entries.find(e => e.requestId === requestId);
+    if (entry) {
+      partial.url = entry.originalUrl;
+      this.registry.upsert(tabId, partial);
+    }
+  }
+
+  async getResponseBody(tabId, requestId) {
+    if (!chrome?.debugger || !this.attachedTabs.has(tabId)) return null;
+    try {
+      const res = await chrome.debugger.sendCommand({ tabId }, 'Network.getResponseBody', { requestId });
+      return res;
+    } catch (e) {
+      console.error('getResponseBody failed:', e);
+      return null;
+    }
+  }
+
+  async getResourceContent(tabId, frameId, url) {
+    if (!chrome?.debugger || !this.attachedTabs.has(tabId)) return null;
+    try {
+      const res = await chrome.debugger.sendCommand({ tabId }, 'Page.getResourceContent', { frameId, url });
+      return res;
+    } catch (e) {
+      console.error('getResourceContent failed:', e);
+      return null;
+    }
+  }
+
+  async detach(tabId) {
+    if (!chrome?.debugger) return;
+    this.attachedTabs.delete(tabId);
+    try {
+      await chrome.debugger.detach({ tabId });
+    } catch (e) {
+      // Already detached — fine
+    }
+  }
+
+  onDetach(source, reason) {
+    if (source.tabId) {
+      this.attachedTabs.delete(source.tabId);
+    }
+  }
+}

package/background/download-engine.js

@@ -0,0 +1,316 @@
+// background/download-engine.js
+
+export class DownloadEngine {
+  constructor(registry, debuggerCapture) {
+    this.registry = registry;
+    this.debuggerCapture = debuggerCapture;
+    this.CONCURRENCY_LIMIT = 5;
+    this.RETRY_LIMIT = 3;
+    this.MAX_ZIP_SIZE = 100 * 1024 * 1024;
+  }
+
+  async downloadBatch(tabId, urls) {
+    const batchId = Date.now();
+    
+    const items = urls.map(url => ({
+      url,
+      filename: this.registry.extractFilename(url),
+      status: 'queued'
+    }));
+
+    await chrome.storage.session.set({
+      [`download_progress_${tabId}`]: {
+        batchId,
+        total: urls.length,
+        completed: 0,
+        failed: 0,
+        items
+      }
+    });
+
+    const queue = [...urls];
+    const workers = [];
+
+    const processQueue = async () => {
+      while (queue.length > 0) {
+        const url = queue.shift();
+        await this.processItem(tabId, url, batchId);
+      }
+    };
+
+    for (let i = 0; i < this.CONCURRENCY_LIMIT; i++) {
+      workers.push(processQueue());
+    }
+
+    await Promise.all(workers);
+    await this.registry.markBatchDownloaded(tabId, urls, batchId);
+  }
+
+  async processItem(tabId, url, batchId, retries = 0) {
+    let success = false;
+    let errorMsg = null;
+    try {
+      const entries = await this.registry.getAll(tabId);
+      const normalized = this.registry.normalizeUrl(url);
+      const entry = entries.find(e => e.url === normalized);
+
+      const filename = entry ? entry.filename : this.registry.extractFilename(url);
+      const path = await this.buildMirroredPath(filename);
+
+      if (entry && entry.requestId) {
+        const bodyRes = await this.debuggerCapture.getResponseBody(tabId, entry.requestId);
+        if (bodyRes && bodyRes.body) {
+          let dataUrl;
+          if (bodyRes.base64Encoded) {
+            dataUrl = `data:${entry.mimeType || 'application/octet-stream'};base64,${bodyRes.body}`;
+          } else {
+            dataUrl = `data:${entry.mimeType || 'text/plain'};charset=utf-8,${encodeURIComponent(bodyRes.body)}`;
+          }
+          await this.chromeDownload(dataUrl, path);
+          success = true;
+        }
+      }
+
+      if (!success) {
+        try {
+          await this.chromeDownload(entry ? entry.originalUrl : url, path);
+          success = true;
+        } catch(e) {
+          throw e;
+        }
+      }
+    } catch (e) {
+      if (retries < this.RETRY_LIMIT) {
+        return this.processItem(tabId, url, batchId, retries + 1);
+      }
+      errorMsg = e.message;
+    }
+
+    await this.updateProgress(tabId, url, success, errorMsg);
+  }
+
+  async chromeDownload(url, filename) {
+    return new Promise((resolve, reject) => {
+      chrome.downloads.download({ 
+        url, 
+        filename, 
+        conflictAction: 'uniquify', 
+        saveAs: false 
+      }, (downloadId) => {
+        if (chrome.runtime.lastError) {
+          return reject(new Error(chrome.runtime.lastError.message));
+        }
+        
+        const listener = (delta) => {
+          if (delta.id === downloadId && delta.state) {
+            if (delta.state.current === 'complete') {
+              chrome.downloads.onChanged.removeListener(listener);
+              resolve();
+            } else if (delta.state.current === 'interrupted') {
+              chrome.downloads.onChanged.removeListener(listener);
+              reject(new Error('Download interrupted'));
+            }
+          }
+        };
+        chrome.downloads.onChanged.addListener(listener);
+      });
+    });
+  }
+
+  async updateProgress(tabId, url, success, errorMsg) {
+    const key = `download_progress_${tabId}`;
+    const data = await chrome.storage.session.get(key);
+    if (data[key]) {
+      const prog = data[key];
+      const item = prog.items.find(i => i.url === url);
+      if (item) {
+        item.status = success ? 'completed' : 'failed';
+        if (errorMsg) item.error = errorMsg;
+      }
+      if (success) prog.completed++;
+      else prog.failed++;
+      await chrome.storage.session.set({ [key]: prog });
+    }
+  }
+
+  async buildMirroredPath(filename) {
+    const folder = await this.getUserDownloadFolder();
+    const safeFilename = filename.replace(/[<>:"|?*\\]/g, '_').replace(/\s+/g, '_').substring(0, 200);
+    return `${folder}/${safeFilename}`;
+  }
+
+  async getUserDownloadFolder() {
+    const res = await chrome.storage.local.get('user_prefs');
+    return res.user_prefs?.downloadFolder || 'SourceGrab';
+  }
+
+  async downloadZip(tabId, urls, zipFilename) {
+    const entries = await this.registry.getAll(tabId);
+    let totalSize = 0;
+    const targets = [];
+    
+    for (const url of urls) {
+      const normalized = this.registry.normalizeUrl(url);
+      const entry = entries.find(e => e.url === normalized);
+      if (entry) {
+        totalSize += (entry.size || 0);
+        targets.push(entry);
+      }
+    }
+
+    if (totalSize > this.MAX_ZIP_SIZE) {
+      return { success: false, error: 'Exceeds maximum zip size' };
+    }
+
+    const zip = new ZipBuilder();
+
+    for (const entry of targets) {
+      let data = null;
+      if (entry.requestId) {
+        const bodyRes = await this.debuggerCapture.getResponseBody(tabId, entry.requestId);
+        if (bodyRes && bodyRes.body) {
+          if (bodyRes.base64Encoded) {
+            data = base64ToUint8Array(bodyRes.body);
+          } else {
+            data = new TextEncoder().encode(bodyRes.body);
+          }
+        }
+      }
+
+      if (!data) {
+        try {
+          const res = await fetch(entry.originalUrl);
+          const buf = await res.arrayBuffer();
+          data = new Uint8Array(buf);
+        } catch (e) {
+          console.warn('ZIP fallback fetch failed for:', entry.originalUrl, e);
+        }
+      }
+
+      if (data) {
+        const path = `${new URL(entry.originalUrl).hostname}/${entry.filename}`;
+        zip.addFile(path, data);
+      }
+    }
+
+    const zipBytes = zip.generate();
+    const dataUrl = uint8ArrayToDataUrl(zipBytes, 'application/zip');
+    
+    return new Promise((resolve) => {
+      chrome.downloads.download({ 
+        url: dataUrl, 
+        filename: zipFilename,
+        conflictAction: 'uniquify',
+        saveAs: false
+      }, (downloadId) => {
+        if (chrome.runtime.lastError) {
+          console.error("ZIP download failed:", chrome.runtime.lastError.message);
+          resolve({ success: false, error: chrome.runtime.lastError.message });
+        } else {
+          resolve({ success: true });
+        }
+      });
+    });
+  }
+}
+
+class ZipBuilder {
+  constructor() {
+    this.files = [];
+  }
+
+  addFile(filename, uint8ArrayData) {
+    this.files.push({ name: filename, data: uint8ArrayData });
+  }
+
+  generate() {
+    let localHeaders = [];
+    let centralDirs = [];
+    let offset = 0;
+    
+    for (const file of this.files) {
+      const nameBuf = new TextEncoder().encode(file.name);
+      const data = file.data;
+      const crc = crc32(data);
+      const size = data.length;
+      
+      const lh = new Uint8Array(30 + nameBuf.length + size);
+      const lhView = new DataView(lh.buffer);
+      lhView.setUint32(0, 0x04034b50, true);
+      lhView.setUint16(4, 10, true);
+      lhView.setUint16(6, 0, true);
+      lhView.setUint16(8, 0, true); // STORE
+      lhView.setUint32(14, crc, true);
+      lhView.setUint32(18, size, true);
+      lhView.setUint32(22, size, true);
+      lhView.setUint16(26, nameBuf.length, true);
+      lh.set(nameBuf, 30);
+      lh.set(data, 30 + nameBuf.length);
+      localHeaders.push(lh);
+      
+      const cd = new Uint8Array(46 + nameBuf.length);
+      const cdView = new DataView(cd.buffer);
+      cdView.setUint32(0, 0x02014b50, true);
+      cdView.setUint16(4, 10, true);
+      cdView.setUint16(6, 10, true);
+      cdView.setUint16(8, 0, true);
+      cdView.setUint16(10, 0, true); // STORE
+      cdView.setUint32(16, crc, true);
+      cdView.setUint32(20, size, true);
+      cdView.setUint32(24, size, true);
+      cdView.setUint16(28, nameBuf.length, true);
+      cdView.setUint32(42, offset, true);
+      cd.set(nameBuf, 46);
+      centralDirs.push(cd);
+      
+      offset += lh.length;
+    }
+    
+    let cdSize = centralDirs.reduce((acc, cd) => acc + cd.length, 0);
+    const eocd = new Uint8Array(22);
+    const eocdView = new DataView(eocd.buffer);
+    eocdView.setUint32(0, 0x06054b50, true);
+    eocdView.setUint16(8, this.files.length, true);
+    eocdView.setUint16(10, this.files.length, true);
+    eocdView.setUint32(12, cdSize, true);
+    eocdView.setUint32(16, offset, true);
+    
+    const totalSize = offset + cdSize + 22;
+    const out = new Uint8Array(totalSize);
+    let pos = 0;
+    for (const lh of localHeaders) { out.set(lh, pos); pos += lh.length; }
+    for (const cd of centralDirs) { out.set(cd, pos); pos += cd.length; }
+    out.set(eocd, pos);
+    return out;
+  }
+}
+
+const crcTable = new Uint32Array(256);
+for (let i = 0; i < 256; i++) {
+  let c = i;
+  for (let j = 0; j < 8; j++) c = c & 1 ? 0xEDB88320 ^ (c >>> 1) : c >>> 1;
+  crcTable[i] = c;
+}
+
+function crc32(buf) {
+  let c = 0xFFFFFFFF;
+  for (let i = 0; i < buf.length; i++) c = crcTable[(c ^ buf[i]) & 0xFF] ^ (c >>> 8);
+  return (c ^ 0xFFFFFFFF) >>> 0;
+}
+
+function base64ToUint8Array(base64) {
+  const binary = atob(base64);
+  const bytes = new Uint8Array(binary.length);
+  for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
+  return bytes;
+}
+
+function uint8ArrayToDataUrl(bytes, mimeType) {
+  let binary = '';
+  const chunkSize = 8192;
+  for (let i = 0; i < bytes.length; i += chunkSize) {
+    const chunk = bytes.subarray(i, Math.min(i + chunkSize, bytes.length));
+    binary += String.fromCharCode.apply(null, chunk);
+  }
+  return `data:${mimeType};base64,${btoa(binary)}`;
+}