glidercli 0.3.1 → 0.3.2

package/bin/glider.js

@@ -37,6 +37,15 @@ const DEBUG_URL = `http://127.0.0.1:${DEBUG_PORT}`;
 const LIB_DIR = path.join(__dirname, '..', 'lib');
 const STATE_FILE = '/tmp/glider-state.json';
 const LOG_FILE = '/tmp/glider.log';
+const REGISTRY_FILE = path.join(LIB_DIR, 'registry.json');
+
+// Load pattern registry
+let REGISTRY = {};
+if (fs.existsSync(REGISTRY_FILE)) {
+  try {
+    REGISTRY = JSON.parse(fs.readFileSync(REGISTRY_FILE, 'utf8'));
+  } catch (e) { /* ignore parse errors */ }
+}
 
 // Direct CDP module
 const { DirectCDP, checkChrome } = require(path.join(LIB_DIR, 'cdp-direct.js'));
@@ -982,6 +991,164 @@ async function cmdExtract(opts = []) {
   }
 }
 
+// Registry pattern execution - bulletproof extraction using predefined patterns
+async function cmdRegistry(patternName, opts = []) {
+  if (!patternName) {
+    // List all patterns
+    const patterns = Object.keys(REGISTRY);
+    if (patterns.length === 0) {
+      log.warn('No patterns in registry');
+      return;
+    }
+    console.log(`${GREEN}${patterns.length}${NC} pattern(s) available:\n`);
+    for (const name of patterns) {
+      const p = REGISTRY[name];
+      console.log(`  ${CYAN}${name}${NC}`);
+      console.log(`      ${DIM}${p.description || 'No description'}${NC}`);
+    }
+    return;
+  }
+
+  const pattern = REGISTRY[patternName];
+  if (!pattern) {
+    log.fail(`Pattern not found: ${patternName}`);
+    log.info('Run "glider registry" to see available patterns');
+    process.exit(1);
+  }
+
+  // Parse options - for favicon: glider favicon [output.webp]
+  // The first arg that looks like a file path is output, anything else is URL
+  let outputFile = null;
+  let url = null;
+  for (let i = 0; i < opts.length; i++) {
+    const arg = opts[i];
+    if (arg === '--output' || arg === '-o') {
+      outputFile = opts[++i];
+    } else if (arg.startsWith('-')) {
+      // skip flags
+    } else if (arg.includes('/') && !arg.startsWith('http') && (arg.endsWith('.webp') || arg.endsWith('.png') || arg.endsWith('.ico'))) {
+      // Looks like a file path
+      outputFile = arg;
+    } else if (!url) {
+      url = arg;
+    }
+  }
+
+  // If URL provided, navigate first
+  if (url) {
+    if (!url.startsWith('http')) url = 'https://' + url;
+    log.info(`Navigating to: ${url}`);
+    await cmdGoto(url);
+    await new Promise(r => setTimeout(r, 2000));
+  }
+
+  log.info(`Running pattern: ${patternName}`);
+
+  try {
+    const result = await httpPost('/cdp', {
+      method: 'Runtime.evaluate',
+      params: {
+        expression: pattern.pattern,
+        returnByValue: true,
+        awaitPromise: true,
+      }
+    });
+
+    let value = result?.result?.value;
+    
+    if (value === undefined || value === null) {
+      log.fail('Pattern returned no value');
+      process.exit(1);
+    }
+
+    // Handle postprocessing for favicon
+    if (patternName === 'favicon' && pattern.postprocess) {
+      const base64 = value;
+      if (!base64 || base64.length < 50) {
+        log.fail('No favicon data received');
+        process.exit(1);
+      }
+
+      // Determine output path
+      if (!outputFile) {
+        const currentUrl = await httpPost('/cdp', {
+          method: 'Runtime.evaluate',
+          params: { expression: 'window.location.hostname', returnByValue: true }
+        });
+        const hostname = currentUrl?.result?.value?.replace(/^www\./, '').split('.')[0] || 'favicon';
+        outputFile = `/tmp/${hostname}-favicon.webp`;
+      }
+
+      // Save and convert
+      const tempFile = `/tmp/favicon-temp-${Date.now()}`;
+      const buffer = Buffer.from(base64, 'base64');
+      
+      // Detect if ICO
+      const isIco = buffer[0] === 0 && buffer[1] === 0 && buffer[2] === 1;
+      const tempPath = isIco ? `${tempFile}.ico` : `${tempFile}.png`;
+      fs.writeFileSync(tempPath, buffer);
+      log.ok(`Downloaded: ${buffer.length} bytes`);
+
+      // Convert to webp
+      if (outputFile.endsWith('.webp')) {
+        try {
+          let pngPath = tempPath;
+          if (isIco) {
+            pngPath = `${tempFile}.png`;
+            execSync(`magick "${tempPath}[0]" -resize 32x32 "${pngPath}" 2>/dev/null || convert "${tempPath}[0]" -resize 32x32 "${pngPath}" 2>/dev/null`);
+          }
+          execSync(`cwebp "${pngPath}" -o "${outputFile}" -q 90 2>/dev/null`);
+          log.ok(`Saved: ${outputFile}`);
+          
+          // Cleanup
+          try { fs.unlinkSync(tempPath); } catch {}
+          if (pngPath !== tempPath) try { fs.unlinkSync(pngPath); } catch {}
+        } catch (e) {
+          // Fallback - save as-is
+          const fallback = outputFile.replace('.webp', isIco ? '.ico' : '.png');
+          fs.copyFileSync(tempPath, fallback);
+          log.warn(`Conversion failed, saved as: ${fallback}`);
+          outputFile = fallback;
+        }
+      } else {
+        fs.copyFileSync(tempPath, outputFile);
+        log.ok(`Saved: ${outputFile}`);
+      }
+
+      // Also copy to dist if in spoonfeeder project
+      const distPath = outputFile.replace('/public/', '/dist/web/');
+      if (distPath !== outputFile && fs.existsSync(outputFile)) {
+        try {
+          const distDir = path.dirname(distPath);
+          if (fs.existsSync(distDir)) {
+            fs.copyFileSync(outputFile, distPath);
+            log.ok(`Copied to dist: ${distPath}`);
+          }
+        } catch {}
+      }
+
+      console.log(outputFile);
+      return;
+    }
+
+    // Standard output
+    if (outputFile) {
+      const output = typeof value === 'object' ? JSON.stringify(value, null, 2) : String(value);
+      fs.writeFileSync(outputFile, output);
+      log.ok(`Saved to ${outputFile}`);
+    } else {
+      if (typeof value === 'object') {
+        console.log(JSON.stringify(value, null, 2));
+      } else {
+        console.log(value);
+      }
+    }
+  } catch (e) {
+    log.fail(`Pattern failed: ${e.message}`);
+    process.exit(1);
+  }
+}
+
 // Explore site (clicks around, captures network)
 async function cmdExplore(url, opts = []) {
   if (!url) {
@@ -1337,6 +1504,7 @@ ${B5}MULTI-TAB${NC}
     ${BW}spawn${NC} <urls...>     Open multiple tabs
     ${BW}extract${NC} [opts]      Extract from all tabs
     ${BW}explore${NC} <url>       Crawl site, capture network
+    ${BW}favicon${NC} <url> [out] Extract favicon from site ${DIM}(webp)${NC}
 
 ${B5}AUTOMATION${NC}
     ${BW}run${NC} <task.yaml>     Execute YAML task file
@@ -1516,6 +1684,15 @@ async function main() {
     case 'explore':
       await cmdExplore(args[1], args.slice(2));
       break;
+    case 'favicon':
+      // Use registry pattern - bulletproof method
+      await cmdRegistry('favicon', args.slice(1));
+      break;
+    case 'registry':
+    case 'reg':
+      // Run a registry pattern
+      await cmdRegistry(args[1], args.slice(2));
+      break;
     case 'loop':
     case 'ralph':  // alias for loop - Ralph Wiggum pattern
       // Parse loop options

package/lib/bfavicon.js

@@ -0,0 +1,252 @@
+#!/usr/bin/env node
+/**
+ * bfavicon.js - Bulletproof favicon extractor via CDP
+ * 
+ * Usage:
+ *   ./bfavicon.js <url> [output.webp]
+ */
+
+const WebSocket = require('ws');
+const fs = require('fs');
+const { execSync } = require('child_process');
+
+const RELAY_URL = process.env.RELAY_URL || 'ws://127.0.0.1:19988/cdp';
+
+const GREEN = '\x1b[32m';
+const RED = '\x1b[31m';
+const CYAN = '\x1b[36m';
+const YELLOW = '\x1b[33m';
+const NC = '\x1b[0m';
+
+const log = {
+  ok: (msg) => console.error(`${GREEN}✓${NC} ${msg}`),
+  fail: (msg) => console.error(`${RED}✗${NC} ${msg}`),
+  info: (msg) => console.error(`${CYAN}→${NC} ${msg}`),
+  warn: (msg) => console.error(`${YELLOW}⚠${NC} ${msg}`),
+};
+
+class FaviconExtractor {
+  constructor() {
+    this.ws = null;
+    this.messageId = 0;
+    this.pending = new Map();
+  }
+
+  async connect() {
+    return new Promise((resolve, reject) => {
+      this.ws = new WebSocket(RELAY_URL);
+      this.ws.on('open', () => resolve());
+      this.ws.on('error', (err) => reject(new Error(`Connection failed: ${err.message}`)));
+      this.ws.on('message', (data) => {
+        const msg = JSON.parse(data.toString());
+        if (msg.id !== undefined) {
+          const pending = this.pending.get(msg.id);
+          if (pending) {
+            this.pending.delete(msg.id);
+            msg.error ? pending.reject(new Error(msg.error.message)) : pending.resolve(msg.result);
+          }
+        }
+      });
+    });
+  }
+
+  async send(method, params = {}) {
+    const id = ++this.messageId;
+    return new Promise((resolve, reject) => {
+      this.pending.set(id, { resolve, reject });
+      this.ws.send(JSON.stringify({ id, method, params }));
+      setTimeout(() => {
+        if (this.pending.has(id)) {
+          this.pending.delete(id);
+          reject(new Error('Timeout'));
+        }
+      }, 30000);
+    });
+  }
+
+  async navigate(url) {
+    await this.send('Page.navigate', { url });
+    await new Promise(r => setTimeout(r, 3000));
+  }
+
+  async extractFavicon() {
+    // Bulletproof extraction - try everything, return first success
+    const script = `
+      (async () => {
+        const origin = window.location.origin;
+        const results = [];
+        
+        // Build list of ALL possible favicon URLs
+        const urls = new Set();
+        
+        // 1. From link tags
+        document.querySelectorAll('link[rel*="icon"], link[rel="apple-touch-icon"], link[rel="apple-touch-icon-precomposed"], link[rel="mask-icon"]')
+          .forEach(l => l.href && urls.add(l.href));
+        
+        // 2. From meta tags
+        document.querySelectorAll('meta[property="og:image"], meta[name="msapplication-TileImage"]')
+          .forEach(m => m.content && urls.add(m.content));
+        
+        // 3. Default locations - try ALL common paths
+        const defaults = [
+          '/favicon.ico', '/favicon.png', '/favicon.svg', '/favicon.webp',
+          '/apple-touch-icon.png', '/apple-touch-icon-precomposed.png',
+          '/apple-touch-icon-180x180.png', '/apple-touch-icon-152x152.png',
+          '/icon.png', '/icon.ico', '/logo.png', '/logo.ico',
+          '/images/favicon.ico', '/images/favicon.png',
+          '/assets/favicon.ico', '/assets/favicon.png',
+          '/static/favicon.ico', '/static/favicon.png',
+        ];
+        defaults.forEach(p => urls.add(origin + p));
+        
+        // 4. Check manifest
+        const manifest = document.querySelector('link[rel="manifest"]');
+        if (manifest?.href) {
+          try {
+            const m = await fetch(manifest.href).then(r => r.json());
+            (m.icons || []).forEach(i => urls.add(new URL(i.src, manifest.href).href));
+          } catch {}
+        }
+        
+        // 5. Look for any img with icon/logo/favicon in src
+        document.querySelectorAll('img[src*="icon"], img[src*="logo"], img[src*="favicon"]')
+          .forEach(img => img.src && urls.add(img.src));
+        
+        // Try each URL - fetch directly, no HEAD check
+        for (const url of urls) {
+          try {
+            const resp = await fetch(url, { mode: 'cors', credentials: 'include' });
+            if (!resp.ok) continue;
+            
+            const blob = await resp.blob();
+            if (blob.size < 50) continue; // Skip tiny/empty
+            
+            // Convert to base64
+            const base64 = await new Promise((resolve, reject) => {
+              const reader = new FileReader();
+              reader.onload = () => resolve(reader.result.split(',')[1]);
+              reader.onerror = reject;
+              reader.readAsDataURL(blob);
+            });
+            
+            return { url, base64, size: blob.size, type: blob.type };
+          } catch (e) {
+            // Silent fail, try next
+          }
+        }
+        
+        return null;
+      })()
+    `;
+
+    const result = await this.send('Runtime.evaluate', {
+      expression: script,
+      awaitPromise: true,
+      returnByValue: true
+    });
+
+    return result?.value;
+  }
+
+  close() {
+    if (this.ws) this.ws.close();
+  }
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+  
+  if (args.length === 0 || args[0] === '--help' || args[0] === '-h') {
+    console.log(`Usage: glider favicon <url> [output.webp]`);
+    process.exit(0);
+  }
+
+  let url = args[0];
+  let outputPath = args[1];
+
+  if (!url.startsWith('http')) url = 'https://' + url;
+
+  log.info(`Extracting favicon from: ${url}`);
+
+  const extractor = new FaviconExtractor();
+
+  try {
+    await extractor.connect();
+    log.ok('Connected to relay');
+
+    await extractor.navigate(url);
+    log.ok('Page loaded');
+
+    const favicon = await extractor.extractFavicon();
+    
+    if (!favicon) {
+      log.fail('No favicon found');
+      process.exit(1);
+    }
+
+    log.ok(`Found: ${favicon.url} (${favicon.size} bytes, ${favicon.type})`);
+
+    // Determine output path
+    if (!outputPath) {
+      const hostname = new URL(url).hostname.replace(/^www\./, '').split('.')[0];
+      outputPath = `/tmp/${hostname}-favicon.png`;
+    }
+
+    // Save the file
+    const buffer = Buffer.from(favicon.base64, 'base64');
+    const tempFile = `/tmp/favicon-temp-${Date.now()}`;
+    
+    // Detect format and save appropriately
+    const isIco = favicon.type.includes('icon') || favicon.url.endsWith('.ico');
+    const tempPath = isIco ? `${tempFile}.ico` : `${tempFile}.png`;
+    fs.writeFileSync(tempPath, buffer);
+
+    // Convert to webp if requested
+    if (outputPath.endsWith('.webp')) {
+      try {
+        // If ico, convert to png first
+        let pngPath = tempPath;
+        if (isIco) {
+          pngPath = `${tempFile}.png`;
+          execSync(`magick "${tempPath}" -resize 32x32 "${pngPath}" 2>/dev/null || convert "${tempPath}" -resize 32x32 "${pngPath}" 2>/dev/null`);
+        }
+        execSync(`cwebp "${pngPath}" -o "${outputPath}" -q 90 2>/dev/null`);
+        log.ok(`Saved as WebP: ${outputPath}`);
+        // Cleanup
+        try { fs.unlinkSync(tempPath); } catch {}
+        if (pngPath !== tempPath) try { fs.unlinkSync(pngPath); } catch {}
+      } catch (e) {
+        // Fallback - just save as-is
+        const fallbackPath = outputPath.replace('.webp', isIco ? '.ico' : '.png');
+        fs.copyFileSync(tempPath, fallbackPath);
+        log.warn(`Conversion failed, saved as: ${fallbackPath}`);
+        outputPath = fallbackPath;
+      }
+    } else {
+      fs.copyFileSync(tempPath, outputPath);
+      log.ok(`Saved: ${outputPath}`);
+    }
+
+    // Copy to dist too
+    const distPath = outputPath.replace('/public/', '/dist/web/');
+    if (distPath !== outputPath && fs.existsSync(outputPath)) {
+      try {
+        const distDir = require('path').dirname(distPath);
+        if (fs.existsSync(distDir)) {
+          fs.copyFileSync(outputPath, distPath);
+          log.ok(`Copied to dist: ${distPath}`);
+        }
+      } catch {}
+    }
+
+    console.log(outputPath);
+
+  } catch (e) {
+    log.fail(e.message);
+    process.exit(1);
+  } finally {
+    extractor.close();
+  }
+}
+
+main();

package/lib/registry.json

@@ -0,0 +1,73 @@
+{
+  "favicon": {
+    "description": "Extract favicon from current page using browser session",
+    "pattern": "(async () => { const resp = await fetch(document.querySelector('link[rel*=icon]')?.href || window.location.origin + '/favicon.ico', { credentials: 'include' }); const blob = await resp.blob(); const reader = new FileReader(); return await new Promise(resolve => { reader.onload = () => resolve(reader.result.split(',')[1]); reader.readAsDataURL(blob); }); })()",
+    "output": "base64",
+    "postprocess": ["base64-decode", "convert-webp"]
+  },
+  "favicon-all": {
+    "description": "Get all favicon URLs from current page",
+    "pattern": "(() => { const urls = []; document.querySelectorAll('link[rel*=icon]').forEach(l => urls.push(l.href)); if (urls.length === 0) urls.push(window.location.origin + '/favicon.ico'); return urls; })()",
+    "output": "json"
+  },
+  "title": {
+    "description": "Get page title",
+    "pattern": "document.title",
+    "output": "string"
+  },
+  "url": {
+    "description": "Get current URL",
+    "pattern": "window.location.href",
+    "output": "string"
+  },
+  "links": {
+    "description": "Get all links on page",
+    "pattern": "Array.from(document.querySelectorAll('a[href]')).map(a => ({ text: a.innerText.trim(), href: a.href })).filter(l => l.href.startsWith('http'))",
+    "output": "json"
+  },
+  "images": {
+    "description": "Get all image URLs",
+    "pattern": "Array.from(document.querySelectorAll('img[src]')).map(i => i.src)",
+    "output": "json"
+  },
+  "text": {
+    "description": "Get page text content",
+    "pattern": "document.body.innerText",
+    "output": "string"
+  },
+  "meta": {
+    "description": "Get all meta tags",
+    "pattern": "Array.from(document.querySelectorAll('meta')).map(m => ({ name: m.name || m.property, content: m.content })).filter(m => m.content)",
+    "output": "json"
+  },
+  "og": {
+    "description": "Get Open Graph metadata",
+    "pattern": "(() => { const og = {}; document.querySelectorAll('meta[property^=\"og:\"]').forEach(m => og[m.property.replace('og:','')] = m.content); return og; })()",
+    "output": "json"
+  },
+  "forms": {
+    "description": "Get all forms and their inputs",
+    "pattern": "Array.from(document.querySelectorAll('form')).map(f => ({ action: f.action, method: f.method, inputs: Array.from(f.querySelectorAll('input,select,textarea')).map(i => ({ name: i.name, type: i.type, id: i.id })) }))",
+    "output": "json"
+  },
+  "cookies": {
+    "description": "Get all cookies",
+    "pattern": "document.cookie.split(';').map(c => c.trim()).filter(c => c).map(c => { const [k,...v] = c.split('='); return { name: k, value: v.join('=') }; })",
+    "output": "json"
+  },
+  "storage": {
+    "description": "Get localStorage keys",
+    "pattern": "Object.keys(localStorage)",
+    "output": "json"
+  },
+  "scripts": {
+    "description": "Get all script sources",
+    "pattern": "Array.from(document.querySelectorAll('script[src]')).map(s => s.src)",
+    "output": "json"
+  },
+  "styles": {
+    "description": "Get all stylesheet links",
+    "pattern": "Array.from(document.querySelectorAll('link[rel=stylesheet]')).map(l => l.href)",
+    "output": "json"
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "glidercli",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "description": "Browser automation CLI. Control Chrome from terminal via CDP, run YAML task files, autonomous loops until completion.",
   "main": "index.js",
   "bin": {