omniwire 3.2.2 → 3.3.1

package/.omc/state/hud-stdin-cache.json

@@ -1 +1 @@
-{"session_id":"8ef02123-7368-447d-82e3-ee14a27328b0","transcript_path":"C:\\Users\\Admin\\.claude\\projects\\C--Users-Admin\\8ef02123-7368-447d-82e3-ee14a27328b0.jsonl","cwd":"C:\\Users\\Admin\\omniwire","model":{"id":"claude-opus-4-6[1m]","display_name":"Opus 4.6 (1M context)"},"workspace":{"current_dir":"C:\\Users\\Admin\\omniwire","project_dir":"C:\\Users\\Admin","added_dirs":["C:/Users/Admin"]},"version":"2.1.87","output_style":{"name":"default"},"cost":{"total_cost_usd":5.113285649999999,"total_duration_ms":1663275,"total_api_duration_ms":659204,"total_lines_added":42,"total_lines_removed":27},"context_window":{"total_input_tokens":801,"total_output_tokens":23773,"context_window_size":1000000,"current_usage":{"input_tokens":1,"output_tokens":170,"cache_creation_input_tokens":354,"cache_read_input_tokens":103104},"used_percentage":10,"remaining_percentage":90},"exceeds_200k_tokens":false,"rate_limits":{"five_hour":{"used_percentage":35,"resets_at":1774828800},"seven_day":{"used_percentage":41,"resets_at":1775206800}}}
+{"session_id":"8ef02123-7368-447d-82e3-ee14a27328b0","transcript_path":"C:\\Users\\Admin\\.claude\\projects\\C--Users-Admin\\8ef02123-7368-447d-82e3-ee14a27328b0.jsonl","cwd":"C:\\Users\\Admin\\omniwire","model":{"id":"claude-opus-4-6[1m]","display_name":"Opus 4.6 (1M context)"},"workspace":{"current_dir":"C:\\Users\\Admin\\omniwire","project_dir":"C:\\Users\\Admin","added_dirs":["C:/Users/Admin"]},"version":"2.1.87","output_style":{"name":"default"},"cost":{"total_cost_usd":13.206143100000004,"total_duration_ms":2660349,"total_api_duration_ms":1904685,"total_lines_added":283,"total_lines_removed":41},"context_window":{"total_input_tokens":176085,"total_output_tokens":78423,"context_window_size":1000000,"current_usage":{"input_tokens":1,"output_tokens":275,"cache_creation_input_tokens":560,"cache_read_input_tokens":152806},"used_percentage":15,"remaining_percentage":85},"exceeds_200k_tokens":false,"rate_limits":{"five_hour":{"used_percentage":1,"resets_at":1774846800},"seven_day":{"used_percentage":41,"resets_at":1775206800}}}

package/README.md

@@ -19,7 +19,7 @@
 
 **The infrastructure layer for AI agent swarms.**
 
-86 MCP tools · A2A protocol · OmniMesh VPN · nftables firewall · CDP browser · cookie sync · 2FA TOTP · bi-directional sync · CyberBase persistence
+88 MCP tools · A2A protocol · OmniMesh VPN · nftables firewall · CDP browser · cookie sync · 2FA TOTP · bi-directional sync · CyberBase persistence
 
 </div>
 
@@ -687,6 +687,8 @@ omniwire/
 
 | Version | Date | Changes |
 |---------|------|---------|
+| **v3.3.1** | 2026-03-30 | New: `omniwire_scrape` tool — Scrapling-powered web scraping (static/browser/stealth modes, Cloudflare bypass, TLS spoofing). 88 tools. |
+| **v3.3.0** | 2026-03-30 | New: `omniwire_coc` tool — unified CyberBase + Obsidian + Canvas sync. Auto-creates vault + canvas. `mirror-db` exports entire DB as .md. Configurable vault via `OMNIWIRE_VAULT_ROOT` env. |
 | **v3.2.2** | 2026-03-30 | Fix: sync GitHub/npm metadata — badge, description, mermaid diagram all reflect 86 tools |
 | **v3.2.1** | 2026-03-30 | New: 5 bi-directional sync tools (`omniwire_sync`, `omniwire_sync_rules`, `omniwire_sync_hooks`, `omniwire_sync_memory`, `omniwire_sync_agents`) — 86 tools total |
 | **v3.2.0** | 2026-03-29 | New: `omniwire_2fa` TOTP manager — add/generate/verify/import/export 2FA codes, CyberBase + 1Password persistence, otpauth:// URI import, bulk code generation |

package/dist/mcp/server.js

@@ -6,7 +6,7 @@
 // authenticated, encrypted SSH channels. The "exec" references below are SSH2 methods.
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { z } from 'zod';
-import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
+import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync } from 'node:fs';
 import { join } from 'node:path';
 import { ShellManager, kernelExec } from '../nodes/shell.js';
 import { RealtimeChannel } from '../nodes/realtime.js';
@@ -248,10 +248,40 @@ async function drainCb() {
 }
 // -- Obsidian + Canvas auto-sync ------------------------------------------------
 // Mirrors CyberBase writes to local Obsidian vault + Canvas mindmap.
-// Vault path is resolved at startup; if it doesn't exist, sync is silently skipped.
-const VAULT_ROOT = join(process.env.USERPROFILE ?? process.env.HOME ?? '', 'Documents', 'BuisnessProjects', 'CyberBase');
-const CANVAS_PATH = join(VAULT_ROOT, 'CyberBase MindMap.canvas');
+// Vault path resolved from env or default; if it doesn't exist, sync is silently skipped.
+// Set OMNIWIRE_VAULT_ROOT to override, or OMNIWIRE_CANVAS_NAME for custom canvas filename.
+const VAULT_ROOT = process.env.OMNIWIRE_VAULT_ROOT ?? join(process.env.USERPROFILE ?? process.env.HOME ?? '', 'Documents', 'OmniWire');
+const CANVAS_NAME = process.env.OMNIWIRE_CANVAS_NAME ?? 'OmniWire MindMap.canvas';
+const CANVAS_PATH = join(VAULT_ROOT, CANVAS_NAME);
 const vaultExists = existsSync(VAULT_ROOT);
+/** Ensure the Obsidian vault root and canvas file exist, creating them if needed */
+function ensureVault() {
+    try {
+        if (!existsSync(VAULT_ROOT))
+            mkdirSync(VAULT_ROOT, { recursive: true });
+        if (!existsSync(CANVAS_PATH)) {
+            // Create a default canvas with a central hub node
+            const defaultCanvas = {
+                nodes: [{
+                        id: 'core',
+                        type: 'text',
+                        text: '## OmniWire\nCentral knowledge hub',
+                        x: 0,
+                        y: 0,
+                        width: 300,
+                        height: 120,
+                        color: '4',
+                    }],
+                edges: [],
+            };
+            writeFileSync(CANVAS_PATH, JSON.stringify(defaultCanvas, null, '\t'), 'utf-8');
+        }
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
 /** Map CyberBase category → Obsidian vault subfolder */
 function vaultFolder(category) {
     const cat = category.toLowerCase();
@@ -279,8 +309,8 @@ function sanitizeFilename(key) {
 }
 /** Auto-sync a knowledge entry to Obsidian vault as a .md file */
 function syncObsidian(category, key, value) {
-    if (!vaultExists)
-        return;
+    if (!existsSync(VAULT_ROOT))
+        ensureVault();
     try {
         const folder = join(VAULT_ROOT, vaultFolder(category));
         if (!existsSync(folder))
@@ -359,7 +389,9 @@ function canvasColor(category) {
 }
 /** Auto-sync a knowledge entry to the Canvas mindmap — adds or updates a node */
 function syncCanvas(category, key, value) {
-    if (!vaultExists || !existsSync(CANVAS_PATH))
+    if (!existsSync(CANVAS_PATH))
+        ensureVault();
+    if (!existsSync(CANVAS_PATH))
         return;
     try {
         const raw = readFileSync(CANVAS_PATH, 'utf-8');
@@ -3993,8 +4025,7 @@ echo "port-knock configured: ${ports.join(' -> ')} -> port ${target}"`;
         if (action === 'sync-obsidian') {
             if (!key || !value)
                 return fail('key and value required');
-            if (!vaultExists)
-                return fail(`Obsidian vault not found at ${VAULT_ROOT}`);
+            ensureVault();
             const cat = category ?? 'general';
             syncObsidian(cat, key, value);
             const folder = vaultFolder(cat);
@@ -4003,14 +4034,207 @@ echo "port-knock configured: ${ports.join(' -> ')} -> port ${target}"`;
         if (action === 'sync-canvas') {
             if (!key || !value)
                 return fail('key and value required');
-            if (!vaultExists || !existsSync(CANVAS_PATH))
-                return fail(`Canvas not found at ${CANVAS_PATH}`);
+            ensureVault();
             const cat = category ?? 'general';
             syncCanvas(cat, key, value);
             return okBrief(`synced to Canvas: node auto_${sanitizeFilename(cat)}_${sanitizeFilename(key)} added/updated`);
         }
         return fail('invalid action');
     });
+    // --- Tool: omniwire_coc ---
+    // COC = CyberBase + Obsidian + Canvas — the default sync mode.
+    // Single tool call writes to all three destinations simultaneously.
+    // Also supports 'mirror-db' to export the entire DB as Obsidian .md files.
+    server.tool('omniwire_coc', 'COC (CyberBase + Obsidian + Canvas) — unified sync. Default: writes to PostgreSQL, Obsidian vault (.md), and Canvas mindmap in one call. Use "mirror-db" to export the entire knowledge DB as Obsidian-formatted markdown files. Use "init" to set up the vault + canvas from scratch.', {
+        action: z.enum(['save', 'mirror-db', 'init', 'status']).describe('save: write to all 3 destinations. mirror-db: export entire DB as .md files. init: create vault + canvas. status: show sync state.'),
+        category: z.string().optional().describe('Knowledge category (e.g., tools, vulns, infra, notes)'),
+        key: z.string().optional().describe('Entry key'),
+        value: z.string().optional().describe('Entry value'),
+        entries: z.array(z.object({ category: z.string(), key: z.string(), value: z.string() })).optional().describe('Bulk save: array of {category, key, value}'),
+    }, async ({ action, category, key, value, entries }) => {
+        if (action === 'init') {
+            const created = ensureVault();
+            if (!created)
+                return fail('Failed to create vault directory');
+            // Create standard subdirectories
+            const dirs = ['projects', 'infrastructure', 'knowledge', 'knowledge/security-kb', 'system', 'logs', 'sync', 'memory'];
+            for (const dir of dirs) {
+                const p = join(VAULT_ROOT, dir);
+                if (!existsSync(p))
+                    mkdirSync(p, { recursive: true });
+            }
+            return okBrief(`COC vault initialized at ${VAULT_ROOT}\n  Canvas: ${CANVAS_NAME}\n  Directories: ${dirs.join(', ')}`);
+        }
+        if (action === 'status') {
+            const vaultOk = existsSync(VAULT_ROOT);
+            const canvasOk = existsSync(CANVAS_PATH);
+            const h = getCbHealth();
+            let canvasNodes = 0;
+            if (canvasOk) {
+                try {
+                    const raw = JSON.parse(readFileSync(CANVAS_PATH, 'utf-8'));
+                    canvasNodes = raw.nodes?.length ?? 0;
+                }
+                catch { /* ignore */ }
+            }
+            // Count .md files in vault
+            let mdCount = 0;
+            const countMd = (dir) => {
+                try {
+                    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+                        if (entry.isDirectory())
+                            countMd(join(dir, entry.name));
+                        else if (entry.name.endsWith('.md'))
+                            mdCount++;
+                    }
+                }
+                catch { /* ignore */ }
+            };
+            if (vaultOk)
+                countMd(VAULT_ROOT);
+            return okBrief(`COC Status:\n  Vault: ${vaultOk ? 'OK' : 'MISSING'} (${VAULT_ROOT})\n  Canvas: ${canvasOk ? `OK (${canvasNodes} nodes)` : 'MISSING'}\n  CyberBase: ${h.healthy ? 'OK' : 'UNHEALTHY'} (queue: ${h.queueSize})\n  Obsidian files: ${mdCount} .md files`);
+        }
+        if (action === 'save') {
+            // Bulk save
+            if (entries?.length) {
+                for (const e of entries) {
+                    cb(e.category, e.key, e.value);
+                }
+                return okBrief(`COC: saved ${entries.length} entries → CyberBase + Obsidian + Canvas`);
+            }
+            // Single save
+            if (!key || !value)
+                return fail('key and value required (or use entries[] for bulk)');
+            const cat = category ?? 'general';
+            cb(cat, key, value);
+            return okBrief(`COC: saved ${cat}:${key} (${value.length} chars) → CyberBase + Obsidian + Canvas`);
+        }
+        if (action === 'mirror-db') {
+            if (!cbManager)
+                return fail('no CyberBase connection');
+            ensureVault();
+            // Export all knowledge entries from PostgreSQL and write as .md files
+            const r = await cbManager.exec('contabo', `psql -h 127.0.0.1 -U cyberbase -d cyberbase -t -A -F '|' -c "SET statement_timeout='30s'; SELECT key, value->>'data', updated_at FROM knowledge WHERE source_tool='omniwire' ORDER BY key;" 2>/dev/null`);
+            if (!r.stdout.trim())
+                return okBrief('mirror-db: no entries found in CyberBase');
+            const lines = r.stdout.trim().split('\n').filter((l) => l.includes('|'));
+            let synced = 0;
+            let skipped = 0;
+            for (const line of lines) {
+                const parts = line.split('|');
+                if (parts.length < 2) {
+                    skipped++;
+                    continue;
+                }
+                const fullKey = parts[0].trim();
+                const val = parts.slice(1, -1).join('|').trim(); // rejoin in case value has pipes
+                if (!fullKey || !val) {
+                    skipped++;
+                    continue;
+                }
+                // Parse category:key format
+                const colonIdx = fullKey.indexOf(':');
+                const cat = colonIdx > 0 ? fullKey.slice(0, colonIdx) : 'general';
+                const entryKey = colonIdx > 0 ? fullKey.slice(colonIdx + 1) : fullKey;
+                syncObsidian(cat, entryKey, val);
+                if (val.length > 50)
+                    syncCanvas(cat, entryKey, val);
+                synced++;
+            }
+            return okBrief(`COC mirror-db: ${synced} entries synced to Obsidian + Canvas, ${skipped} skipped\n  Vault: ${VAULT_ROOT}`);
+        }
+        return fail('invalid action');
+    });
+    // --- Tool: omniwire_scrape ---
+    // Scrapling-powered web scraping: static HTTP (TLS spoofing), browser (JS rendering), stealth (anti-bot bypass).
+    // Runs via Scrapling MCP server on Contabo (port 8931) or falls back to CLI.
+    server.tool('omniwire_scrape', 'Scrape web pages using Scrapling — adaptive, anti-bot web scraping. Modes: http (fast TLS-spoofed static fetch), browser (Playwright JS rendering), stealth (Camoufox + Cloudflare bypass). Returns markdown/html/text. Powered by Scrapling on Contabo.', {
+        url: z.string().describe('Target URL to scrape'),
+        urls: z.array(z.string()).optional().describe('Multiple URLs for bulk scraping (uses session pooling)'),
+        mode: z.enum(['http', 'browser', 'stealth']).default('http').describe('http=fast static, browser=JS rendering, stealth=anti-bot+Cloudflare'),
+        extraction_type: z.enum(['markdown', 'html', 'text']).default('markdown').describe('Output format'),
+        css_selector: z.string().optional().describe('CSS selector to extract specific elements only'),
+        solve_cloudflare: z.boolean().optional().describe('Solve Cloudflare Turnstile (stealth mode only)'),
+        wait_selector: z.string().optional().describe('Wait for this CSS selector before extracting (browser/stealth)'),
+        network_idle: z.boolean().optional().describe('Wait for network idle before extracting'),
+        proxy: z.string().optional().describe('Proxy URL (http://user:pass@host:port)'),
+        timeout: z.number().default(30).describe('Timeout in seconds'),
+        impersonate: z.string().optional().describe('TLS fingerprint: chrome, safari, firefox (http mode)'),
+        node: z.string().optional().describe('Node to run on (default: contabo)'),
+        label: z.string().optional().describe('Short label for task tracking'),
+    }, async ({ url, urls, mode, extraction_type, css_selector, solve_cloudflare, wait_selector, network_idle, proxy, timeout, impersonate, node: targetNode, label }) => {
+        if (!manager)
+            return fail('NodeManager not initialized');
+        const target = targetNode ?? 'contabo';
+        // Build the Scrapling Python command based on mode
+        const allUrls = urls?.length ? urls : [url];
+        const urlList = allUrls.map(u => `'${u.replace(/'/g, "'\\''")}'`).join(' ');
+        // Map mode to Scrapling fetcher
+        const fetcherMap = {
+            http: 'Fetcher',
+            browser: 'DynamicFetcher',
+            stealth: 'StealthyFetcher',
+        };
+        const fetcher = fetcherMap[mode] ?? 'Fetcher';
+        // Build Python script
+        const proxyArg = proxy ? `, proxy='${proxy.replace(/'/g, "'\\''")}'` : '';
+        const impersonateArg = impersonate ? `, impersonate='${impersonate}'` : '';
+        const timeoutArg = `, timeout=${timeout}`;
+        const cfArg = solve_cloudflare ? ', solve_cloudflare=True' : '';
+        const waitArg = wait_selector ? `, wait_selector='${wait_selector.replace(/'/g, "'\\''")}'` : '';
+        const idleArg = network_idle ? ', network_idle=True' : '';
+        const selectorArg = css_selector ? `.css('${css_selector.replace(/'/g, "'\\''")}')` : '';
+        // Extraction type mapping
+        const extractMap = {
+            markdown: '.get_all_text()',
+            html: '.prettify() if hasattr(page, "prettify") else str(page)',
+            text: '.get_all_text()',
+        };
+        const extract = selectorArg ? `.getall()` : extractMap[extraction_type] ?? '.get_all_text()';
+        const script = `
+import json, sys
+try:
+    from scrapling import ${fetcher}
+    results = []
+    urls = ${JSON.stringify(allUrls)}
+    for u in urls:
+        try:
+            page = ${fetcher}().get(u${proxyArg}${impersonateArg}${timeoutArg}${cfArg}${waitArg}${idleArg})
+            if page.status == 200:
+                content = page${selectorArg}${extract}
+                if isinstance(content, list):
+                    content = '\\n'.join(str(c) for c in content)
+                results.append({"url": u, "status": page.status, "content": str(content)[:50000]})
+            else:
+                results.append({"url": u, "status": page.status, "content": f"HTTP {page.status}"})
+        except Exception as e:
+            results.append({"url": u, "status": 0, "error": str(e)[:500]})
+    print(json.dumps(results))
+except Exception as e:
+    print(json.dumps([{"error": str(e)}]))
+`.trim();
+        try {
+            const r = await manager.exec(target, `python3 -c ${JSON.stringify(script)}`);
+            const output = r.stdout.trim();
+            try {
+                const results = JSON.parse(output);
+                if (results.length === 1) {
+                    const res = results[0];
+                    if (res.error)
+                        return fail(`scrape error: ${res.error}`);
+                    return okBrief(`[${res.status}] ${res.url}\n\n${res.content}`);
+                }
+                const summary = results.map((r) => `[${r.status ?? 'ERR'}] ${r.url ?? '?'}: ${r.error ?? `${(r.content ?? '').length} chars`}`).join('\n');
+                return okBrief(`Scraped ${results.length} URLs:\n${summary}\n\n${results.map((r) => r.content ?? '').join('\n---\n').slice(0, 50000)}`);
+            }
+            catch {
+                return okBrief(output.slice(0, 10000));
+            }
+        }
+        catch (e) {
+            return fail(`scrape failed on ${target}: ${e.message}`);
+        }
+    });
     // --- Tool 54: omniwire_omnimesh ---
     server.tool('omniwire_omnimesh', 'OmniMesh — built-in WireGuard mesh network manager. Create, manage, and monitor a full-mesh or hub-spoke WireGuard VPN across all nodes and any OS (Linux/Windows/macOS). Actions: status, init, add-peer, remove-peer, genkeys, deploy-config, up, down, install, health, rotate-keys, discover-endpoint, topology, sync-peers.', {
         action: z.enum([