@xelth/eck-snapshot 5.9.0 → 6.0.1

package/src/utils/fileUtils.js

@@ -11,56 +11,86 @@ import { minimatch } from 'minimatch';
 /**
  * Scanner for detecting and redacting secrets (API keys, tokens)
  */
-export const SecretScanner = {
-  patterns: [
-    // Service-specific patterns
-    { name: 'GitHub Token', regex: /gh[pous]_[a-zA-Z0-9]{36}/g },
-    { name: 'AWS Access Key', regex: /(?:AKIA|ASIA)[0-9A-Z]{16}/g },
-    { name: 'OpenAI API Key', regex: /sk-[a-zA-Z0-9]{32,}/g },
-    { name: 'Stripe Secret Key', regex: /sk_live_[0-9a-zA-Z]{24}/g },
-    { name: 'Google API Key', regex: /AIza[0-9A-Za-z\-_]{35}/g },
-    { name: 'Slack Token', regex: /xox[baprs]-[0-9a-zA-Z\-]{10,}/g },
-    { name: 'NPM Token', regex: /npm_[a-zA-Z0-9]{36}/g },
-    { name: 'Private Key', regex: /-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/g },
-    // Generic high-entropy patterns near sensitive keywords
-    {
-      name: 'Generic Secret',
-      regex: /(?:api[_-]?key|secret|password|token|auth|pwd|credential)\s*[:=]\s*["']([a-zA-Z0-9\-_.]{16,})["']/gi
-    }
-  ],
-
-  /**
-   * Scans content and replaces detected secrets with a placeholder
-   * @param {string} content - File content to scan
-   * @param {string} filePath - Path for logging context
-   * @returns {{content: string, found: string[]}} Redacted content and list of found secret types
-   */
-  redact(content, filePath) {
-    let redactedContent = content;
-    const foundSecrets = [];
-
-    for (const pattern of this.patterns) {
-      // Reset regex lastIndex for global patterns
-      pattern.regex.lastIndex = 0;
-
-      const matches = [...content.matchAll(pattern.regex)];
-      if (matches.length > 0) {
-        for (const match of matches) {
-          // For generic pattern, use captured group; for specific patterns, use full match
-          const secretValue = match[1] || match[0];
-          const placeholder = `[REDACTED_${pattern.name.replace(/\s+/g, '_').toUpperCase()}]`;
-          redactedContent = redactedContent.replace(secretValue, placeholder);
-          foundSecrets.push(pattern.name);
-        }
-      }
-    }
-
-    return {
-      content: redactedContent,
-      found: [...new Set(foundSecrets)]
-    };
-  }
-};
+export const SecretScanner = {
+  patterns: [
+    // Service-specific patterns
+    { name: 'GitHub Token', regex: /gh[pous]_[a-zA-Z0-9]{36}/g },
+    { name: 'AWS Access Key', regex: /(?:AKIA|ASIA)[0-9A-Z]{16}/g },
+    { name: 'OpenAI API Key', regex: /sk-[a-zA-Z0-9]{32,}/g },
+    { name: 'Stripe Secret Key', regex: /sk_live_[0-9a-zA-Z]{24}/g },
+    { name: 'Google API Key', regex: /AIza[0-9A-Za-z\-_]{35}/g },
+    { name: 'Slack Token', regex: /xox[baprs]-[0-9a-zA-Z\-]{10,}/g },
+    { name: 'NPM Token', regex: /npm_[a-zA-Z0-9]{36}/g },
+    { name: 'Private Key', regex: /-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/g },
+    // Generic high-entropy patterns near sensitive keywords
+    {
+      name: 'Generic Secret',
+      regex: /(?:api[_-]?key|secret|password|token|auth|pwd|credential)\s*[:=]\s*["']([a-zA-Z0-9\-_.]{16,})["']/gi
+    }
+  ],
+
+  /**
+   * Calculates Shannon Entropy of a string
+   */
+  calculateEntropy(str) {
+    const len = str.length;
+    const frequencies = Array.from(str).reduce((freq, c) => {
+      freq[c] = (freq[c] || 0) + 1;
+      return freq;
+    }, {});
+    return Object.values(frequencies).reduce((sum, f) => {
+      const p = f / len;
+      return sum - (p * Math.log2(p));
+    }, 0);
+  },
+
+  /**
+   * Scans content and replaces detected secrets with a placeholder
+   * @param {string} content - File content to scan
+   * @param {string} filePath - Path for logging context
+   * @returns {{content: string, found: string[]}} Redacted content and list of found secret types
+   */
+  redact(content, filePath) {
+    let redactedContent = content;
+    const foundSecrets = [];
+
+    for (const pattern of this.patterns) {
+      // Reset regex lastIndex for global patterns
+      pattern.regex.lastIndex = 0;
+
+      const matches = [...content.matchAll(pattern.regex)];
+      if (matches.length > 0) {
+        for (const match of matches) {
+          // For generic pattern, use captured group; for specific patterns, use full match
+          const secretValue = match[1] || match[0];
+          const placeholder = `[REDACTED_${pattern.name.replace(/\s+/g, '_').toUpperCase()}]`;
+          redactedContent = redactedContent.replace(secretValue, placeholder);
+          foundSecrets.push(pattern.name);
+        }
+      }
+    }
+
+    // Second pass: Shannon Entropy check for arbitrary hardcoded secrets
+    // Look for long strings assigned to variable names that might be keys
+    const entropyRegex = /(?:const|let|var|set|export|define)\s+([A-Za-z0-9_]*(?:KEY|TOKEN|SECRET|PASSWORD)[A-Za-z0-9_]*)\s*=\s*["']([a-zA-Z0-9+/=_-]{20,128})["']/gi;
+    const entropyMatches = [...redactedContent.matchAll(entropyRegex)];
+    
+    for (const match of entropyMatches) {
+      const secretValue = match[2];
+      // Check entropy - random base64 usually has entropy > 4.5
+      if (this.calculateEntropy(secretValue) > 4.5 && !secretValue.includes('REDACTED')) {
+        const placeholder = `[REDACTED_HIGH_ENTROPY_SECRET]`;
+        redactedContent = redactedContent.replace(secretValue, placeholder);
+        foundSecrets.push('High Entropy Secret');
+      }
+    }
+
+    return {
+      content: redactedContent,
+      found: [...new Set(foundSecrets)]
+    };
+  }
+};
 
 export function parseSize(sizeStr) {
   const units = { B: 1, KB: 1024, MB: 1024 ** 2, GB: 1024 ** 3 };
@@ -838,15 +868,16 @@ export async function applyProfileFilter(allFiles, profileString, repoPath) {
 export async function initializeEckManifest(projectPath) {
   const eckDir = path.join(projectPath, '.eck');
 
-  // Load setup configuration to check AI generation settings
-  let aiGenerationEnabled = false;
-  try {
-    const setupConfig = await loadSetupConfig();
-    aiGenerationEnabled = setupConfig?.aiInstructions?.manifestInitialization?.aiGenerationEnabled ?? false;
-  } catch (error) {
-    // If setup config fails to load, default to disabled
-    console.warn(`   ⚠️ Could not load setup config: ${error.message}. AI generation disabled.`);
-  }
+  // Load setup configuration to check AI generation settings and project context
+  let aiGenerationEnabled = false;
+  let setupConfig = null;
+  try {
+    setupConfig = await loadSetupConfig();
+    aiGenerationEnabled = setupConfig?.aiInstructions?.manifestInitialization?.aiGenerationEnabled ?? false;
+  } catch (error) {
+    // If setup config fails to load, default to disabled
+    console.warn(`   ⚠️ Could not load setup config: ${error.message}. AI generation disabled.`);
+  }
 
   try {
     // Check if .eck directory already exists and has all required files
@@ -894,33 +925,38 @@ export async function initializeEckManifest(projectPath) {
       delete staticFacts.allDetections;
     }
     
-    const staticFactsJson = JSON.stringify(staticFacts, null, 2);
-    // --- END NEW LOGIC ---
-    
-    // 3. Define smarter templates and prompts with "STUB NOTICES"
+    const staticFactsJson = JSON.stringify(staticFacts, null, 2);
+    // --- END NEW LOGIC ---
+
+    // Extract Context from setup.json if available
+    const projName = setupConfig?.projectContext?.name || staticFacts.type || 'project';
+    const projType = setupConfig?.projectContext?.type || staticFacts.type || 'unknown';
+    const projStack = setupConfig?.projectContext?.architecture?.stack?.join(', ') || 'TBD';
+    const projAi = setupConfig?.projectContext?.architecture?.aiIntegration || 'None';
+    
+    // 3. Define smarter templates and prompts using setup.json context
     const templateConfigs = {
-      'ENVIRONMENT.md': {
-        prompt: `Generate raw YAML for .eck/ENVIRONMENT.md based on these project facts:\n${staticFactsJson}\nInclude project_type, runtime, and agent_id: local_dev. NO markdown fences.`,
-        fallback: `project_type: ${staticFacts.type || 'unknown'}
-# [STUB: ENVIRONMENT.MD]
-# ARCHITECT: Task Coder to verify environment variables.
-# CODER: Scan project for .env files or config files and fill this. Remove this stub notice.
-agent_id: local_dev
-`
+      'ENVIRONMENT.md': {
+        prompt: `Generate raw YAML for .eck/ENVIRONMENT.md based on these project facts:\n${staticFactsJson}\nInclude project_type, runtime, and agent_id: local_dev. NO markdown fences.`,
+        fallback: `project_type: ${projType}
+agent_id: local_dev
+# Generated from setup.json
+`
       },
-      'CONTEXT.md': {
-        prompt: `Analyze these project files and dependencies:\n${staticFactsJson}\nGenerate a professional # Project Overview in Markdown. Describe the actual architecture and purpose of this specific project. Be technical and concise. Start with '# Project Overview'.`,
-        fallback: `# [STUB: CONTEXT.MD]
-
-## 🚨 ATTENTION ARCHITECT & CODER
-**ARCHITECT:** This file is EMPTY. You MUST prioritize a task for the Coder to analyze the source code and define the Project Overview, Architecture, and Key Technologies.
-**CODER:** Use your tools to read the code, write the real content, and DELETE this entire stub header.
-
-## Description
-(Placeholder: A ${staticFacts.type || 'project'} project)
-
-## Architecture
-(Placeholder: TBD)`
+      'CONTEXT.md': {
+        prompt: `Analyze these project files and dependencies:\n${staticFactsJson}\nGenerate a professional # Project Overview in Markdown. Describe the actual architecture and purpose of this specific project. Be technical and concise. Start with '# Project Overview'.`,
+        fallback: `# Project Overview
+
+## Description
+Project name: ${projName}
+Type: ${projType}
+AI Integration: ${projAi}
+
+## Architecture
+Stack: ${projStack}
+
+*(Auto-generated from setup.json)*
+`
       },
       'OPERATIONS.md': {
         prompt: `Look at the dependencies and files:\n${staticFactsJson}\nGenerate a Markdown guide for common operations (Setup, Run, Test, Build) using the correct commands for this tech stack. Start with '# Common Operations'.`,
@@ -938,11 +974,40 @@ ${staticFacts.type === 'nodejs' ? 'npm install' : 'TBD'}`
 
 **ARCHITECT:** Set a real roadmap based on user goals. **CODER:** Remove this stub marker once a real goal is added.`
       },
-      'TECH_DEBT.md': {
-        prompt: `Given this is a ${staticFacts.type} project, list 2-3 common technical debt items. Start with '# Technical Debt'.`,
-        fallback: `# [STUB: TECH_DEBT.MD]
-
-**CODER:** Scan for TODOs/FIXMEs or structural issues and list them here. Remove this stub marker.`
+      'TECH_DEBT.md': {
+        prompt: `Given this is a ${staticFacts.type} project, list 2-3 common technical debt items. Start with '# Technical Debt'.`,
+        fallback: `# [STUB: TECH_DEBT.MD]
+
+**CODER:** Scan for TODOs/FIXMEs or structural issues and list them here. Remove this stub marker.`
+      },
+      'DEPLOY_CHECKLIST.md': {
+        prompt: `Based on the project type (${staticFacts.type}), generate a pre-deployment checklist. Start with '# Deployment Checklist'.`,
+        fallback: `# [STUB: DEPLOY_CHECKLIST.MD]
+
+## 🚨 ATTENTION CODER
+Verify required build steps before deployment.
+
+## Pre-Deployment Checklist
+- [ ] Verify all tests pass
+- [ ] Build assets (e.g., npm run build)
+- [ ] Check environment variables
+
+**CODER:** Update this checklist with actual build/deploy steps for this project.`
+      },
+      'RUNTIME_STATE.md': {
+        prompt: `Based on the project type (${staticFacts.type}), generate a template for RUNTIME_STATE.md. Start with '# Runtime State'.`,
+        fallback: `# Runtime State
+
+## 🚨 ATTENTION CODER
+Always check this file and verify the actual runtime state (ports, running processes, env variables) BEFORE writing code. Update this file if ports or access methods change.
+
+- **Server:** e.g., running on port 3210
+- **Services:** e.g., Scraper running on port 3211
+- **Auth:** e.g., admin@local / password
+- **Verification Commands:**
+  - \`pm2 ls\`
+  - \`curl http://localhost:3210/health\`
+`
       },
       'JOURNAL.md': {
         fallback: `# Development Journal

package/src/utils/gitUtils.js

@@ -25,14 +25,18 @@ export async function getGitAnchor(repoPath) {
   }
 }
 
-export async function getChangedFiles(repoPath, anchorHash) {
-  try {
-    const { stdout } = await execa('git', ['diff', '--name-only', anchorHash, 'HEAD'], { cwd: repoPath });
-    return stdout.split('\n').filter(Boolean);
-  } catch (e) {
-    throw new Error(`Failed to get git diff: ${e.message}`);
-  }
-}
+export async function getChangedFiles(repoPath, anchorHash, includeWorkingTree = false) {
+  try {
+    const args = ['diff', '--name-only', anchorHash];
+    if (!includeWorkingTree) {
+      args.push('HEAD');
+    }
+    const { stdout } = await execa('git', args, { cwd: repoPath });
+    return stdout.split('\n').filter(Boolean);
+  } catch (e) {
+    throw new Error(`Failed to get git diff: ${e.message}`);
+  }
+}
 
 export async function getGitDiffOutput(repoPath, anchorHash, excludeFiles = []) {
   try {

package/src/utils/opencodeAgentsGenerator.js

@@ -1,5 +1,9 @@
 import fs from 'fs/promises';
 import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
 
 /**
  * Generates AGENTS.md for OpenCode integration (GLM Z.AI ecosystem only)
@@ -30,11 +34,12 @@ export async function generateOpenCodeAgents(repoPath, mode, tree, confidentialF
       color: '#10a37f'
     };
 
-    const templatePath = path.join(repoPath, '..', '..', 'src', 'templates', 'opencode', 'junior-architect.template.md');
+    const templatePath = path.join(__dirname, '..', 'templates', 'opencode', 'junior-architect.template.md');
     try {
       let templateContent = await fs.readFile(templatePath, 'utf-8');
       body = templateContent.replace('{{tree}}', tree);
     } catch (error) {
+      console.warn(`⚠️ Could not load JAZ template from ${templatePath}: ${error.message}`);
       body = `# 🧠 ROLE: Swarm Orchestrator (GLM-4.7)\n\nDirectory:\n\`\`\`\n${tree}\n\`\`\``;
     }
   } else {
@@ -48,10 +53,11 @@ export async function generateOpenCodeAgents(repoPath, mode, tree, confidentialF
       color: '#44BA81'
     };
 
-    const templatePath = path.join(repoPath, '..', '..', 'src', 'templates', 'opencode', 'coder.template.md');
+    const templatePath = path.join(__dirname, '..', 'templates', 'opencode', 'coder.template.md');
     try {
       body = await fs.readFile(templatePath, 'utf-8');
     } catch (error) {
+      console.warn(`⚠️ Could not load Coder template from ${templatePath}: ${error.message}`);
       body = `# 🛠️ ROLE: Expert Developer`;
     }
   }

package/src/utils/tokenEstimator.js

@@ -137,29 +137,29 @@ export async function addTrainingPoint(projectType, fileSizeInBytes, estimatedTo
   }
 }
 
-/**
- * Fetch global token weights from Telemetry Hub and merge them into local training data
- */
-export async function syncTokenWeights() {
-  try {
-    console.log('Fetching global token weights from Telemetry Hub...');
-    const res = await fetch('https://xelth.com/T/tokens/weights');
-    if (!res.ok) throw new Error(res.statusText);
-    const data = await res.json();
-
-    if (data && data.coefficients && Object.keys(data.coefficients).length > 0) {
-      const localData = await loadTrainingData();
-      // Global coefficients override local ones
-      localData.coefficients = { ...localData.coefficients, ...data.coefficients };
-      await saveTrainingData(localData);
-      console.log('Global token weights synchronized successfully.');
-    } else {
-      console.log('No global weights available yet.');
-    }
-  } catch (e) {
-    console.log('Failed to sync token weights: ' + e.message);
-  }
-}
+/**
+ * Fetch global token weights from Telemetry Hub and merge them into local training data
+ */
+export async function syncTokenWeights(silent = false) {
+  try {
+    if (!silent) console.log('Fetching global token weights from Telemetry Hub...');
+    const res = await fetch('https://xelth.com/T/tokens/weights');
+    if (!res.ok) throw new Error(res.statusText);
+    const data = await res.json();
+
+    if (data && data.coefficients && Object.keys(data.coefficients).length > 0) {
+      const localData = await loadTrainingData();
+      // Global coefficients override local ones
+      localData.coefficients = { ...localData.coefficients, ...data.coefficients };
+      await saveTrainingData(localData);
+      if (!silent) console.log('Global token weights synchronized successfully.');
+    } else {
+      if (!silent) console.log('No global weights available yet.');
+    }
+  } catch (e) {
+    if (!silent) console.log('Failed to sync token weights: ' + e.message);
+  }
+}
 
 /**
  * Update polynomial coefficients using least squares fitting
@@ -217,27 +217,31 @@ function updateCoefficients(data, projectType) {
   ];
 }
 
-/**
- * Show current estimation statistics
- */
-export async function showEstimationStats() {
-  const data = await loadTrainingData();
-  
-  console.log('\n📊 Token Estimation Statistics:');
-  console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
-  
-  for (const [projectType, coefficients] of Object.entries(data.coefficients)) {
-    const points = data.trainingPoints[projectType] || [];
-    console.log(`\n🔸 ${projectType}:`);
-    console.log(`   Coefficients: [${coefficients.map(c => c.toFixed(6)).join(', ')}]`);
-    console.log(`   Training points: ${points.length}`);
-    
-    if (points.length > 0) {
-      const errors = points.map(p => Math.abs(p.actualTokens - p.estimatedTokens));
-      const avgError = errors.reduce((a, b) => a + b, 0) / errors.length;
-      console.log(`   Average error: ${Math.round(avgError)} tokens`);
-    }
-  }
-  
-  console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
+/**
+ * Show current estimation statistics
+ */
+export async function showEstimationStats() {
+  const data = await loadTrainingData();
+  
+  console.log('\n📊 Token Estimation Statistics:');
+  console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
+  
+  for (const [projectType, coefficients] of Object.entries(data.coefficients)) {
+    const points = data.trainingPoints[projectType] || [];
+    console.log(`\n🔸 ${projectType}:`);
+    console.log(`   Coefficients: [${coefficients.map(c => c.toFixed(6)).join(', ')}]`);
+    console.log(`   Training points: ${points.length}`);
+    
+    if (points.length > 0) {
+      // Recalculate error against current coefficients, ignoring old stored estimate
+      const errors = points.map(p => {
+        const currentEstimate = evaluatePolynomial(coefficients, p.fileSizeInBytes);
+        return Math.abs(p.actualTokens - currentEstimate);
+      });
+      const avgError = errors.reduce((a, b) => a + b, 0) / errors.length;
+      console.log(`   Average error: ${Math.round(avgError)} tokens`);
+    }
+  }
+  
+  console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
 }