arcvision 0.2.14 → 0.2.15

package/src/engine/context_sorter.js

@@ -0,0 +1,59 @@
+/**
+ * Sort the context object deterministically to ensure consistent output
+ * @param {Object} context - The context object to sort
+ * @returns {Object} - The sorted context object
+ */
+function sortContext(context) {
+    // Create a copy of the context to avoid modifying the original
+    const sortedContext = { ...context };
+    
+    // Sort nodes by ID
+    if (Array.isArray(sortedContext.nodes)) {
+        sortedContext.nodes = [...sortedContext.nodes].sort((a, b) => a.id.localeCompare(b.id));
+    }
+    
+    // Sort edges by from, to, and relation
+    if (Array.isArray(sortedContext.edges)) {
+        sortedContext.edges = [...sortedContext.edges].sort((a, b) => {
+            if (a.from !== b.from) {
+                return a.from.localeCompare(b.from);
+            }
+            if (a.to !== b.to) {
+                return a.to.localeCompare(b.to);
+            }
+            return a.relation.localeCompare(b.relation);
+        });
+    }
+    
+    // Sort metrics alphabetically
+    if (sortedContext.metrics && typeof sortedContext.metrics === 'object') {
+        const sortedMetrics = {};
+        const keys = Object.keys(sortedContext.metrics).sort();
+        for (const key of keys) {
+            sortedMetrics[key] = sortedContext.metrics[key];
+        }
+        sortedContext.metrics = sortedMetrics;
+    }
+    
+    // Sort authority cores by authority score descending
+    if (Array.isArray(sortedContext.authority_cores)) {
+        sortedContext.authority_cores = [...sortedContext.authority_cores]
+            .sort((a, b) => (b.authorityScore || 0) - (a.authorityScore || 0));
+    }
+    
+    // Sort hidden coupling by count descending
+    if (Array.isArray(sortedContext.hidden_coupling)) {
+        sortedContext.hidden_coupling = [...sortedContext.hidden_coupling]
+            .sort((a, b) => (b.count || 0) - (a.count || 0));
+    }
+    
+    // Sort context surface by score descending
+    if (sortedContext.contextSurface && Array.isArray(sortedContext.contextSurface)) {
+        sortedContext.contextSurface = [...sortedContext.contextSurface]
+            .sort((a, b) => (b.score || 0) - (a.score || 0));
+    }
+    
+    return sortedContext;
+}
+
+module.exports = { sortContext };

package/src/engine/context_validator.js

@@ -0,0 +1,200 @@
+const fs = require('fs');
+const path = require('path');
+const Ajv = require('ajv');
+
+// Create AJV instance with options for detailed error reporting
+const ajv = new Ajv({ allErrors: true, strict: false });
+
+/**
+ * Validate the context object against the Arcvision schema and invariants using AJV
+ * @param {Object} context - The context object to validate
+ * @param {string} schemaPath - Path to the schema file (optional, defaults to standard schema)
+ * @returns {Object} - Validation result with valid flag and errors
+ */
+function validateContext(context, schemaPath = null) {
+    try {
+        // Use provided schema path or default to the standard schema
+        // When bundled, the schema file is located differently
+        let schemaFilePath;
+        if (schemaPath) {
+            schemaFilePath = schemaPath;
+        } else {
+            // Try multiple possible locations for the schema file
+            const possiblePaths = [
+                path.join(__dirname, '../../schema/arcvision_context_schema_v1.json'), // Standard path
+                path.join(__dirname, '../schema/arcvision_context_schema_v1.json'),    // Bundled path
+                path.join(__dirname, 'schema/arcvision_context_schema_v1.json'),       // Alternative bundled path
+                path.join(process.cwd(), 'schema/arcvision_context_schema_v1.json'),    // Current working directory
+            ];
+            
+            for (const possiblePath of possiblePaths) {
+                if (fs.existsSync(possiblePath)) {
+                    schemaFilePath = possiblePath;
+                    break;
+                }
+            }
+            
+            if (!schemaFilePath) {
+                console.warn('⚠️  Schema file not found, using basic validation only');
+                // Return early with basic validation if schema not found
+                return { valid: true, warnings: ['Schema validation skipped - schema file not found'] };
+            }
+        }
+        
+        if (!fs.existsSync(schemaFilePath)) {
+            console.warn(`⚠️  Schema file not found: ${schemaFilePath}, using basic validation only`);
+            // Return early with basic validation if schema not found
+            return { valid: true, warnings: ['Schema validation skipped - schema file not found'] };
+        }
+        
+        let schema;
+        try {
+            schema = JSON.parse(fs.readFileSync(schemaFilePath, 'utf8'));
+        } catch (error) {
+            console.warn(`⚠️  Failed to parse schema file: ${error.message}, using basic validation only`);
+            return { valid: true, warnings: ['Schema validation skipped - failed to parse schema file'] };
+        }
+        
+        // Compile the schema with AJV
+        let validate;
+        try {
+            validate = ajv.compile(schema);
+        } catch (error) {
+            console.warn(`⚠️  Failed to compile schema: ${error.message}, using basic validation only`);
+            return { valid: true, warnings: ['Schema validation skipped - failed to compile schema'] };
+        }
+        
+        // Validate the context against schema first
+        const schemaValid = validate(context);
+
+        if (!schemaValid) {
+            // Format errors for human readability
+            const errors = (validate.errors || []).map(err => {
+                const path = err.instancePath || "root";
+                const message = err.message || "schema violation";
+                return `${path}: ${message}`;
+            });
+
+            return { valid: false, errors, warnings: [] };
+        }
+
+        // Now validate against system invariants
+        const invariantValidation = validateAgainstInvariants(context);
+        
+        if (!invariantValidation.valid) {
+            return {
+                valid: false,
+                errors: invariantValidation.errors,
+                warnings: invariantValidation.warnings || []
+            };
+        }
+
+        return { valid: true, warnings: [] };
+    } catch (error) {
+        return {
+            valid: false,
+            errors: [`Validation error: ${error.message}`],
+            warnings: [],
+            details: null
+        };
+    }
+}
+
+/**
+ * Validate the context against its own invariants
+ * @param {Object} context - The context object to validate
+ * @returns {Object} - Validation result with valid flag and errors
+ */
+function validateAgainstInvariants(context) {
+    const errors = [];
+    
+    // Check if invariants exist in the context
+    if (!context.invariants || !Array.isArray(context.invariants)) {
+        // If no invariants exist, we consider this valid (they may not have been generated yet)
+        return { valid: true };
+    }
+    
+    // Validate each invariant
+    for (const invariant of context.invariants) {
+        // Check if required fields exist
+        if (!invariant.id) {
+            errors.push(`Invariant missing required 'id' field: ${JSON.stringify(invariant)}`);
+        }
+        
+        if (!invariant.statement) {
+            errors.push(`Invariant with id '${invariant.id}' missing required 'statement' field`);
+        }
+        
+        if (!invariant.scope) {
+            errors.push(`Invariant with id '${invariant.id}' missing required 'scope' field`);
+        }
+        
+        if (typeof invariant.critical_path !== 'boolean' && invariant.critical_path !== undefined) {
+            errors.push(`Invariant with id '${invariant.id}' has invalid 'critical_path' field (must be boolean)`);
+        }
+        
+        if (!invariant.status) {
+            errors.push(`Invariant with id '${invariant.id}' missing required 'status' field`);
+        }
+        
+        // Validate status enum
+        const validStatuses = ['active', 'disabled', 'pending', 'suspected'];
+        if (invariant.status && !validStatuses.includes(invariant.status)) {
+            errors.push(`Invariant with id '${invariant.id}' has invalid 'status' field: ${invariant.status}`);
+        }
+    }
+    
+    // Validate ownership section if present
+    if (context.ownership) {
+        for (const [concept, ownershipInfo] of Object.entries(context.ownership)) {
+            if (!ownershipInfo.source_of_truth) {
+                errors.push(`Ownership for concept '${concept}' missing required 'source_of_truth' field`);
+            }
+            
+            if (ownershipInfo.derived_in && !Array.isArray(ownershipInfo.derived_in)) {
+                errors.push(`Ownership for concept '${concept}' has invalid 'derived_in' field (must be array)`);
+            }
+            
+            if (ownershipInfo.consumers && !Array.isArray(ownershipInfo.consumers)) {
+                errors.push(`Ownership for concept '${concept}' has invalid 'consumers' field (must be array)`);
+            }
+            
+            if (ownershipInfo.invalidators && !Array.isArray(ownershipInfo.invalidators)) {
+                errors.push(`Ownership for concept '${concept}' has invalid 'invalidators' field (must be array)`);
+            }
+        }
+    }
+    
+    // Validate failure modes section if present
+    if (context.failure_modes && Array.isArray(context.failure_modes)) {
+        for (const failureMode of context.failure_modes) {
+            if (!failureMode.id) {
+                errors.push(`Failure mode missing required 'id' field: ${JSON.stringify(failureMode)}`);
+            }
+            
+            if (!failureMode.trigger) {
+                errors.push(`Failure mode with id '${failureMode.id}' missing required 'trigger' field`);
+            }
+            
+            if (!failureMode.broken_invariant) {
+                errors.push(`Failure mode with id '${failureMode.id}' missing required 'broken_invariant' field`);
+            }
+            
+            if (!failureMode.observed_symptom) {
+                errors.push(`Failure mode with id '${failureMode.id}' missing required 'observed_symptom' field`);
+            }
+            
+            if (!failureMode.root_cause) {
+                errors.push(`Failure mode with id '${failureMode.id}' missing required 'root_cause' field`);
+            }
+        }
+    }
+    
+    return {
+        valid: errors.length === 0,
+        errors,
+        warnings: []
+    };
+}
+
+module.exports = { validateContext };

package/src/engine/id-generator.js

@@ -0,0 +1,16 @@
+const crypto = require('crypto');
+
+/**
+ * Generate a stable, deterministic ID based on the file path
+ * Same path will always produce the same ID
+ * @param {string} relativePath - The relative path to generate ID for
+ * @returns {string} - 16-character deterministic ID
+ */
+function stableId(relativePath) {
+    // Normalize the path by replacing backslashes with forward slashes and converting to lowercase
+    const normalized = relativePath.replace(/\\/g, '/').toLowerCase();
+    // Generate SHA-256 hash and take first 16 characters
+    return crypto.createHash('sha256').update(normalized).digest('hex').substring(0, 16);
+}
+
+module.exports = { stableId };

package/src/engine/pass1_facts.js

@@ -0,0 +1,260 @@
+/**
+ * Pass 1: Syntactic Fact Emission
+ * 
+ * Objectives:
+ * - content-agnostic parsing (delegates to parser-enhanced)
+ * - emits raw indisputable facts: "File A imports string 'B'", "File A calls method 'foo'"
+ * - NO resolution of what 'B' is (that's Pass 2)
+ */
+
+const { glob } = require('glob');
+const path = require('path');
+const fs = require('fs');
+const parser = require('../core/parser-enhanced');
+const pluginManager = require('../plugins/plugin-manager');
+
+/**
+ * Executes Pass 1: Scans directory and emits raw facts
+ * @param {string} directory - Root directory to scan
+ * @param {Object} options - Scan options
+ * @returns {Promise<Array>} List of file nodes with raw metadata (facts)
+ */
+async function executePass1(directory, options = {}) {
+    console.log('🧱 PASS 1: Syntactic Fact Extraction...');
+
+    // Normalize helper
+    const normalize = p => p.replace(/\\/g, '/');
+
+    const scanOptions = {
+        ignore: ['**/node_modules/**', '**/.git/**', '**/dist/**', '**/build/**', ...options.ignore || []],
+        cwd: directory,
+        absolute: true
+    };
+
+    // Load plugins
+    const pluginDir = path.join(__dirname, '../plugins');
+    if (fs.existsSync(pluginDir)) {
+        pluginManager.loadPluginsFromDirectory(pluginDir);
+    }
+
+    // Find all relevant code files for structural analysis
+    const files = await glob('**/*.{js,jsx,ts,tsx,cjs,mjs,json,lua}', {
+        ...scanOptions,
+        ignore: [...scanOptions.ignore, 
+            '**/*.d.ts', 
+            '**/.next/**', 
+            '**/coverage/**', 
+            '**/arcvision.context.json', 
+            '**/package-lock.json', 
+            '**/yarn.lock',
+            // Additional non-code file types to exclude
+            '**/*.png', '**/*.jpg', '**/*.jpeg', '**/*.gif', '**/*.svg',
+            '**/*.pdf', '**/*.doc', '**/*.docx', '**/*.xls', '**/*.xlsx',
+            '**/*.zip', '**/*.tar', '**/*.gz', '**/*.exe', '**/*.dll',
+            '**/*.log', '**/*.tmp', '**/*.DS_Store', '**/Thumbs.db',
+            '**/*.lock', '**/node_modules/**', '**/.git/**', '**/dist/**', '**/build/**',
+            '**/target/**', '**/out/**', '**/vendor/**', '**/*.min.js', '**/*.bundle.js',
+            '**/public/**', '**/static/**', '**/assets/**', '**/images/**', '**/fonts/**',
+            '**/tests/**', '**/test/**', '**/__tests__/**', '**/__mocks__/**',
+            '**/*.test.js', '**/*.test.ts', '**/*.spec.js', '**/*.spec.ts',
+            '**/.*ignore', '**/.*config*', '**/.*rc', '**/.*lock', '**/.*cache',
+            '**/.*history', '**/.*backup', '**/.*temp', '**/.*tmp',
+            '**/README.*', '**/CHANGELOG.*', '**/TODO', '**/NOTES',
+            '**/Dockerfile', '**/docker-compose.yml', '**/*.dockerfile',
+            '**/Makefile', '**/*.mk', '**/*.pyc', '**/__pycache__/**',
+            '**/*.so', '**/*.dylib', '**/*.o', '**/*.obj', '**/*.jar', '**/*.war',
+            '**/*.lock', '**/*.pid', '**/*.seed', '**/*.pid', '**/*.sock'
+        ]
+    });
+    
+    // Use only code files for analysis
+    const allFiles = files;
+    
+    console.log(`   Identified ${allFiles.length} files for analysis`);
+
+    // Optimize concurrency based on file count to balance performance and memory usage
+    const BASE_CONCURRENCY = 25; // Reduced base concurrency for better resource management
+    const OPTIMAL_CONCURRENCY = Math.min(BASE_CONCURRENCY, Math.max(5, Math.floor(allFiles.length / 10))); // Adaptive concurrency
+    const CONCURRENCY = OPTIMAL_CONCURRENCY;
+    const rawNodes = [];
+    let totalFacts = 0;
+
+    // Process files in parallel batches
+    for (let i = 0; i < allFiles.length; i += CONCURRENCY) {
+        const batch = allFiles.slice(i, i + CONCURRENCY);
+        
+        // Process the batch in parallel
+        const promises = batch.map(async (file) => {
+            try {
+                const relativePath = path.relative(directory, file);
+                const normalizedRelativePath = normalize(relativePath);
+
+                let metadata = {};
+
+                if (file.endsWith('.json')) {
+                    const content = fs.readFileSync(file, 'utf-8');
+                    metadata = {
+                        id: file,
+                        isJson: true,
+                        raw: content
+                    };
+                } else if (file.endsWith('.lua')) {
+                    const content = fs.readFileSync(file, 'utf-8');
+                    metadata = {
+                        id: file,
+                        isLua: true,
+                        raw: content,
+                        // Extract potential invocations from Lua content
+                        potentialInvocations: extractLuaInvocations(content)
+                    };
+                } else {
+                    metadata = parser.parseFile(file);
+                }
+
+                metadata = await pluginManager.processFile(file, metadata);
+
+                // Determine node type based on file extension
+                const isLuaFile = file.endsWith('.lua');
+                const nodeType = isLuaFile ? 'execution_script' : 'file';
+                const nodeRole = isLuaFile ? 'atomic_redis_operation' : 'Structure';
+                
+                const node = {
+                    id: normalizedRelativePath,
+                    filePath: file,
+                    type: nodeType,
+                    role: nodeRole,
+                    facts: {
+                        imports: metadata.imports || [],
+                        exports: metadata.exports || [],
+                        functions: metadata.functions || [],
+                        classes: metadata.classes || [],
+                        types: metadata.types || [],
+                        calls: {
+                            functions: metadata.functionCalls || [],
+                            methods: metadata.methodCalls || [],
+                            constructors: metadata.constructorCalls || []
+                        },
+                        typeAnalysis: {
+                            typeImports: metadata.typeImports || [],
+                            interfaceDeps: metadata.interfaceDependencies || [],
+                            genericDeps: metadata.genericDependencies || []
+                        },
+                        di: {
+                            injections: metadata.constructorInjections || [],
+                            hooks: metadata.hookDependencies || [],
+                            context: metadata.contextUsages || []
+                        },
+                        react: metadata.componentUsage || [],
+                        potentialInvocations: metadata.potentialInvocations || [],
+                        invariantIndicators: extractInvariantIndicators(metadata.raw || '')
+                    }
+                };
+                
+                // Add language and execution boundary for Lua files
+                if (isLuaFile) {
+                    node.language = 'lua';
+                    node.execution_boundary = 'redis';
+                }
+
+                return { node, factCount: (metadata.imports?.length || 0) + (metadata.functionCalls?.length || 0) + (metadata.invariantIndicators?.length || 0) };
+            } catch (e) {
+                // Only warn if we're not in a high-performance mode
+                if (allFiles.length < 1000) { // Only show warnings for smaller projects
+                    console.warn(`⚠️ Pass 1 failed for ${file}: ${e.message}`);
+                }
+                return null;
+            }
+        });
+
+        const results = await Promise.all(promises);
+        
+        results.filter(Boolean).forEach(res => {
+            rawNodes.push(res.node);
+            totalFacts += res.factCount;
+        });
+
+        if (i % (CONCURRENCY * 4) === 0 && i > 0) {
+            console.log(`   ... processed ${Math.min(i + CONCURRENCY, allFiles.length)} / ${allFiles.length} files`);
+        }
+    }
+
+    console.log(`   ✓ Scanned ${rawNodes.length} files`);
+    console.log(`   ✓ Extracted ${totalFacts} raw syntactic facts (including invariant indicators)`);
+
+    return rawNodes;
+}
+
+/**
+ * Extract potential Lua script invocations from content
+ * @param {string} content - Lua script content
+ * @returns {Array} Array of potential invocations
+ */
+function extractLuaInvocations(content) {
+    const invocations = [];
+    
+    // Look for common BullMQ Lua script patterns
+    const patterns = [
+        /readFileSync\(['"].*\.lua['"].*\)/gi,
+        /defineCommand\(\s*\{\s*lua\s*:.*\}/gi,
+        /redis\.call\(['"].*['"].*\)/gi,
+        /redis\.pcall\(['"].*['"].*\)/gi
+    ];
+    
+    patterns.forEach(pattern => {
+        let match;
+        while ((match = pattern.exec(content)) !== null) {
+            invocations.push(match[0]);
+        }
+    });
+    
+    return invocations;
+}
+
+/**
+ * Extract potential invariant indicators from content
+ * @param {string} content - File content
+ * @returns {Array} Array of potential invariant indicators
+ */
+function extractInvariantIndicators(content) {
+    const indicators = [];
+    
+    // Look for common invariant/pattern validation patterns
+    const patterns = [
+        /assert\(/gi,                    // assert() calls
+        /validate[A-Z]/gi,              // validate functions
+        /check[A-Z]/gi,                 // check functions
+        /ensure[A-Z]/gi,                // ensure functions
+        /guard[A-Z]/gi,                 // guard functions
+        /verify[A-Z]/gi,                // verify functions
+        /Invariant/gi,                  // Invariant mentions
+        /constraint/gi,                 // constraint mentions
+        /must.*be/gi,                   // must-be patterns
+        /should.*be/gi,                 // should-be patterns
+        /expected.*to/gi,               // expected-to patterns
+        /boundary.*check/gi,            // boundary checks
+        /range.*check/gi,               // range checks
+        /within.*bounds/gi,             // bounds checks
+        /access.*control/gi,            // access controls
+        /permission.*check/gi,          // permission checks
+        /if.*!condition.*throw/gi,      // guard clauses
+        /state.*transition/gi,          // state transitions
+        /setState/gi,                   // state setters
+        /update.*state/gi,              // state updaters
+        /transition/gi                  // transition patterns
+    ];
+    
+    patterns.forEach(pattern => {
+        let match;
+        while ((match = pattern.exec(content)) !== null) {
+            indicators.push({
+                pattern: pattern.source,
+                match: match[0],
+                position: match.index
+            });
+        }
+    });
+    
+    return indicators;
+}
+
+module.exports = { executePass1 };