@unrdf/kgn 5.0.1

package/src/parser/variables.js

@@ -0,0 +1,344 @@
+/**
+ * Variable Extractor - Extract and validate template variables
+ * Migrated from ~/unjucks with enhanced pattern matching
+ */
+
+export class VariableExtractor {
+  constructor(options = {}) {
+    this.includeFilters = options.includeFilters !== false;
+    this.includeFunctions = options.includeFunctions !== false;
+    this.strict = options.strict !== false;
+  }
+
+  /**
+   * Extract all variables from template content
+   */
+  extract(content) {
+    const variables = new Set();
+    const filters = new Set();
+    const functions = new Set();
+
+    // Extract {{ variable }} patterns
+    this.extractOutputVariables(content, variables, filters);
+    
+    // Extract {% for variable in ... %} patterns
+    this.extractLoopVariables(content, variables);
+    
+    // Extract {% if variable %} patterns
+    this.extractConditionalVariables(content, variables);
+    
+    // Extract {% set variable = ... %} patterns
+    this.extractAssignmentVariables(content, variables);
+    
+    // Extract function calls
+    if (this.includeFunctions) {
+      this.extractFunctionCalls(content, functions);
+    }
+
+    return {
+      variables: Array.from(variables).sort(),
+      filters: this.includeFilters ? Array.from(filters).sort() : [],
+      functions: this.includeFunctions ? Array.from(functions).sort() : [],
+      totalVariables: variables.size,
+      complexity: this.calculateComplexity(content)
+    };
+  }
+
+  /**
+   * Extract variables from output expressions {{ ... }}
+   */
+  extractOutputVariables(content, variables, filters) {
+    // Match {{ variable | filter1 | filter2 }} patterns
+    const outputPattern = /\{\{\s*([^}]+?)\s*\}\}/g;
+    let match;
+
+    while ((match = outputPattern.exec(content)) !== null) {
+      const expression = match[1].trim();
+      this.parseExpression(expression, variables, filters);
+    }
+  }
+
+  /**
+   * Extract variables from loop constructs
+   */
+  extractLoopVariables(content, variables) {
+    // Match {% for item in collection %} patterns
+    const forPattern = /\{\%\s*for\s+(\w+)(?:\s*,\s*(\w+))?\s+in\s+([^%]+?)\s*\%\}/g;
+    let match;
+
+    while ((match = forPattern.exec(content)) !== null) {
+      const [, itemVar, indexVar, collection] = match;
+      
+      // Collection is a variable we need
+      const collectionVar = collection.trim().split('.')[0].split('|')[0].trim();
+      if (collectionVar && this.isValidVariable(collectionVar)) {
+        variables.add(collectionVar);
+      }
+      
+      // Note: itemVar and indexVar are loop-local, not template variables
+    }
+  }
+
+  /**
+   * Extract variables from conditional statements
+   */
+  extractConditionalVariables(content, variables) {
+    // Match {% if condition %} patterns
+    const ifPattern = /\{\%\s*if\s+([^%]+?)\s*\%\}/g;
+    const elifPattern = /\{\%\s*elif\s+([^%]+?)\s*\%\}/g;
+    
+    let match;
+
+    // Process if statements
+    while ((match = ifPattern.exec(content)) !== null) {
+      const condition = match[1].trim();
+      this.parseCondition(condition, variables);
+    }
+
+    // Process elif statements
+    while ((match = elifPattern.exec(content)) !== null) {
+      const condition = match[1].trim();
+      this.parseCondition(condition, variables);
+    }
+  }
+
+  /**
+   * Extract variables from assignment statements
+   */
+  extractAssignmentVariables(content, variables) {
+    // Match {% set variable = expression %} patterns
+    const setPattern = /\{\%\s*set\s+\w+\s*=\s*([^%]+?)\s*\%\}/g;
+    let match;
+
+    while ((match = setPattern.exec(content)) !== null) {
+      const expression = match[1].trim();
+      this.parseExpression(expression, variables, new Set());
+    }
+  }
+
+  /**
+   * Extract function calls from template
+   */
+  extractFunctionCalls(content, functions) {
+    // Match function calls like func(args)
+    const functionPattern = /(\w+)\s*\(/g;
+    let match;
+
+    while ((match = functionPattern.exec(content)) !== null) {
+      const funcName = match[1];
+      if (!this.isBuiltinFunction(funcName)) {
+        functions.add(funcName);
+      }
+    }
+  }
+
+  /**
+   * Parse expression to extract variables and filters
+   */
+  parseExpression(expression, variables, filters) {
+    // Handle complex expressions with operators
+    const parts = expression.split(/[+\-*\/\(\)]/);
+    
+    for (let part of parts) {
+      part = part.trim();
+      
+      // Skip empty parts and literals
+      if (!part || this.isLiteral(part)) continue;
+      
+      // Check for filter chain: variable | filter1 | filter2
+      if (part.includes('|')) {
+        const [varPart, ...filterParts] = part.split('|').map(p => p.trim());
+        
+        // Extract variable
+        const rootVar = this.extractRootVariable(varPart);
+        if (rootVar && this.isValidVariable(rootVar)) {
+          variables.add(rootVar);
+        }
+        
+        // Extract filters
+        filterParts.forEach(filter => {
+          const filterName = filter.split('(')[0].trim();
+          if (filterName && this.includeFilters) {
+            filters.add(filterName);
+          }
+        });
+      } else {
+        // Simple variable reference
+        const rootVar = this.extractRootVariable(part);
+        if (rootVar && this.isValidVariable(rootVar)) {
+          variables.add(rootVar);
+        }
+      }
+    }
+  }
+
+  /**
+   * Parse conditional expression to extract variables
+   */
+  parseCondition(condition, variables) {
+    // Handle operators: and, or, not, ==, !=, <, >, <=, >=, in, is
+    const conditionParts = condition.split(/\s+(?:and|or|not|==|!=|<=|>=|<|>|in|is)\s+/);
+    
+    conditionParts.forEach(part => {
+      part = part.trim();
+      if (part && !this.isLiteral(part)) {
+        const rootVar = this.extractRootVariable(part);
+        if (rootVar && this.isValidVariable(rootVar)) {
+          variables.add(rootVar);
+        }
+      }
+    });
+  }
+
+  /**
+   * Extract root variable from complex expression
+   */
+  extractRootVariable(expression) {
+    // Handle dot notation: user.name -> user
+    // Handle array access: users[0] -> users
+    // Handle function calls: len(users) -> users
+    
+    let cleaned = expression.trim();
+    
+    // Remove function calls but keep arguments
+    cleaned = cleaned.replace(/\w+\(([^)]+)\)/g, '$1');
+    
+    // Extract variable before dot or bracket
+    const match = cleaned.match(/^([a-zA-Z_][a-zA-Z0-9_]*)/);
+    return match ? match[1] : null;
+  }
+
+  /**
+   * Check if value is a literal (string, number, boolean)
+   */
+  isLiteral(value) {
+    // String literals
+    if ((value.startsWith('"') && value.endsWith('"')) || 
+        (value.startsWith("'") && value.endsWith("'"))) {
+      return true;
+    }
+    
+    // Number literals
+    if (/^-?\d+(\.\d+)?$/.test(value)) {
+      return true;
+    }
+    
+    // Boolean literals
+    if (['true', 'false', 'True', 'False'].includes(value)) {
+      return true;
+    }
+    
+    // None/null literals
+    if (['none', 'null', 'None', 'Null'].includes(value)) {
+      return true;
+    }
+    
+    return false;
+  }
+
+  /**
+   * Check if variable name is valid
+   */
+  isValidVariable(name) {
+    // Valid variable names: letters, numbers, underscore
+    // Must start with letter or underscore
+    return /^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name) && 
+           !this.isBuiltinVariable(name);
+  }
+
+  /**
+   * Check if variable is a builtin
+   */
+  isBuiltinVariable(name) {
+    const builtins = [
+      'loop', 'super', 'self', 'varargs', 'kwargs', 
+      'joiner', 'cycler', 'range', 'lipsum'
+    ];
+    return builtins.includes(name);
+  }
+
+  /**
+   * Check if function is a builtin
+   */
+  isBuiltinFunction(name) {
+    const builtins = [
+      'range', 'lipsum', 'dict', 'list', 'tuple', 'set',
+      'len', 'str', 'int', 'float', 'bool'
+    ];
+    return builtins.includes(name);
+  }
+
+  /**
+   * Calculate template complexity
+   */
+  calculateComplexity(content) {
+    const patterns = {
+      variables: /\{\{\s*[^}]+?\s*\}\}/g,
+      conditions: /\{\%\s*if\s+[^%]+?\s*\%\}/g,
+      loops: /\{\%\s*for\s+[^%]+?\s*\%\}/g,
+      includes: /\{\%\s*include\s+[^%]+?\s*\%\}/g,
+      macros: /\{\%\s*macro\s+[^%]+?\s*\%\}/g,
+      blocks: /\{\%\s*block\s+[^%]+?\s*\%\}/g
+    };
+
+    let complexity = 0;
+    Object.values(patterns).forEach(pattern => {
+      const matches = content.match(pattern);
+      complexity += matches ? matches.length : 0;
+    });
+
+    return complexity;
+  }
+
+  /**
+   * Validate extracted variables against provided context
+   */
+  validateContext(extractedVars, context, options = {}) {
+    const errors = [];
+    const warnings = [];
+    
+    const availableVars = new Set([
+      ...Object.keys(context),
+      '__meta', // Always available
+      ...options.additionalVars || []
+    ]);
+
+    extractedVars.forEach(varName => {
+      if (!availableVars.has(varName)) {
+        if (options.strictMode) {
+          errors.push(`Required variable '${varName}' not found in context`);
+        } else {
+          warnings.push(`Variable '${varName}' not found in context`);
+        }
+      }
+    });
+
+    // Check for unused context variables
+    if (options.warnUnused) {
+      Object.keys(context).forEach(contextVar => {
+        if (!extractedVars.includes(contextVar) && !contextVar.startsWith('_')) {
+          warnings.push(`Context variable '${contextVar}' is unused in template`);
+        }
+      });
+    }
+
+    return {
+      valid: errors.length === 0,
+      errors,
+      warnings
+    };
+  }
+
+  /**
+   * Get extractor statistics
+   */
+  getStats() {
+    return {
+      includeFilters: this.includeFilters,
+      includeFunctions: this.includeFunctions,
+      strict: this.strict
+    };
+  }
+}
+
+export default VariableExtractor;

package/src/renderer/deterministic.js

@@ -0,0 +1,245 @@
+/**
+ * Deterministic Renderer - Ensures identical output across template runs
+ * Migrated from ~/unjucks with enhanced deterministic guarantees
+ */
+
+import crypto from 'crypto';
+
+export class DeterministicRenderer {
+  constructor(options = {}) {
+    this.staticBuildTime = options.staticBuildTime || '2024-01-01T00:00:00.000Z';
+    this.blockNonDeterministic = options.blockNonDeterministic !== false;
+    this.strictMode = options.strictMode !== false;
+  }
+
+  /**
+   * Render template with deterministic context
+   */
+  async render(nunjucksEnv, templateContent, context) {
+    try {
+      // Create deterministic context
+      const deterministicContext = this.createDeterministicContext(context);
+      
+      // Block non-deterministic operations if enabled
+      if (this.blockNonDeterministic) {
+        this.validateDeterministicContext(deterministicContext);
+      }
+
+      // Render with enhanced error handling
+      const rendered = nunjucksEnv.renderString(templateContent, deterministicContext);
+      
+      return this.postProcessOutput(rendered);
+
+    } catch (error) {
+      if (this.strictMode) {
+        throw new Error(`Deterministic rendering failed: ${error.message}`);
+      }
+      
+      // Fallback to non-deterministic rendering
+      console.warn('[DeterministicRenderer] Fallback to non-deterministic rendering:', error.message);
+      return nunjucksEnv.renderString(templateContent, context);
+    }
+  }
+
+  /**
+   * Render string template with deterministic context
+   */
+  renderString(nunjucksEnv, templateContent, context) {
+    const deterministicContext = this.createDeterministicContext(context);
+    const rendered = nunjucksEnv.renderString(templateContent, deterministicContext);
+    return this.postProcessOutput(rendered);
+  }
+
+  /**
+   * Create deterministic context by replacing non-deterministic values
+   */
+  createDeterministicContext(context) {
+    const deterministicContext = { ...context };
+
+    // Add deterministic metadata
+    deterministicContext.__deterministic = {
+      buildTime: this.staticBuildTime,
+      renderTime: this.staticBuildTime, // Use same time for consistency
+      hash: this.hashContent(JSON.stringify(context)),
+      mode: 'deterministic'
+    };
+
+    // Override potentially non-deterministic built-ins
+    this.overrideNonDeterministicValues(deterministicContext);
+
+    return deterministicContext;
+  }
+
+  /**
+   * Override non-deterministic values in context
+   */
+  overrideNonDeterministicValues(context) {
+    // Replace any Date objects with static date
+    this.replaceInObject(context, (key, value) => {
+      if (value instanceof Date) {
+        return new Date(this.staticBuildTime);
+      }
+      
+      // Replace current timestamp references
+      if (typeof value === 'string' && this.looksLikeTimestamp(value)) {
+        return this.staticBuildTime;
+      }
+      
+      return value;
+    });
+  }
+
+  /**
+   * Recursively replace values in object
+   */
+  replaceInObject(obj, replacer) {
+    for (const [key, value] of Object.entries(obj)) {
+      if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
+        this.replaceInObject(value, replacer);
+      } else if (Array.isArray(value)) {
+        for (let i = 0; i < value.length; i++) {
+          if (typeof value[i] === 'object' && value[i] !== null) {
+            this.replaceInObject(value[i], replacer);
+          } else {
+            value[i] = replacer(i.toString(), value[i]);
+          }
+        }
+      } else {
+        obj[key] = replacer(key, value);
+      }
+    }
+  }
+
+  /**
+   * Check if string looks like a timestamp
+   */
+  looksLikeTimestamp(value) {
+    // ISO 8601 format
+    if (/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{3})?Z?$/.test(value)) {
+      return true;
+    }
+    
+    // Unix timestamp (seconds or milliseconds)
+    if (/^\d{10,13}$/.test(value)) {
+      return true;
+    }
+    
+    return false;
+  }
+
+  /**
+   * Validate context doesn't contain non-deterministic operations
+   */
+  validateDeterministicContext(context) {
+    const nonDeterministicPatterns = [
+      /Math\.random/,
+      /Date\.now/,
+      /new Date\(\)/,
+      /crypto\.randomUUID/,
+      /Math\.floor\(Math\.random/
+    ];
+
+    const contextStr = JSON.stringify(context);
+    
+    nonDeterministicPatterns.forEach(pattern => {
+      if (pattern.test(contextStr)) {
+        throw new Error(`Non-deterministic operation detected: ${pattern.source}`);
+      }
+    });
+  }
+
+  /**
+   * Post-process rendered output for consistency
+   */
+  postProcessOutput(output) {
+    // Normalize line endings
+    let processed = output.replace(/\r\n|\r/g, '\n');
+    
+    // Remove trailing whitespace from lines (but keep empty lines)
+    processed = processed.replace(/[^\S\n]+$/gm, '');
+    
+    // Ensure consistent final newline
+    if (processed.length > 0 && !processed.endsWith('\n')) {
+      processed += '\n';
+    }
+
+    return processed;
+  }
+
+  /**
+   * Get deterministic time string
+   */
+  getDeterministicTime() {
+    return this.staticBuildTime;
+  }
+
+  /**
+   * Generate deterministic hash
+   */
+  hashContent(content) {
+    return crypto.createHash('sha256').update(content, 'utf8').digest('hex');
+  }
+
+  /**
+   * Verify template produces identical output across runs
+   */
+  async verifyDeterminism(nunjucksEnv, templateContent, context, iterations = 3) {
+    const outputs = [];
+    const hashes = new Set();
+
+    for (let i = 0; i < iterations; i++) {
+      const output = await this.render(nunjucksEnv, templateContent, context);
+      const hash = this.hashContent(output);
+      
+      outputs.push({ iteration: i + 1, output, hash });
+      hashes.add(hash);
+    }
+
+    const isDeterministic = hashes.size === 1;
+    
+    return {
+      isDeterministic,
+      iterations,
+      uniqueOutputs: hashes.size,
+      outputs: outputs.slice(0, 2), // Include first 2 for comparison
+      firstHash: outputs[0]?.hash,
+      allHashesSame: isDeterministic
+    };
+  }
+
+  /**
+   * Create reproducibility report
+   */
+  async createReproducibilityReport(nunjucksEnv, templateContent, context) {
+    const verification = await this.verifyDeterminism(nunjucksEnv, templateContent, context, 5);
+    
+    const report = {
+      timestamp: this.getDeterministicTime(),
+      templateHash: this.hashContent(templateContent),
+      contextHash: this.hashContent(JSON.stringify(context)),
+      deterministicSettings: {
+        staticBuildTime: this.staticBuildTime,
+        blockNonDeterministic: this.blockNonDeterministic,
+        strictMode: this.strictMode
+      },
+      verification,
+      reproducible: verification.isDeterministic,
+      confidence: verification.isDeterministic ? 'HIGH' : 'LOW'
+    };
+
+    return report;
+  }
+
+  /**
+   * Get renderer statistics
+   */
+  getStats() {
+    return {
+      staticBuildTime: this.staticBuildTime,
+      blockNonDeterministic: this.blockNonDeterministic,
+      strictMode: this.strictMode
+    };
+  }
+}
+
+export default DeterministicRenderer;

package/src/renderer/index.js

@@ -0,0 +1,6 @@
+/**
+ * Renderer Module Exports
+ */
+
+export { DeterministicRenderer } from './deterministic.js';
+export default { DeterministicRenderer };