kumi 0.0.10 → 0.0.11

data/lib/kumi/analyzer.rb

@@ -7,15 +7,19 @@ module Kumi
     DEFAULT_PASSES = [
       Core::Analyzer::Passes::NameIndexer,                     # 1. Finds all names and checks for duplicates.
       Core::Analyzer::Passes::InputCollector,                  # 2. Collects field metadata from input declarations.
-      Core::Analyzer::Passes::DeclarationValidator,            # 3. Checks the basic structure of each rule.
-      Core::Analyzer::Passes::SemanticConstraintValidator,     # 4. Validates DSL semantic constraints at AST level.
-      Core::Analyzer::Passes::DependencyResolver,              # 5. Builds the dependency graph with conditional dependencies.
-      Core::Analyzer::Passes::UnsatDetector,                   # 6. Detects unsatisfiable constraints and analyzes cascade mutual exclusion.
-      Core::Analyzer::Passes::Toposorter,                      # 7. Creates the final evaluation order, allowing safe cycles.
-      Core::Analyzer::Passes::BroadcastDetector, # 8. Detects which operations should be broadcast over arrays (must run before type inference).
-      Core::Analyzer::Passes::TypeInferencer,                  # 9. Infers types for all declarations (uses vectorization metadata).
-      Core::Analyzer::Passes::TypeConsistencyChecker,          # 10. Validates declared vs inferred type consistency.
-      Core::Analyzer::Passes::TypeChecker                      # 11. Validates types using inferred information.
+      Core::Analyzer::Passes::DeclarationValidator,            # 4. Checks the basic structure of each rule.
+      Core::Analyzer::Passes::SemanticConstraintValidator,     # 5. Validates DSL semantic constraints at AST level.
+      Core::Analyzer::Passes::DependencyResolver,              # 6. Builds the dependency graph with conditional dependencies.
+      Core::Analyzer::Passes::UnsatDetector,                   # 7. Detects unsatisfiable constraints and analyzes cascade mutual exclusion.
+      Core::Analyzer::Passes::Toposorter,                      # 8. Creates the final evaluation order, allowing safe cycles.
+      Core::Analyzer::Passes::BroadcastDetector,               # 9. Detects which operations should be broadcast over arrays.
+      Core::Analyzer::Passes::TypeInferencerPass,              # 10. Infers types for all declarations (uses vectorization metadata).
+      Core::Analyzer::Passes::TypeConsistencyChecker,          # 11. Validates declared vs inferred type consistency.
+      Core::Analyzer::Passes::TypeChecker,                     # 12. Validates types using inferred information.
+      Core::Analyzer::Passes::InputAccessPlannerPass,          # 13. Plans access strategies for input fields.
+      Core::Analyzer::Passes::ScopeResolutionPass,             # 14. Plans execution scope and lifting needs for declarations.
+      Core::Analyzer::Passes::JoinReducePlanningPass,          # 15. Plans join/reduce operations (Generates IR Structs)
+      Core::Analyzer::Passes::LowerToIRPass # 16. Lowers the schema to IR (Generates IR Structs)
     ].freeze
 
     def self.analyze!(schema, passes: DEFAULT_PASSES, **opts)
@@ -33,7 +37,13 @@ module Kumi
         begin
           state = pass_instance.run(errors)
         rescue StandardError => e
-          errors << Core::ErrorReporter.create_error(e.message, location: nil, type: :semantic)
+          # TODO: - GREATLY improve this, need to capture the context of the error
+          # and the pass that failed and line number if relevant
+          pass_name = pass_class.name.split("::").last
+          message = "Error in Analysis Pass(#{pass_name}): #{e.message}"
+          errors << Core::ErrorReporter.create_error(message, location: nil, type: :semantic, backtrace: e.backtrace)
+
+          raise
         end
       end
       state
@@ -41,11 +51,14 @@ module Kumi
 
     def self.handle_analysis_errors(errors)
       type_errors = errors.select { |e| e.type == :type }
+      semantic_errors = errors.select { |e| e.type == :semantic }
       first_error_location = errors.first.location
 
       raise Errors::TypeError.new(format_errors(errors), first_error_location) if type_errors.any?
 
-      raise Errors::SemanticError.new(format_errors(errors), first_error_location)
+      raise Errors::SemanticError.new(format_errors(errors), first_error_location) if first_error_location || semantic_errors
+
+      raise Errors::AnalysisError.new(format_errors(errors))
     end
 
     def self.create_analysis_result(state)
@@ -63,7 +76,16 @@ module Kumi
     def self.format_errors(errors)
       return "" if errors.empty?
 
-      errors.map(&:to_s).join("\n")
+      backtrace = errors.first.backtrace
+
+      message = errors.map(&:to_s).join("\n")
+
+      message.tap do |msg|
+        if backtrace && !backtrace.empty?
+          msg << "\n\nBacktrace:\n"
+          msg << backtrace[0..10].join("\n") # Limit to first 10 lines for readability
+        end
+      end
     end
   end
 end

data/lib/kumi/compiler.rb

@@ -3,11 +3,6 @@
 module Kumi
   # Compiles an analyzed schema into executable lambdas
   class Compiler < Core::CompilerBase
-    include Kumi::Core::Compiler::ReferenceCompiler
-    include Kumi::Core::Compiler::PathTraversalCompiler
-    include Kumi::Core::Compiler::ExpressionCompiler
-    include Kumi::Core::Compiler::FunctionInvoker
-
     def self.compile(schema, analyzer:)
       new(schema, analyzer).compile
     end
@@ -18,13 +13,8 @@ module Kumi
     end
 
     def compile
-      build_index
-      @analysis.topo_order.each do |name|
-        decl = @index[name] or raise("Unknown binding #{name}")
-        compile_declaration(decl)
-      end
-
-      Core::CompiledSchema.new(@bindings.freeze)
+      # Switch to LIR: Use the analysis state instead of old compilation
+      Runtime::Executable.from_analysis(@analysis.state)
     end
   end
 end

data/lib/kumi/core/analyzer/passes/broadcast_detector.rb

@@ -5,11 +5,11 @@ module Kumi
     module Analyzer
       module Passes
         # Detects which operations should be broadcast over arrays
-        # DEPENDENCIES: :inputs, :declarations
+        # DEPENDENCIES: :input_metadata, :declarations
         # PRODUCES: :broadcasts
         class BroadcastDetector < PassBase
           def run(errors)
-            input_meta = get_state(:inputs) || {}
+            input_meta = get_state(:input_metadata) || {}
             definitions = get_state(:declarations) || {}
 
             # Find array fields with their element types
@@ -40,6 +40,10 @@ module Kumi
               result = analyze_value_vectorization(name, decl.expression, array_fields, nested_paths, vectorized_values, errors,
                                                    definitions)
 
+              if ENV["DEBUG_BROADCAST_CLEAN"]
+                puts "#{name}: #{result[:type]} #{format_broadcast_info(result)}"
+              end
+
               case result[:type]
               when :vectorized
                 compiler_metadata[:vectorized_operations][name] = result[:info]
@@ -70,6 +74,43 @@ module Kumi
 
           private
 
+          def infer_argument_scope(arg, array_fields, nested_paths)
+            case arg
+            when Kumi::Syntax::InputElementReference
+              if nested_paths.key?(arg.path)
+                # Extract scope from path - each array dimension in the path
+                arg.path.select.with_index { |_seg, i| nested_paths[arg.path[0..i]] }
+              else
+                arg.path.select { |seg| array_fields.key?(seg) }
+              end
+            when Kumi::Syntax::CallExpression
+              # For nested calls, find the deepest input reference
+              deepest_scope = []
+              arg.args.each do |nested_arg|
+                scope = infer_argument_scope(nested_arg, array_fields, nested_paths)
+                deepest_scope = scope if scope.length > deepest_scope.length
+              end
+              deepest_scope
+            else
+              []
+            end
+          end
+
+          def format_broadcast_info(result)
+            case result[:type]
+            when :vectorized
+              info = result[:info]
+              "→ #{info[:source]} (path: #{info[:path]&.join('.')})"
+            when :reduction
+              info = result[:info]
+              "→ fn:#{info[:function]} (arg: #{info[:argument]&.class&.name&.split('::')&.last})"
+            when :scalar
+              "→ scalar"
+            else
+              "→ #{result[:info]}"
+            end
+          end
+
           def compute_compilation_metadata(name, _decl, compiler_metadata, _vectorized_values, _array_fields)
             metadata = {
               operation_mode: :broadcast, # Default mode
@@ -169,7 +210,7 @@ module Kumi
             current_access_mode = parent_access_mode
             if current_meta[:type] == :array
               array_depth += 1
-              current_access_mode = current_meta[:access_mode] || :object # Default to :object if not specified
+              current_access_mode = current_meta[:access_mode] || :field # Default to :field if not specified
             end
 
             # If this field has children, recurse into them
@@ -208,7 +249,6 @@ module Kumi
               # Check if this path exists in nested_paths metadata (supports nested arrays)
               if nested_paths.key?(expr.path)
                 { type: :vectorized, info: { source: :nested_array_access, path: expr.path, nested_metadata: nested_paths[expr.path] } }
-              # Fallback to old array_fields detection for backward compatibility
               elsif array_fields.key?(expr.path.first)
                 { type: :vectorized, info: { source: :array_field_access, path: expr.path } }
               else
@@ -236,76 +276,106 @@ module Kumi
           end
 
           def analyze_call_vectorization(_name, expr, array_fields, nested_paths, vectorized_values, errors, definitions = nil)
-            # Check if this is a reduction function using function registry metadata
-            if Kumi::Registry.reducer?(expr.fn_name)
-              # Only treat as reduction if the argument is actually vectorized
-              arg_info = analyze_argument_vectorization(expr.args.first, array_fields, nested_paths, vectorized_values, definitions)
-              if arg_info[:vectorized]
-                # Pre-compute which argument indices need flattening
-                flatten_indices = []
-                expr.args.each_with_index do |arg, index|
-                  arg_vectorization = analyze_argument_vectorization(arg, array_fields, nested_paths, vectorized_values, definitions)
-                  flatten_indices << index if arg_vectorization[:vectorized]
-                end
-
-                { type: :reduction, info: {
-                  function: expr.fn_name,
-                  source: arg_info[:source],
-                  argument: expr.args.first,
-                  flatten_argument_indices: flatten_indices
-                } }
-              else
-                # Not a vectorized reduction - just a regular function call
-                { type: :scalar }
-              end
+            entry         = Kumi::Registry.entry(expr.fn_name)
+            is_reducer    = entry&.reducer
+            is_structure  = entry&.structure_function
 
-            else
+            # 1) Analyze all args once
+            arg_infos = expr.args.map do |arg|
+              analyze_argument_vectorization(arg, array_fields, nested_paths, vectorized_values, definitions)
+            end
+            vec_idx   = arg_infos.each_index.select { |i| arg_infos[i][:vectorized] }
+            vec_any   = !vec_idx.empty?
 
-              # Special case: cascade_and takes individual trait arguments
-              if expr.fn_name == :cascade_and
-                # Check if any of the individual arguments are vectorized traits
-                vectorized_trait = expr.args.find do |arg|
-                  arg.is_a?(Kumi::Syntax::DeclarationReference) && vectorized_values[arg.name]&.[](:vectorized)
-                end
-                if vectorized_trait
-                  return { type: :vectorized, info: { source: :cascade_condition_with_vectorized_trait, trait: vectorized_trait.name } }
-                end
+            # 2) Special form: cascade_and (vectorized if any trait arg is vectorized)
+            if expr.fn_name == :cascade_and
+              vectorized_trait = expr.args.find do |arg|
+                arg.is_a?(Kumi::Syntax::DeclarationReference) && vectorized_values[arg.name]&.[](:vectorized)
               end
-
-              # Analyze arguments to determine function behavior
-              arg_infos = expr.args.map do |arg|
-                analyze_argument_vectorization(arg, array_fields, nested_paths, vectorized_values, definitions)
+              if vectorized_trait
+                return { type: :vectorized,
+                         info: { source: :cascade_condition_with_vectorized_trait, trait: vectorized_trait&.name } }
               end
 
-              if arg_infos.any? { |info| info[:vectorized] }
-                # Check for dimension mismatches when multiple arguments are vectorized
-                vectorized_sources = arg_infos.select { |info| info[:vectorized] }.filter_map { |info| info[:array_source] }.uniq
+              return { type: :scalar }
+            end
 
-                if vectorized_sources.length > 1
-                  # Multiple different array sources - this is a dimension mismatch
-                  # Generate enhanced error message with type information
-                  enhanced_message = build_dimension_mismatch_error(expr, arg_infos, array_fields, vectorized_sources)
+            # 3) Reducers: only reduce when the input is actually vectorized
+            if is_reducer
+              return { type: :scalar } unless vec_any
 
-                  report_error(errors, enhanced_message, location: expr.loc, type: :semantic)
-                  return { type: :scalar } # Treat as scalar to prevent further errors
-                end
+              # which args were vectorized?
+              flatten_indices = vec_idx.dup
+              vectorized_arg_index = vec_idx.first
+              argument_ast = expr.args[vectorized_arg_index]
+
+              src_info = arg_infos[vectorized_arg_index]
+
+              return {
+                type: :reduction,
+                info: {
+                  function: expr.fn_name,
+                  source: src_info[:source],
+                  argument: argument_ast, # << keep AST of the vectorized argument
+                  flatten_argument_indices: flatten_indices
+                }
+              }
+            end
 
-                # Check if this is a structure function that should work on the array as-is
-                if structure_function?(expr.fn_name)
-                  # Structure functions like size should work on structure as-is (scalar)
-                  { type: :scalar }
+            # 4) Structure (non-reducer) functions like `size`
+            if is_structure
+              # If any arg is itself a PURE reducer call (e.g., size(sum(x))), the inner collapses first ⇒ outer is scalar
+              # But dual-nature functions (both reducer AND structure) should be treated as structure functions when nested
+              return { type: :scalar } if expr.args.any? do |a|
+                if a.is_a?(Kumi::Syntax::CallExpression)
+                  arg_entry = Kumi::Registry.entry(a.fn_name)
+                  arg_entry&.reducer && !arg_entry&.structure_function # Pure reducer only
                 else
-                  # This is a vectorized operation - broadcast over elements
-                  { type: :vectorized, info: {
-                    operation: expr.fn_name,
-                    vectorized_args: arg_infos.map.with_index { |info, i| [i, info[:vectorized]] }.to_h
-                  } }
+                  false
                 end
-              else
-                # No vectorized arguments - regular scalar function
-                { type: :scalar }
               end
+
+              # Structure fn over a vectorized element path ⇒ per-parent vectorization
+              return { type: :scalar } unless vec_any
+
+              src_info     = arg_infos[vec_idx.first]
+              parent_scope = src_info[:parent_scope] || src_info[:source] # fallback if analyzer encodes parent separately
+              return {
+                type: :vectorized,
+                info: {
+                  operation: expr.fn_name,
+                  source: src_info[:source],
+                  parent_scope: parent_scope,
+                  vectorized_args: vec_idx.to_h { |i| [i, true] }
+                }
+              }
+
+              # Structure fn over a scalar/materialized container ⇒ scalar
+
             end
+
+            # 5) Generic vectorized map (non-structure, non-reducer)
+            if vec_any
+              # Dimension / source compatibility check
+              sources = vec_idx.map { |i| arg_infos[i][:array_source] }.compact.uniq
+              if sources.size > 1
+                enhanced_message = build_dimension_mismatch_error(expr, arg_infos, array_fields, sources)
+                report_error(errors, enhanced_message, location: expr.loc, type: :semantic)
+                return { type: :scalar } # fail safe to prevent cascading errors
+              end
+
+              return {
+                type: :vectorized,
+                info: {
+                  operation: expr.fn_name,
+                  source: arg_infos[vec_idx.first][:source],
+                  vectorized_args: vec_idx.to_h { |i| [i, true] }
+                }
+              }
+            end
+
+            # 6) Pure scalar
+            { type: :scalar }
           end
 
           def structure_function?(fn_name)
@@ -337,9 +407,32 @@ module Kumi
               end
 
             when Kumi::Syntax::CallExpression
-              # Recursively check
+              # Recursively check nested call
               result = analyze_value_vectorization(nil, arg, array_fields, nested_paths, vectorized_values, [], definitions)
-              { vectorized: result[:type] == :vectorized, source: :expression }
+              # Handle different result types appropriately
+              case result[:type]
+              when :reduction
+                # Reductions can produce vectors if they preserve some dimensions
+                # This aligns with lower_to_ir logic for grouped reductions
+                info = result[:info]
+                if info && info[:argument]
+                  # Check if the reduction argument has array scope that would be preserved
+                  arg_scope = infer_argument_scope(info[:argument], array_fields, nested_paths)
+                  if arg_scope.length > 1
+                    # Multi-dimensional reduction - likely preserves outer dimension (per-player)
+                    { vectorized: true, source: :grouped_reduction, array_source: arg_scope.first }
+                  else
+                    # Single dimension or scalar reduction
+                    { vectorized: false, source: :scalar_from_reduction }
+                  end
+                else
+                  { vectorized: false, source: :scalar_from_reduction }
+                end
+              when :vectorized
+                { vectorized: true, source: :expression }
+              else
+                { vectorized: false, source: :scalar }
+              end
 
             else
               { vectorized: false }

data/lib/kumi/core/analyzer/passes/dependency_resolver.rb

@@ -26,7 +26,7 @@ module Kumi
 
           def run(errors)
             definitions = get_state(:declarations)
-            input_meta = get_state(:inputs)
+            input_meta = get_state(:input_metadata)
 
             dependency_graph = Hash.new { |h, k| h[k] = [] }
             reverse_dependencies = Hash.new { |h, k| h[k] = [] }

data/lib/kumi/core/analyzer/passes/input_access_planner_pass.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Core
+    module Analyzer
+      module Passes
+        class InputAccessPlannerPass < PassBase
+          def run(errors)
+            input_metadata = get_state(:input_metadata)
+
+            options = {
+              on_missing: :error,
+              key_policy: :indifferent
+            }
+
+            # TODO : Allow by input definition on policies or at least general policy definition
+            plans = Kumi::Core::Compiler::AccessPlanner.plan(input_metadata, options)
+
+            # Quick validation
+            validate_plans!(plans, errors)
+
+            # Create new state with access plans
+            state.with(:access_plans, plans.freeze)
+          end
+
+          private
+
+          def validate_plans!(plans, errors)
+            plans.each do |path, plan_list|
+              add_error(errors, nil, "No access plans generated for path: #{path}") if plan_list.nil? || plan_list.empty?
+
+              plan_list&.each do |plan|
+                unless plan[:operations].is_a?(Array)
+                  add_error(errors, nil, "Invalid operations for path #{path}: expected Array, got #{plan[:operations].class}")
+                end
+
+                unless plan[:mode].is_a?(Symbol)
+                  add_error(errors, nil, "Invalid mode for path #{path}: expected Symbol, got #{plan[:mode].class}")
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end