kumi 0.0.10 → 0.0.11

data/lib/kumi/core/analyzer/passes/pass_base.rb

@@ -27,10 +27,10 @@ module Kumi
 
           attr_reader :schema, :state
 
-          # Iterate over all declarations (attributes and traits) in the schema
+          # Iterate over all declarations (values and traits) in the schema
           # @yield [Syntax::Attribute|Syntax::Trait] Each declaration
           def each_decl(&block)
-            schema.attributes.each(&block)
+            schema.values.each(&block)
             schema.traits.each(&block)
           end
 

data/lib/kumi/core/analyzer/passes/scope_resolution_pass.rb

@@ -0,0 +1,346 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Core
+    module Analyzer
+      module Passes
+        # Plans per-declaration execution scope and join/lift needs.
+        # Determines the dimensional scope (array nesting level) for each declaration
+        # based on vectorization metadata and input paths.
+        #
+        # DEPENDENCIES: :declarations, :input_metadata, :broadcasts
+        # PRODUCES: :scope_plans, :decl_shapes
+        class ScopeResolutionPass < PassBase
+          include Kumi::Core::Analyzer::Plans
+
+          def run(_errors)
+            declarations = get_state(:declarations, required: true)
+            input_metadata = get_state(:input_metadata, required: true)
+            broadcasts = get_state(:broadcasts) || {}
+            dependencies = get_state(:dependencies) || {}
+            
+            puts "Available dependencies: #{dependencies.keys.inspect}" if ENV["DEBUG_SCOPE_RESOLUTION"]
+
+            scope_plans = {}
+            decl_shapes = {}
+
+            initial_scopes = {}
+            declarations.each do |name, decl|
+              debug_output(name, decl) if ENV["DEBUG_SCOPE_RESOLUTION"]
+              target_scope = infer_target_scope(name, decl, broadcasts, input_metadata)
+              result_kind = determine_result_kind(name, target_scope, broadcasts)
+              initial_scopes[name] = target_scope
+              debug_result(target_scope, result_kind) if ENV["DEBUG_SCOPE_RESOLUTION"]
+            end
+
+            final_scopes = propagate_scope_constraints(initial_scopes, declarations, input_metadata)
+            final_scopes.each do |name, target_scope|
+              result_kind = determine_result_kind(name, target_scope, broadcasts)
+              plan = build_scope_plan(target_scope)
+              scope_plans[name] = plan
+              decl_shapes[name] = { scope: target_scope, result: result_kind }.freeze
+            end
+
+            # Return new state with scope information
+            state.with(:scope_plans, scope_plans.freeze)
+                 .with(:decl_shapes, decl_shapes.freeze)
+          end
+
+          private
+
+          def propagate_scope_constraints(initial_scopes, declarations, input_metadata)
+            scopes = initial_scopes.dup
+            puts "\n=== Propagating scope constraints ===" if ENV["DEBUG_SCOPE_RESOLUTION"]
+
+            declarations.each do |name, decl|
+              case decl.expression
+              when Kumi::Syntax::ArrayExpression
+                propagate_from_array_expression(name, decl.expression, scopes, declarations, input_metadata)
+              when Kumi::Syntax::CascadeExpression
+                propagate_from_cascade_expression(name, decl.expression, scopes, declarations, input_metadata)
+              end
+            end
+
+            puts "Final propagated scopes: #{scopes.inspect}" if ENV["DEBUG_SCOPE_RESOLUTION"]
+            scopes
+          end
+
+          def propagate_from_array_expression(name, array_expr, scopes, declarations, input_metadata)
+            puts "Analyzing array expression in #{name}: #{array_expr.elements.map(&:class)}" if ENV["DEBUG_SCOPE_RESOLUTION"]
+
+            anchor_scope = nil
+            declaration_refs = []
+
+            array_expr.elements.each do |element|
+              case element
+              when Kumi::Syntax::InputElementReference
+                path_scope = dims_from_path(element.path, input_metadata)
+                puts "Found input anchor: #{element.path} -> scope #{path_scope}" if ENV["DEBUG_SCOPE_RESOLUTION"]
+                anchor_scope = path_scope if path_scope.length > (anchor_scope&.length || 0)
+              when Kumi::Syntax::DeclarationReference
+                declaration_refs << element.name
+              end
+            end
+
+            if anchor_scope && !anchor_scope.empty?
+              declaration_refs.each do |ref_name|
+                current_scope = scopes[ref_name] || []
+                if anchor_scope.length > current_scope.length
+                  puts "Propagating scope #{anchor_scope} to #{ref_name} (was #{current_scope})" if ENV["DEBUG_SCOPE_RESOLUTION"]
+                  scopes[ref_name] = anchor_scope
+                  propagate_to_dependencies(ref_name, anchor_scope, scopes, declarations, input_metadata)
+                end
+              end
+            end
+          end
+
+          def propagate_from_cascade_expression(name, cascade_expr, scopes, declarations, input_metadata)
+            puts "Analyzing cascade expression in #{name}" if ENV["DEBUG_SCOPE_RESOLUTION"]
+            
+            # Cascade should propagate its own scope to condition dependencies
+            cascade_scope = scopes[name] || []
+            return if cascade_scope.empty?
+            
+            puts "Propagating cascade scope #{cascade_scope} to condition dependencies" if ENV["DEBUG_SCOPE_RESOLUTION"]
+            
+            cascade_expr.cases.each do |case_expr|
+              find_declaration_references(case_expr.condition).each do |ref_name|
+                current_scope = scopes[ref_name] || []
+                if cascade_scope.length > current_scope.length
+                  puts "Propagating scope #{cascade_scope} to cascade condition #{ref_name} (was #{current_scope})" if ENV["DEBUG_SCOPE_RESOLUTION"]
+                  scopes[ref_name] = cascade_scope
+                  propagate_to_dependencies(ref_name, cascade_scope, scopes, declarations, input_metadata)
+                end
+              end
+            end
+          end
+
+          def propagate_to_dependencies(decl_name, required_scope, scopes, declarations, input_metadata)
+            return unless declarations[decl_name]
+            
+            decl = declarations[decl_name]
+            puts "Propagating #{required_scope} into dependencies of #{decl_name}" if ENV["DEBUG_SCOPE_RESOLUTION"]
+            
+            case decl.expression
+            when Kumi::Syntax::CascadeExpression
+              decl.expression.cases.each do |case_expr|
+                find_declaration_references(case_expr.condition).each do |ref_name|
+                  current_scope = scopes[ref_name] || []
+                  if required_scope.length > current_scope.length
+                    puts "Propagating scope #{required_scope} to trait dependency #{ref_name}" if ENV["DEBUG_SCOPE_RESOLUTION"]
+                    scopes[ref_name] = required_scope
+                    update_reduction_scope_if_needed(ref_name, required_scope, declarations, input_metadata)
+                  end
+                end
+              end
+            end
+          end
+
+          def find_declaration_references(expr)
+            refs = []
+            case expr
+            when Kumi::Syntax::DeclarationReference
+              refs << expr.name
+            when Kumi::Syntax::CallExpression
+              expr.args.each { |arg| refs.concat(find_declaration_references(arg)) }
+            when Kumi::Syntax::ArrayExpression
+              expr.elements.each { |elem| refs.concat(find_declaration_references(elem)) }
+            end
+            refs
+          end
+
+          def update_reduction_scope_if_needed(decl_name, required_scope, declarations, input_metadata)
+            decl = declarations[decl_name]
+            return unless decl
+            puts "Checking if #{decl_name} needs reduction scope update for #{required_scope}" if ENV["DEBUG_SCOPE_RESOLUTION"]
+          end
+
+          def debug_output(name, decl)
+            puts "\n=== Resolving scope for #{name} ==="
+            puts "Declaration: #{decl.inspect}"
+          end
+
+          def debug_result(target_scope, result_kind)
+            puts "Target scope: #{target_scope.inspect}"
+            puts "Result kind: #{result_kind.inspect}"
+          end
+
+          def build_scope_plan(target_scope)
+            Scope.new(
+              scope: target_scope,
+              lifts: [], # Will be computed during IR lowering per call-site
+              join_hint: nil, # Will be set to :zip when multiple vectorized args exist
+              arg_shapes: {} # Optional: filled during lowering
+            )
+          end
+
+          def determine_result_kind(name, target_scope, broadcasts)
+            return :scalar if broadcasts.dig(:reduction_operations, name)
+            return :scalar if target_scope.empty?
+
+            { array: :dense }
+          end
+
+          # Derive scope from vectorization metadata or from deepest input path
+          def infer_target_scope(name, decl, broadcasts, input_metadata)
+            # First check vectorized operations
+            vec = broadcasts.dig(:vectorized_operations, name)
+            if vec
+              puts "Vectorization info: #{vec.inspect}" if ENV["DEBUG_SCOPE_RESOLUTION"]
+
+              case vec[:source]
+              when :nested_array_access, :array_field_access
+                path = vec[:path] || []
+                return dims_from_path(path, input_metadata)
+              when :cascade_with_vectorized_conditions_or_results,
+                   :cascade_condition_with_vectorized_trait
+                # Fallback: derive from first input path seen in expression
+                path = find_first_input_path(decl.expression) || []
+                return dims_from_path(path, input_metadata)
+              else
+                return []
+              end
+            end
+
+            # Check if this is a reduction operation that should preserve some scope
+            red = broadcasts.dig(:reduction_operations, name)
+            if red
+              puts "Reduction info: #{red.inspect}" if ENV["DEBUG_SCOPE_RESOLUTION"]
+              
+              # Infer the natural scope for this reduction
+              # For expressions like fn(:any?, input.players.score_matrices.session.points > 1000)
+              # we want to reduce over session dimension but preserve the players dimension
+              scope = infer_reduction_target_scope(decl.expression, input_metadata)
+              puts "Inferred reduction target scope: #{scope.inspect}" if ENV["DEBUG_SCOPE_RESOLUTION"]
+              return scope
+            end
+
+            return []
+          end
+
+          def infer_reduction_target_scope(expr, input_metadata)
+            # For reduction expressions, we need to analyze the argument to the reducer
+            # and determine which dimensions should be preserved vs reduced
+            case expr
+            when Kumi::Syntax::CallExpression
+              if reducer_function?(expr.fn_name)
+                # Find the argument being reduced
+                arg = expr.args.first
+                if arg
+                  # Get the full scope from the argument
+                  full_scope = infer_scope_from_argument(arg, input_metadata)
+                  
+                  # For array reductions, we typically want to preserve
+                  # the outermost dimension (e.g., keep :players, reduce :score_matrices/:session)
+                  if full_scope.length > 1
+                    return full_scope[0..0]  # Keep only the first dimension
+                  end
+                end
+              else
+                # Recursively check if any argument contains a reducer
+                # This handles cases like (fn(:sum, ...) >= 3500)
+                expr.args.each do |arg|
+                  nested_scope = infer_reduction_target_scope(arg, input_metadata)
+                  return nested_scope if !nested_scope.empty?
+                end
+              end
+            end
+            []
+          end
+
+          def reducer_function?(fn_name)
+            entry = Kumi::Registry.entry(fn_name)
+            entry&.reducer == true
+          end
+
+          def infer_scope_from_argument(arg, input_metadata)
+            case arg
+            when Kumi::Syntax::InputElementReference
+              dims_from_path(arg.path, input_metadata)
+            when Kumi::Syntax::InputReference
+              dims_from_path([arg.name], input_metadata)
+            when Kumi::Syntax::CallExpression
+              # For expressions like (input.players.score_matrices.session.points > 1000),
+              # we need to find the deepest input path
+              deepest_path = find_deepest_input_path(arg)
+              deepest_path ? dims_from_path(deepest_path, input_metadata) : []
+            else
+              []
+            end
+          end
+
+          def find_deepest_input_path(expr)
+            paths = collect_input_paths(expr)
+            paths.max_by(&:length)
+          end
+
+          def collect_input_paths(expr)
+            paths = []
+            case expr
+            when Kumi::Syntax::InputElementReference
+              paths << expr.path
+            when Kumi::Syntax::InputReference
+              paths << [expr.name]
+            when Kumi::Syntax::CallExpression
+              expr.args.each { |arg| paths.concat(collect_input_paths(arg)) }
+            when Kumi::Syntax::ArrayExpression
+              expr.elements.each { |elem| paths.concat(collect_input_paths(elem)) }
+            end
+            paths
+          end
+
+          def find_first_input_path(expr)
+            return nil unless expr
+
+            # Handle InputElementReference directly
+            return expr.path if expr.is_a?(Kumi::Syntax::InputElementReference)
+
+            # Handle InputReference (convert to path array)
+            return [expr.name] if expr.is_a?(Kumi::Syntax::InputReference)
+
+            # Recursively search in CallExpression arguments
+            if expr.is_a?(Kumi::Syntax::CallExpression) && expr.args
+              expr.args.each do |arg|
+                path = find_first_input_path(arg)
+                return path if path
+              end
+            end
+
+            # Search in CascadeExpression cases
+            if expr.is_a?(Kumi::Syntax::CascadeExpression) && expr.cases
+              expr.cases.each do |case_item|
+                path = find_first_input_path(case_item.condition)
+                return path if path
+
+                path = find_first_input_path(case_item.result)
+                return path if path
+              end
+            end
+
+            # Search in expression field if present
+            return find_first_input_path(expr.expression) if expr.respond_to?(:expression)
+
+            nil
+          end
+
+          # Map an input path like [:regions, :offices, :salary] to container dims [:regions, :offices]
+          def dims_from_path(path, input_metadata)
+            dims = []
+            meta = input_metadata
+
+            path.each do |seg|
+              field = meta[seg] || meta[seg.to_sym] || meta[seg.to_s]
+              break unless field
+
+              dims << seg.to_sym if field[:type] == :array
+
+              meta = field[:children] || {}
+            end
+
+            dims
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kumi/core/analyzer/passes/semantic_constraint_validator.rb

@@ -103,13 +103,14 @@ module Kumi
 
             # Validate that access_mode is consistent with children structure
             if input_decl.access_mode == :element
+
               # Element mode arrays can only have exactly one direct child
               if input_decl.children.size > 1
                 error_msg = "array with access_mode :element can only have one direct child element, " \
                             "but found #{input_decl.children.size} children"
                 report_error(errors, error_msg, location: input_decl.loc, type: :semantic)
               end
-            elsif input_decl.access_mode == :object
+            elsif input_decl.access_mode == :field
               # Object mode allows multiple children
             end
 

data/lib/kumi/core/analyzer/passes/toposorter.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "pry"
 module Kumi
   module Core
     module Analyzer
@@ -34,7 +35,6 @@ module Kumi
                 return if safe_conditional_cycle?(cycle_path, graph, cascades)
 
                 # Allow this cycle - it's safe due to cascade mutual exclusion
-
                 report_unexpected_cycle(temp_marks, node, errors)
 
                 return
@@ -42,7 +42,13 @@ module Kumi
 
               temp_marks << node
               current_path = path + [node]
-              Array(graph[node]).each { |edge| visit_node.call(edge.to, current_path) }
+              # Only follow edges to other declarations, not to input fields
+              # This prevents false cycles when a declaration has the same name as an input
+              Array(graph[node]).each do |edge|
+                next if edge.type == :key # Skip input field dependencies
+
+                visit_node.call(edge.to, current_path)
+              end
               temp_marks.delete(node)
               perm_marks << node
 
@@ -100,7 +106,7 @@ module Kumi
           def find_declaration_by_name(name)
             return nil unless schema
 
-            schema.attributes.find { |attr| attr.name == name } ||
+            schema.values.find { |attr| attr.name == name } ||
               schema.traits.find { |trait| trait.name == name }
           end
         end

data/lib/kumi/core/analyzer/passes/type_checker.rb

@@ -5,7 +5,7 @@ module Kumi
     module Analyzer
       module Passes
         # RESPONSIBILITY: Validate function call arity and argument types against FunctionRegistry
-        # DEPENDENCIES: :inferred_types from TypeInferencer
+        # DEPENDENCIES: :inferred_types from TypeInferencerPass
         # PRODUCES: :functions_required - Set of function names used in the schema
         # INTERFACE: new(schema, state).run(errors)
         class TypeChecker < VisitorPass
@@ -113,7 +113,7 @@ module Kumi
 
           def get_declared_field_type(field_name)
             # Get explicitly declared type from input metadata
-            input_meta = get_state(:inputs, required: false) || {}
+            input_meta = get_state(:input_metadata, required: false) || {}
             field_meta = input_meta[field_name]
             field_meta&.dig(:type) || Kumi::Core::Types::ANY
           end
@@ -130,7 +130,7 @@ module Kumi
               "`#{expr.value}` of type #{type} (literal value)"
 
             when Kumi::Syntax::InputReference
-              input_meta = get_state(:inputs, required: false) || {}
+              input_meta = get_state(:input_metadata, required: false) || {}
               field_meta = input_meta[expr.name]
 
               if field_meta&.dig(:type)

data/lib/kumi/core/analyzer/passes/type_consistency_checker.rb

@@ -5,12 +5,12 @@ module Kumi
     module Analyzer
       module Passes
         # RESPONSIBILITY: Validate consistency between declared and inferred types
-        # DEPENDENCIES: :inputs from InputCollector, :inferred_types from TypeInferencer
+        # DEPENDENCIES: :input_metadata from InputCollector, :inferred_types from TypeInferencerPass
         # PRODUCES: None (validation only)
         # INTERFACE: new(schema, state).run(errors)
         class TypeConsistencyChecker < PassBase
           def run(errors)
-            input_meta = get_state(:inputs, required: false) || {}
+            input_meta = get_state(:input_metadata, required: false) || {}
 
             # First, validate that all declared types are valid
             validate_declared_types(input_meta, errors)

data/lib/kumi/core/analyzer/passes/{type_inferencer.rb → type_inferencer_pass.rb}

@@ -8,7 +8,7 @@ module Kumi
         # DEPENDENCIES: Toposorter (needs evaluation_order), DeclarationValidator (needs declarations)
         # PRODUCES: inferred_types hash mapping declaration names to inferred types
         # INTERFACE: new(schema, state).run(errors)
-        class TypeInferencer < PassBase
+        class TypeInferencerPass < PassBase
           def run(errors)
             types = {}
             topo_order = get_state(:evaluation_order)
@@ -49,7 +49,7 @@ module Kumi
               Types.infer_from_value(expr.value)
             when InputReference
               # Look up type from field metadata
-              input_meta = get_state(:inputs, required: false) || {}
+              input_meta = get_state(:input_metadata, required: false) || {}
               meta = input_meta[expr.name]
               meta&.dig(:type) || :any
             when DeclarationReference
@@ -156,7 +156,7 @@ module Kumi
             case expr
             when InputElementReference
               # Get the field type from metadata
-              input_meta = get_state(:inputs, required: false) || {}
+              input_meta = get_state(:input_metadata, required: false) || {}
               array_name = expr.path.first
               field_name = expr.path[1]
 
@@ -198,7 +198,7 @@ module Kumi
 
           def infer_element_reference_type(expr)
             # Get array field metadata
-            input_meta = get_state(:inputs, required: false) || {}
+            input_meta = get_state(:input_metadata, required: false) || {}
 
             return :any unless expr.path.size >= 2
 

data/lib/kumi/core/analyzer/passes/unsat_detector.rb

@@ -5,7 +5,7 @@ module Kumi
     module Analyzer
       module Passes
         # RESPONSIBILITY: Detect unsatisfiable constraints and analyze cascade mutual exclusion
-        # DEPENDENCIES: :declarations from NameIndexer, :inputs from InputCollector
+        # DEPENDENCIES: :declarations from NameIndexer, :input_metadata from InputCollector
         # PRODUCES: :cascades - Hash of cascade mutual exclusion analysis results
         # INTERFACE: new(schema, state).run(errors)
         class UnsatDetector < VisitorPass
@@ -16,7 +16,7 @@ module Kumi
 
           def run(errors)
             definitions = get_state(:declarations)
-            @input_meta = get_state(:inputs) || {}
+            @input_meta = get_state(:input_metadata) || {}
             @definitions = definitions
             @evaluator = ConstantEvaluator.new(definitions)
 

data/lib/kumi/core/analyzer/plans.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Core
+    module Analyzer
+      # Typed plan structures for HIR (High-level Intermediate Representation)
+      # These plans are produced by analyzer passes and consumed by LowerToIRPass
+      # to generate LIR (Low-level IR) operations.
+      module Plans
+        # Scope plan: defines the dimensional execution context for a declaration
+        Scope = Struct.new(:scope, :lifts, :join_hint, :arg_shapes, keyword_init: true) do
+          def initialize(scope: [], lifts: [], join_hint: nil, arg_shapes: {})
+            super
+            freeze
+          end
+
+          def depth
+            scope.size
+          end
+
+          def scalar?
+            scope.empty?
+          end
+        end
+
+        # Join plan: defines how to align multiple arguments at a target scope
+        Join = Struct.new(:policy, :target_scope, keyword_init: true) do
+          def initialize(policy: :zip, target_scope: [])
+            super
+            freeze
+          end
+        end
+
+        # Reduce plan: defines how to reduce dimensions in array operations
+        Reduce = Struct.new(:function, :axis, :source_scope, :result_scope, :flatten_args, keyword_init: true) do
+          def initialize(function:, axis: [], source_scope: [], result_scope: [], flatten_args: [])
+            super
+            freeze
+          end
+
+          def total_reduction?
+            axis == :all || result_scope.empty?
+          end
+
+          def partial_reduction?
+            !total_reduction?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kumi/core/analyzer/structs/access_plan.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Core
+    module Analyzer
+      # One plan for a specific path and mode (path:mode)
+      AccessPlan = Struct.new(:path, :containers, :leaf, :scope, :depth, :mode,
+                              :on_missing, :key_policy, :operations, keyword_init: true) do
+        def initialize(path:, containers:, leaf:, scope:, depth:, mode:, on_missing:, key_policy:, operations:)
+          super
+          freeze
+        end
+
+        def accessor_key = "#{path}:#{mode}"
+        def ndims        = depth
+        def scalar?      = depth.zero?
+      end
+    end
+  end
+end

data/lib/kumi/core/analyzer/structs/input_meta.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Core
+    module Analyzer
+      module Structs
+        # Represents metadata for a single input field produced by InputCollector
+        InputMeta = Struct.new(
+          :type,
+          :domain,
+          :container,
+          :access_mode,
+          :enter_via,
+          :consume_alias,
+          :children,
+          keyword_init: true
+        ) do
+          def deep_freeze!
+            if children
+              children.each_value(&:deep_freeze!)
+              children.freeze
+            end
+            freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kumi/core/compiler/access_builder.rb

@@ -0,0 +1,36 @@
+module Kumi
+  module Core
+    module Compiler
+      class AccessBuilder
+        def self.build(plans)
+          accessors = {}
+          plans.each_value do |variants|
+            variants.each do |plan|
+              key = plan.respond_to?(:accessor_key) ? plan.accessor_key : "#{plan.path}:#{mode}"
+              accessors[key] = build_proc_for(
+                mode: plan.mode,
+                path_key: plan.path,
+                missing: (plan.on_missing || :error).to_sym,
+                key_policy: (plan.key_policy || :indifferent).to_sym,
+                operations: plan.operations
+              )
+            end
+          end
+          accessors.freeze
+        end
+
+        def self.build_proc_for(mode:, path_key:, missing:, key_policy:, operations:)
+          case mode
+          when :read        then Accessors::ReadAccessor.build(operations, path_key, missing, key_policy)
+          when :materialize then Accessors::MaterializeAccessor.build(operations, path_key, missing, key_policy)
+          when :ravel       then Accessors::RavelAccessor.build(operations, path_key, missing, key_policy)
+          when :each_indexed then Accessors::EachIndexedAccessor.build(operations, path_key, missing, key_policy, true)
+          when :each then Accessors::EachAccessor.build(operations, path_key, missing, key_policy)
+          else
+            raise "Unknown accessor mode: #{mode.inspect}"
+          end
+        end
+      end
+    end
+  end
+end