kumi 0.0.18 → 0.0.20

data/lib/kumi/core/analyzer/passes/normalize_to_nast_pass.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Core
+    module Analyzer
+      module Passes
+        class NormalizeToNASTPass < PassBase
+          NAST = Kumi::Core::NAST
+          SELECT_ID = Kumi::RegistryV2::SELECT_ID
+
+          def run(errors)
+            decls = get_state(:declarations)
+            order = get_state(:evaluation_order)
+            @index_table = get_state(:index_table)
+            @registry = get_state(:registry)
+
+            nast_decls = {}
+            order.each do |name|
+              ast = decls[name] or next
+              body = normalize_expr(ast.expression, errors)
+              nast_decls[name] = NAST::Declaration.new(name: name, body: body, loc: ast.loc, meta: { kind: ast.kind })
+            end
+
+            nast = NAST::Module.new(decls: nast_decls)
+            debug "NAST decl keys: #{nast.decls.keys.inspect}"
+            state.with(:nast_module, nast)
+          end
+
+          private
+
+          def normalize_expr(node, errors)
+            case node
+            when Kumi::Syntax::Literal
+              NAST::Const.new(value: node.value, loc: node.loc)
+
+            when Kumi::Syntax::InputReference
+              NAST::InputRef.new(path: [node.name], loc: node.loc)
+
+            when Kumi::Syntax::InputElementReference
+              NAST::InputRef.new(path: node.path, loc: node.loc)
+
+            when Kumi::Syntax::DeclarationReference
+              NAST::Ref.new(name: node.name, loc: node.loc)
+
+            when Kumi::Syntax::CallExpression
+              # Special handling section - very clear what's happening
+              case node.fn_name
+              when :cascade_and
+                # Desugar cascade_and into chained binary core.and operations
+                normalize_cascade_and(node.args, errors, node.loc)
+              when :index
+                normalize_index_ref(node, errors, node.loc)
+              else
+                normalize_call_expression(node, errors)
+              end
+            when Kumi::Syntax::CascadeExpression
+              normalize_cascade(node, errors)
+            when Kumi::Syntax::ArrayExpression
+              args = node.elements.map { |a| normalize_expr(a, errors) }
+              NAST::Tuple.new(args: args, loc: node.loc)
+            when Kumi::Syntax::HashExpression
+              pairs = node.pairs.map do |k, v|
+                NAST::Pair.new(
+                  key: k.value,
+                  value: normalize_expr(v, errors)
+                )
+              end
+
+              NAST::Hash.new(pairs:, loc: node.loc)
+            else
+              add_error(errors, node&.loc, "Unsupported AST node: #{node&.class}")
+              NAST::Const.new(value: nil, loc: node&.loc)
+            end
+          end
+
+          def normalize_call_expression(node, errors)
+            begin
+              fn_name = FnAliases::MAP[node.fn_name] || node.fn_name
+              func = @registry.function(fn_name)
+            rescue StandardError
+              # puts "MISSING_FUNCTION: #{node.fn_name.inspect}"
+              raise
+            end
+
+            if func.expand
+              # 1. Normalize the arguments FIRST.
+              normalized_args = node.args.map { |arg| normalize_expr(arg, errors) }
+
+              # 2. Pass the normalized NAST arguments to the expander.
+              # The expander will return a fully formed NAST node.
+              MacroExpander.expand(func, normalized_args, node.loc, errors)
+            else
+              # Regular, non-expandable function call.
+              args = node.args.map { |a| normalize_expr(a, errors) }
+              NAST::Call.new(fn: func.id.to_sym, args: args, opts: node.opts, loc: node.loc)
+            end
+          end
+
+          def normalize_cascade(cas, errors)
+            # Find the base case (condition = true)
+            base = cas.cases.find do |c|
+              c.condition.is_a?(Kumi::Syntax::Literal) && c.condition.value == true
+            end
+            default_expr = base ? base.result : Kumi::Syntax::Literal.new(nil, loc: cas.loc)
+            branches = cas.cases.reject { |c| c.equal?(base) }
+
+            # Build nested if expression (reverse order)
+            else_n = normalize_expr(default_expr, errors)
+            branches.reverse_each do |br|
+              cond = normalize_expr(br.condition, errors)
+              val = normalize_expr(br.result, errors)
+              else_n = NAST::Call.new(fn: SELECT_ID, args: [cond, val, else_n], loc: br.condition.loc)
+            end
+            else_n
+          end
+
+          def normalize_cascade_and(args, errors, loc)
+            # Desugar cascade_and into chained binary core.and operations
+            case args.size
+            when 0
+              # Edge case: no arguments - should probably be an error
+              add_error(errors, loc, "cascade_and requires at least one argument")
+              NAST::Const.new(value: true, loc: loc)
+            when 1
+              # Single argument: no 'and' needed, just normalize the argument
+              normalize_expr(args[0], errors)
+            else
+              # Multiple arguments: create right-associative binary tree
+              # cascade_and(a, b, c) -> and(a, and(b, c))
+              normalized_args = args.map { |arg| normalize_expr(arg, errors) }
+              build_right_associative_and(normalized_args, loc)
+            end
+          end
+
+          def normalize_index_ref(n, errors, loc)
+            idx_name_n = normalize_expr(n.args.first, errors)
+
+            debug "normalize_index_ref: idx_name_n=#{idx_name_n.inspect}, is Const? #{idx_name_n.is_a?(NAST::Const)}, value=#{idx_name_n.respond_to?(:value) ? idx_name_n.value.inspect : 'N/A'}"
+
+            add_error(errors, loc, "index() needs a symbol") unless idx_name_n.is_a?(NAST::Const) && idx_name_n.value.is_a?(Symbol)
+            idx_name = idx_name_n.value
+            idx_meta = @index_table[idx_name]
+            NAST::IndexRef.new(name: idx_name, input_fqn: idx_meta[:fqn])
+          end
+
+          def build_right_associative_and(normalized_args, loc)
+            # Build: and(first, and(second, and(third, ...)))
+            normalized_args.reverse.reduce do |right, left|
+              NAST::Call.new(fn: :"core.and", args: [left, right], loc: loc)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kumi/core/analyzer/passes/output_schema_pass.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Core
+    module Analyzer
+      module Passes
+        class OutputSchemaPass < PassBase
+          def run(errors)
+            snast_module = get_state(:snast_module)
+            return state unless snast_module
+
+            output_schema = build_output_schema(snast_module)
+            state.with(:output_schema, output_schema.freeze)
+          end
+
+          private
+
+          def build_output_schema(snast_module)
+            snast_module.decls.transform_values do |decl|
+              build_output_field(decl)
+            end
+          end
+
+          def build_output_field(decl)
+            meta = decl.meta
+            stamp = meta[:stamp] || {}
+
+            {
+              kind: meta[:kind],
+              type: stamp[:dtype],
+              axes: stamp[:axes] || []
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kumi/core/analyzer/passes/pass_base.rb

@@ -23,15 +23,29 @@ module Kumi
             raise NotImplementedError, "#{self.class.name} must implement #run"
           end
 
+          # Debug helpers - automatic pattern based on pass class name
+          # InputIndexTablePass -> DEBUG_INPUT_INDEX_TABLE=1
+          # ScopeResolutionPass -> DEBUG_SCOPE_RESOLUTION=1
+          def debug_enabled?
+            class_name = self.class.name.split("::").last
+            env_name = "DEBUG_#{to_underscore(class_name.gsub(/Pass$/, '')).upcase}"
+            ENV[env_name] == "1"
+          end
+
+          def debug(message)
+            class_name = self.class.name.split("::").last.gsub(/Pass$/, "")
+            puts "[#{class_name}] #{message}" if debug_enabled?
+          end
+
           protected
 
           attr_reader :schema, :state
 
           # Iterate over all declarations (values and traits) in the schema
           # @yield [Syntax::Attribute|Syntax::Trait] Each declaration
-          def each_decl(&block)
-            schema.values.each(&block)
-            schema.traits.each(&block)
+          def each_decl(&)
+            schema.values.each(&)
+            schema.traits.each(&)
           end
 
           # Get state value - compatible with old interface
@@ -41,10 +55,22 @@ module Kumi
             state[key]
           end
 
-          # Add error to the error list
+          def error(msg, loc: nil, node: nil)
+            location = loc || node&.loc
+            add_error(@errors, location, msg)
+          end
+
           def add_error(errors, location, message)
             errors << ErrorReporter.create_error(message, location: location, type: :semantic)
           end
+
+          private
+
+          def to_underscore(str)
+            str.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+               .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+               .downcase
+          end
         end
       end
     end

data/lib/kumi/core/analyzer/passes/precompute_access_paths_pass.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Core
+    module Analyzer
+      module Passes
+        class PrecomputeAccessPathsPass < PassBase
+          def run(_errors)
+            plans = get_state(:input_table, required: true)
+            out   = {}
+            plans.each { |p| out[p.path_fqn.to_s] = build_for_plan(p).freeze }
+            state.with(:precomputed_plan_by_fqn, out.freeze)
+          end
+
+          private
+
+          def build_for_plan(plan)
+            # normalize to symbol keys & kinds
+            steps = Array(plan.navigation_steps).map { |h| h.to_h.transform_keys!(&:to_sym) }.freeze
+
+            loop_ixs = steps.each_index.select { |i| steps[i][:kind].to_s == "array_loop" }.freeze
+
+            head_path_by_loop = {}
+            loop_ixs.each { |li| head_path_by_loop[li] = compress_head_path(steps, li).freeze }
+
+            between_loops = {}
+            loop_ixs.each_with_index do |li_from, i|
+              ((i + 1)...loop_ixs.length).each do |j|
+                li_to = loop_ixs[j]
+                between_loops[[li_from, li_to]] = compress_between(steps, li_from, li_to).freeze
+              end
+            end
+
+            last_loop_li = loop_ixs.last
+            tail_range   = last_loop_li ? ((last_loop_li + 1)..(steps.length - 1)) : (0..(steps.length - 1))
+            tail_steps   = tail_range.to_a.empty? ? [] : steps[tail_range]
+            tail_keys_after_last_loop = tail_steps.select { |s| s[:kind].to_s == "property_access" }
+                                                  .map { |s| s[:key].to_sym }.freeze
+            element_terminal = tail_steps.any? { |s| s[:kind].to_s == "element_access" }
+
+            {
+              steps: steps,
+              loop_ixs: loop_ixs,
+              loop_axes: loop_ixs.map { |i| steps[i][:axis].to_sym }.freeze,
+              axis_to_loop: loop_ixs.map { |i| [steps[i][:axis].to_sym, i] }.to_h.freeze,
+              head_path_by_loop: head_path_by_loop.freeze,
+              between_loops: between_loops.freeze,
+              last_loop_li: last_loop_li,
+              tail_keys_after_last_loop: tail_keys_after_last_loop,
+              element_terminal: element_terminal
+            }
+          end
+
+          # steps[0..li] → [[:input, k] or [:field, k], ...] to reach the array for loop li
+          # NOTE: now :array_loop has NO :key; the key came from the prior property_access.
+          def compress_head_path(steps, li)
+            acc = []
+            steps[0..li].each do |s|
+              case s[:kind]
+              when :property_access
+                acc << (acc.empty? ? [:input, s[:key].to_sym] : [:field, s[:key].to_sym])
+              when :element_access
+                # no hop
+              when :array_loop
+                break
+              end
+            end
+            acc
+          end
+
+          # keys from element(li_from) → collection(li_to)
+          def compress_between(steps, li_from, li_to)
+            raise ArgumentError, "order" unless li_from < li_to
+
+            keys = []
+            ((li_from + 1)..li_to).each do |i|
+              s = steps[i]
+              case s[:kind]
+              when :property_access
+                keys << s[:key].to_sym
+              when :element_access
+                # no hop
+              when :array_loop
+                break
+              end
+            end
+            keys
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kumi/core/analyzer/passes/scope_resolution_pass.rb

@@ -18,7 +18,7 @@ module Kumi
             input_metadata = get_state(:input_metadata, required: true)
             broadcasts = get_state(:broadcasts) || {}
             dependencies = get_state(:dependencies) || {}
-            
+
             puts "Available dependencies: #{dependencies.keys.inspect}" if ENV["DEBUG_SCOPE_RESOLUTION"]
 
             scope_plans = {}
@@ -82,55 +82,57 @@ module Kumi
               end
             end
 
-            if anchor_scope && !anchor_scope.empty?
-              declaration_refs.each do |ref_name|
-                current_scope = scopes[ref_name] || []
-                if anchor_scope.length > current_scope.length
-                  puts "Propagating scope #{anchor_scope} to #{ref_name} (was #{current_scope})" if ENV["DEBUG_SCOPE_RESOLUTION"]
-                  scopes[ref_name] = anchor_scope
-                  propagate_to_dependencies(ref_name, anchor_scope, scopes, declarations, input_metadata)
-                end
-              end
+            return unless anchor_scope && !anchor_scope.empty?
+
+            declaration_refs.each do |ref_name|
+              current_scope = scopes[ref_name] || []
+              next unless anchor_scope.length > current_scope.length
+
+              puts "Propagating scope #{anchor_scope} to #{ref_name} (was #{current_scope})" if ENV["DEBUG_SCOPE_RESOLUTION"]
+              scopes[ref_name] = anchor_scope
+              propagate_to_dependencies(ref_name, anchor_scope, scopes, declarations, input_metadata)
             end
           end
 
           def propagate_from_cascade_expression(name, cascade_expr, scopes, declarations, input_metadata)
             puts "Analyzing cascade expression in #{name}" if ENV["DEBUG_SCOPE_RESOLUTION"]
-            
+
             # Cascade should propagate its own scope to condition dependencies
             cascade_scope = scopes[name] || []
             return if cascade_scope.empty?
-            
+
             puts "Propagating cascade scope #{cascade_scope} to condition dependencies" if ENV["DEBUG_SCOPE_RESOLUTION"]
-            
+
             cascade_expr.cases.each do |case_expr|
               find_declaration_references(case_expr.condition).each do |ref_name|
                 current_scope = scopes[ref_name] || []
-                if cascade_scope.length > current_scope.length
-                  puts "Propagating scope #{cascade_scope} to cascade condition #{ref_name} (was #{current_scope})" if ENV["DEBUG_SCOPE_RESOLUTION"]
-                  scopes[ref_name] = cascade_scope
-                  propagate_to_dependencies(ref_name, cascade_scope, scopes, declarations, input_metadata)
+                next unless cascade_scope.length > current_scope.length
+
+                if ENV["DEBUG_SCOPE_RESOLUTION"]
+                  puts "Propagating scope #{cascade_scope} to cascade condition #{ref_name} (was #{current_scope})"
                 end
+                scopes[ref_name] = cascade_scope
+                propagate_to_dependencies(ref_name, cascade_scope, scopes, declarations, input_metadata)
               end
             end
           end
 
           def propagate_to_dependencies(decl_name, required_scope, scopes, declarations, input_metadata)
             return unless declarations[decl_name]
-            
+
             decl = declarations[decl_name]
             puts "Propagating #{required_scope} into dependencies of #{decl_name}" if ENV["DEBUG_SCOPE_RESOLUTION"]
-            
+
             case decl.expression
             when Kumi::Syntax::CascadeExpression
               decl.expression.cases.each do |case_expr|
                 find_declaration_references(case_expr.condition).each do |ref_name|
                   current_scope = scopes[ref_name] || []
-                  if required_scope.length > current_scope.length
-                    puts "Propagating scope #{required_scope} to trait dependency #{ref_name}" if ENV["DEBUG_SCOPE_RESOLUTION"]
-                    scopes[ref_name] = required_scope
-                    update_reduction_scope_if_needed(ref_name, required_scope, declarations, input_metadata)
-                  end
+                  next unless required_scope.length > current_scope.length
+
+                  puts "Propagating scope #{required_scope} to trait dependency #{ref_name}" if ENV["DEBUG_SCOPE_RESOLUTION"]
+                  scopes[ref_name] = required_scope
+                  update_reduction_scope_if_needed(ref_name, required_scope, declarations, input_metadata)
                 end
               end
             end
@@ -149,9 +151,10 @@ module Kumi
             refs
           end
 
-          def update_reduction_scope_if_needed(decl_name, required_scope, declarations, input_metadata)
+          def update_reduction_scope_if_needed(decl_name, required_scope, declarations, _input_metadata)
             decl = declarations[decl_name]
             return unless decl
+
             puts "Checking if #{decl_name} needs reduction scope update for #{required_scope}" if ENV["DEBUG_SCOPE_RESOLUTION"]
           end
 
@@ -206,7 +209,7 @@ module Kumi
             red = broadcasts.dig(:reduction_operations, name)
             if red
               puts "Reduction info: #{red.inspect}" if ENV["DEBUG_SCOPE_RESOLUTION"]
-              
+
               # Infer the natural scope for this reduction
               # For expressions like fn(:any?, input.players.score_matrices.session.points > 1000)
               # we want to reduce over session dimension but preserve the players dimension
@@ -215,7 +218,7 @@ module Kumi
               return scope
             end
 
-            return []
+            []
           end
 
           def infer_reduction_target_scope(expr, input_metadata)
@@ -229,11 +232,11 @@ module Kumi
                 if arg
                   # Get the full scope from the argument
                   full_scope = infer_scope_from_argument(arg, input_metadata)
-                  
+
                   # For array reductions, we typically want to preserve
                   # the outermost dimension (e.g., keep :players, reduce :score_matrices/:session)
                   if full_scope.length > 1
-                    return full_scope[0..0]  # Keep only the first dimension
+                    return full_scope[0..0] # Keep only the first dimension
                   end
                 end
               else
@@ -241,7 +244,7 @@ module Kumi
                 # This handles cases like (fn(:sum, ...) >= 3500)
                 expr.args.each do |arg|
                   nested_scope = infer_reduction_target_scope(arg, input_metadata)
-                  return nested_scope if !nested_scope.empty?
+                  return nested_scope unless nested_scope.empty?
                 end
               end
             end

data/lib/kumi/core/analyzer/passes/semantic_constraint_validator.rb

@@ -17,6 +17,7 @@ module Kumi
         # 4. Expression types are valid for their context
         class SemanticConstraintValidator < VisitorPass
           def run(errors)
+            @registry = state[:registry]
             # Visit value and trait declarations
             each_decl do |decl|
               visit(decl) { |node| validate_semantic_constraints(node, decl, errors) }
@@ -85,10 +86,11 @@ module Kumi
           def validate_function_call(call_expr, errors)
             fn_name = call_expr.fn_name
 
-            # Skip validation if Kumi::Registry.is being mocked for testing
-            return if function_registry_mocked?
+            skip = [:cascade_and] # TODO: - hack
+            return if skip.include? fn_name
 
-            return if Kumi::Registry.supported?(fn_name)
+            # binding.pry
+            return if @registry.resolve_function(fn_name)
 
             report_error(
               errors,
@@ -101,19 +103,6 @@ module Kumi
           def validate_input_declaration(input_decl, errors)
             return unless input_decl.type == :array && input_decl.children.any?
 
-            # Validate that access_mode is consistent with children structure
-            if input_decl.access_mode == :element
-
-              # Element mode arrays can only have exactly one direct child
-              if input_decl.children.size > 1
-                error_msg = "array with access_mode :element can only have one direct child element, " \
-                            "but found #{input_decl.children.size} children"
-                report_error(errors, error_msg, location: input_decl.loc, type: :semantic)
-              end
-            elsif input_decl.access_mode == :field
-              # Object mode allows multiple children
-            end
-
             # Recursively validate children
             input_decl.children.each { |child| validate_input_declaration(child, errors) }
           end
@@ -122,16 +111,6 @@ module Kumi
             # Allow boolean composition functions that operate on trait collections
             %i[all? any? none?].include?(call_expr.fn_name)
           end
-
-          def function_registry_mocked?
-            # Check if Kumi::Registry.is being mocked (for tests)
-
-            # Try to access a method that doesn't exist in the real registry
-            # If it's mocked, this won't raise an error
-            Kumi::Registry.respond_to?(:confirm_support!)
-          rescue StandardError
-            false
-          end
         end
       end
     end