kumi 0.0.10 → 0.0.11

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kumi
 version: !ruby/object:Gem::Version
-  version: 0.0.10
+  version: 0.0.11
 platform: ruby
 authors:
 - André Muta
@@ -31,6 +31,7 @@ extra_rdoc_files: []
 files:
 - ".rspec"
 - ".rubocop.yml"
+- CHANGELOG.md
 - CLAUDE.md
 - LICENSE.txt
 - README.md
@@ -39,6 +40,7 @@ files:
 - docs/DSL.md
 - docs/FUNCTIONS.md
 - docs/SYNTAX.md
+- docs/VECTOR_SEMANTICS.md
 - docs/compiler_design_principles.md
 - docs/development/README.md
 - docs/development/error-reporting.md
@@ -74,23 +76,31 @@ files:
 - lib/kumi/core/analyzer/passes/broadcast_detector.rb
 - lib/kumi/core/analyzer/passes/declaration_validator.rb
 - lib/kumi/core/analyzer/passes/dependency_resolver.rb
+- lib/kumi/core/analyzer/passes/input_access_planner_pass.rb
 - lib/kumi/core/analyzer/passes/input_collector.rb
+- lib/kumi/core/analyzer/passes/join_reduce_planning_pass.rb
+- lib/kumi/core/analyzer/passes/lower_to_ir_pass.rb
 - lib/kumi/core/analyzer/passes/name_indexer.rb
 - lib/kumi/core/analyzer/passes/pass_base.rb
+- lib/kumi/core/analyzer/passes/scope_resolution_pass.rb
 - lib/kumi/core/analyzer/passes/semantic_constraint_validator.rb
 - lib/kumi/core/analyzer/passes/toposorter.rb
 - lib/kumi/core/analyzer/passes/type_checker.rb
 - lib/kumi/core/analyzer/passes/type_consistency_checker.rb
-- lib/kumi/core/analyzer/passes/type_inferencer.rb
+- lib/kumi/core/analyzer/passes/type_inferencer_pass.rb
 - lib/kumi/core/analyzer/passes/unsat_detector.rb
 - lib/kumi/core/analyzer/passes/visitor_pass.rb
+- lib/kumi/core/analyzer/plans.rb
+- lib/kumi/core/analyzer/structs/access_plan.rb
+- lib/kumi/core/analyzer/structs/input_meta.rb
 - lib/kumi/core/atom_unsat_solver.rb
-- lib/kumi/core/cascade_executor_builder.rb
-- lib/kumi/core/compiled_schema.rb
-- lib/kumi/core/compiler/expression_compiler.rb
-- lib/kumi/core/compiler/function_invoker.rb
-- lib/kumi/core/compiler/path_traversal_compiler.rb
-- lib/kumi/core/compiler/reference_compiler.rb
+- lib/kumi/core/compiler/access_builder.rb
+- lib/kumi/core/compiler/access_planner.rb
+- lib/kumi/core/compiler/accessors/base.rb
+- lib/kumi/core/compiler/accessors/each_indexed_accessor.rb
+- lib/kumi/core/compiler/accessors/materialize_accessor.rb
+- lib/kumi/core/compiler/accessors/ravel_accessor.rb
+- lib/kumi/core/compiler/accessors/read_accessor.rb
 - lib/kumi/core/compiler_base.rb
 - lib/kumi/core/constraint_relationship_solver.rb
 - lib/kumi/core/domain/enum_analyzer.rb
@@ -100,7 +110,6 @@ files:
 - lib/kumi/core/error_reporter.rb
 - lib/kumi/core/error_reporting.rb
 - lib/kumi/core/errors.rb
-- lib/kumi/core/evaluation_wrapper.rb
 - lib/kumi/core/explain.rb
 - lib/kumi/core/export.rb
 - lib/kumi/core/export/deserializer.rb
@@ -122,10 +131,14 @@ files:
 - lib/kumi/core/input/type_matcher.rb
 - lib/kumi/core/input/validator.rb
 - lib/kumi/core/input/violation_creator.rb
+- lib/kumi/core/ir.rb
+- lib/kumi/core/ir/execution_engine.rb
+- lib/kumi/core/ir/execution_engine/combinators.rb
+- lib/kumi/core/ir/execution_engine/interpreter.rb
+- lib/kumi/core/ir/execution_engine/values.rb
 - lib/kumi/core/json_schema.rb
 - lib/kumi/core/json_schema/generator.rb
 - lib/kumi/core/json_schema/validator.rb
-- lib/kumi/core/nested_structure_utils.rb
 - lib/kumi/core/ruby_parser.rb
 - lib/kumi/core/ruby_parser/build_context.rb
 - lib/kumi/core/ruby_parser/declaration_reference_proxy.rb
@@ -140,7 +153,6 @@ files:
 - lib/kumi/core/ruby_parser/parser.rb
 - lib/kumi/core/ruby_parser/schema_builder.rb
 - lib/kumi/core/ruby_parser/sugar.rb
-- lib/kumi/core/schema_instance.rb
 - lib/kumi/core/types.rb
 - lib/kumi/core/types/builder.rb
 - lib/kumi/core/types/compatibility.rb
@@ -148,14 +160,13 @@ files:
 - lib/kumi/core/types/inference.rb
 - lib/kumi/core/types/normalizer.rb
 - lib/kumi/core/types/validator.rb
-- lib/kumi/core/vectorized_function_builder.rb
 - lib/kumi/errors.rb
 - lib/kumi/js.rb
-- lib/kumi/js/compiler.rb
-- lib/kumi/js/function_registry.rb
 - lib/kumi/registry.rb
+- lib/kumi/runtime/executable.rb
 - lib/kumi/schema.rb
 - lib/kumi/schema_metadata.rb
+- lib/kumi/support/ir_dump.rb
 - lib/kumi/support/s_expression_printer.rb
 - lib/kumi/syntax/array_expression.rb
 - lib/kumi/syntax/call_expression.rb
@@ -167,12 +178,12 @@ files:
 - lib/kumi/syntax/input_element_reference.rb
 - lib/kumi/syntax/input_reference.rb
 - lib/kumi/syntax/literal.rb
+- lib/kumi/syntax/location.rb
 - lib/kumi/syntax/node.rb
 - lib/kumi/syntax/root.rb
 - lib/kumi/syntax/trait_declaration.rb
 - lib/kumi/syntax/value_declaration.rb
 - lib/kumi/version.rb
-- migrate_to_core_iterative.rb
 - scripts/analyze_broadcast_methods.rb
 - scripts/analyze_cascade_methods.rb
 - scripts/check_broadcasting_coverage.rb

data/lib/kumi/core/cascade_executor_builder.rb

@@ -1,132 +0,0 @@
-# frozen_string_literal: true
-
-module Kumi
-  module Core
-    # Builds cascade execution lambdas from analysis metadata
-    class CascadeExecutorBuilder
-      include NestedStructureUtils
-
-      def self.build_executor(strategy, analysis_state)
-        new(strategy, analysis_state).build
-      end
-
-      def initialize(strategy, analysis_state)
-        @strategy = strategy
-        @analysis_state = analysis_state
-      end
-
-      def build
-        case @strategy[:mode]
-        when :hierarchical
-          build_hierarchical_executor
-        when :nested_array, :deep_nested_array
-          build_nested_array_executor
-        when :simple_array
-          build_simple_array_executor
-        else
-          build_scalar_executor
-        end
-      end
-
-      private
-
-      def build_hierarchical_executor
-        lambda do |cond_results, res_results, base_result, pairs|
-          # Find the result structure to use as template (deepest structure)
-          all_values = (res_results + cond_results + [base_result].compact).select { |v| v.is_a?(Array) }
-          result_template = all_values.max_by { |v| calculate_array_depth(v) }
-
-          return execute_scalar_cascade(cond_results, res_results, base_result, pairs) unless result_template
-
-          # Apply hierarchical cascade logic using the result structure as template
-          map_nested_structure(result_template) do |*indices|
-            result = nil
-
-            # Check conditional cases first with hierarchical broadcasting for conditions
-            pairs.each_with_index do |(_cond, _res), pair_idx|
-              cond_val = navigate_with_hierarchical_broadcasting(cond_results[pair_idx], indices, result_template)
-              next unless cond_val
-
-              res_val = navigate_nested_indices(res_results[pair_idx], indices)
-              result = res_val
-              break
-            end
-
-            # If no conditional case matched, use base case
-            result = navigate_nested_indices(base_result, indices) if result.nil? && base_result
-
-            result
-          end
-        end
-      end
-
-      def build_nested_array_executor
-        lambda do |cond_results, res_results, base_result, pairs|
-          # For nested arrays, we need to find the structure template
-          structure_template = find_structure_template(cond_results + res_results + [base_result].compact)
-          return execute_scalar_cascade(cond_results, res_results, base_result, pairs) unless structure_template
-
-          # Apply cascade logic recursively through the nested structure
-          map_nested_structure(structure_template) do |*indices|
-            result = nil
-
-            # Check conditional cases first
-            pairs.each_with_index do |(_cond, _res), pair_idx|
-              cond_val = navigate_nested_indices(cond_results[pair_idx], indices)
-              next unless cond_val
-
-              res_val = navigate_nested_indices(res_results[pair_idx], indices)
-              result = res_val
-              break
-            end
-
-            # If no conditional case matched, use base case
-            result = navigate_nested_indices(base_result, indices) if result.nil? && base_result
-
-            result
-          end
-        end
-      end
-
-      def build_simple_array_executor
-        lambda do |cond_results, res_results, base_result, pairs|
-          array_length = determine_array_length(cond_results + res_results + [base_result].compact)
-
-          (0...array_length).map do |i|
-            result = nil
-            # Check conditional cases first
-            pairs.each_with_index do |(_cond, _res), pair_idx|
-              cond_val = extract_element_at_index(cond_results[pair_idx], i)
-              next unless cond_val
-
-              res_val = extract_element_at_index(res_results[pair_idx], i)
-              result = res_val
-              break
-            end
-
-            # If no conditional case matched, use base case
-            result = extract_element_at_index(base_result, i) if result.nil? && base_result
-
-            result
-          end
-        end
-      end
-
-      def build_scalar_executor
-        lambda do |cond_results, res_results, base_result, pairs|
-          pairs.each_with_index do |(_cond, _res), pair_idx|
-            return res_results[pair_idx] if cond_results[pair_idx]
-          end
-          base_result
-        end
-      end
-
-      def execute_scalar_cascade(cond_results, res_results, base_result, pairs)
-        pairs.each_with_index do |(_cond, _res), pair_idx|
-          return res_results[pair_idx] if cond_results[pair_idx]
-        end
-        base_result
-      end
-    end
-  end
-end

data/lib/kumi/core/compiled_schema.rb

@@ -1,43 +0,0 @@
-# frozen_string_literal: true
-
-module Kumi
-  module Core
-    class CompiledSchema
-      attr_reader :bindings
-
-      def initialize(bindings)
-        @bindings = bindings.freeze
-      end
-
-      def evaluate(ctx, *key_names)
-        target_keys = key_names.empty? ? @bindings.keys : validate_keys(key_names)
-
-        target_keys.each_with_object({}) do |key, result|
-          result[key] = evaluate_binding(key, ctx)
-        end
-      end
-
-      def evaluate_binding(key, ctx)
-        memo = ctx.instance_variable_get(:@__schema_cache__)
-        return memo[key] if memo&.key?(key)
-
-        value = @bindings[key][1].call(ctx)
-        memo[key] = value if memo
-        value
-      end
-
-      private
-
-      def hash_like?(obj)
-        obj.respond_to?(:key?) && obj.respond_to?(:[])
-      end
-
-      def validate_keys(keys)
-        unknown_keys = keys - @bindings.keys
-        return keys if unknown_keys.empty?
-
-        raise Kumi::Errors::RuntimeError, "No binding named #{unknown_keys.first}"
-      end
-    end
-  end
-end

data/lib/kumi/core/compiler/expression_compiler.rb

@@ -1,146 +0,0 @@
-module Kumi
-  module Core
-    module Compiler
-      module ExpressionCompiler
-        private
-
-        def compile_list(expr)
-          fns = expr.elements.map { |e| compile_expr(e) }
-          ->(ctx) { fns.map { |fn| fn.call(ctx) } }
-        end
-
-        def compile_call(expr)
-          fn_name = expr.fn_name
-          arg_fns = expr.args.map { |a| compile_expr(a) }
-
-          # Get compilation metadata once
-          compilation_meta = @analysis.state[:broadcasts]&.dig(:compilation_metadata, @current_declaration)
-
-          # Check if this is a vectorized operation
-          if vectorized_operation?(expr)
-            # Build vectorized executor at COMPILATION time
-            executor = Core::VectorizedFunctionBuilder.build_executor(fn_name, compilation_meta, @analysis.state)
-
-            lambda do |ctx|
-              # Evaluate arguments and use pre-built executor at RUNTIME
-              values = arg_fns.map { |fn| fn.call(ctx) }
-              executor.call(values, expr.loc)
-            end
-          else
-            # Use pre-computed function call strategy
-            function_strategy = compilation_meta&.dig(:function_call_strategy) || {}
-
-            if function_strategy[:flattening_required]
-              flattening_info = @analysis.state[:broadcasts][:flattening_declarations][@current_declaration]
-              ->(ctx) { invoke_function_with_flattening(fn_name, arg_fns, ctx, expr.loc, expr.args, flattening_info) }
-            else
-              ->(ctx) { invoke_function(fn_name, arg_fns, ctx, expr.loc) }
-            end
-          end
-        end
-
-        def compile_cascade(expr)
-          # Use metadata to determine if this cascade is vectorized
-          broadcast_meta = @analysis.state[:broadcasts]
-          cascade_info = @current_declaration && broadcast_meta&.dig(:vectorized_operations, @current_declaration)
-          is_vectorized = cascade_info && cascade_info[:source] == :cascade_with_vectorized_conditions_or_results
-
-          # Separate conditional cases from base case
-          conditional_cases = expr.cases.select(&:condition)
-          base_case = expr.cases.find { |c| c.condition.nil? }
-
-          # Compile conditional pairs
-          pairs = conditional_cases.map do |c|
-            condition_fn = if is_vectorized
-                             transform_vectorized_condition(c.condition)
-                           else
-                             compile_expr(c.condition)
-                           end
-            result_fn = compile_expr(c.result)
-            [condition_fn, result_fn]
-          end
-
-          # Compile base case
-          base_fn = base_case ? compile_expr(base_case.result) : nil
-
-          if is_vectorized
-            # Capture the current declaration name in the closure
-            current_decl_name = @current_declaration
-
-            # Get pre-computed cascade strategy
-            compilation_meta = @analysis.state[:broadcasts]&.dig(:compilation_metadata, current_decl_name)
-            cascade_info = compilation_meta&.dig(:cascade_info) || {}
-
-            # Build executor at COMPILATION time (outside the lambda)
-            strategy = @analysis.state[:broadcasts][:cascade_strategies][current_decl_name]
-            executor = strategy ? Core::CascadeExecutorBuilder.build_executor(strategy, @analysis.state) : nil
-
-            # Metadata-driven vectorized cascade evaluation
-            lambda do |ctx|
-              # Evaluate all conditions and results
-              cond_results = pairs.map { |cond, _res| cond.call(ctx) }
-              res_results = pairs.map { |_cond, res| res.call(ctx) }
-              base_result = base_fn&.call(ctx)
-
-              if ENV["DEBUG_CASCADE"]
-                puts "DEBUG: Vectorized cascade evaluation for #{current_decl_name}:"
-                cond_results.each_with_index { |cr, i| puts "  cond_results[#{i}]: #{cr.inspect}" }
-                res_results.each_with_index { |rr, i| puts "  res_results[#{i}]: #{rr.inspect}" }
-                puts "  base_result: #{base_result.inspect}"
-                puts "  Pre-computed cascade_info: #{cascade_info.inspect}"
-              end
-
-              # Use pre-built executor at RUNTIME
-              if executor
-                executor.call(cond_results, res_results, base_result, pairs)
-              else
-                # Fallback for cases without strategy
-                pairs.each_with_index do |(_cond, _res), pair_idx|
-                  return res_results[pair_idx] if cond_results[pair_idx]
-                end
-                base_result
-              end
-            end
-          else
-            # Non-vectorized cascade - standard evaluation
-            lambda do |ctx|
-              pairs.each { |cond, res| return res.call(ctx) if cond.call(ctx) }
-              # If no conditional case matched, return base case
-              base_fn&.call(ctx)
-            end
-          end
-        end
-
-        def transform_vectorized_condition(condition_expr)
-          if condition_expr.is_a?(Kumi::Syntax::CallExpression) &&
-             condition_expr.fn_name == :cascade_and
-
-            puts "    transform_vectorized_condition: handling cascade_and with #{condition_expr.args.length} args" if ENV["DEBUG_CASCADE"]
-
-            # For cascade_and in vectorized contexts, we need to compile it as a structure-level operation
-            # rather than element-wise operation
-            return compile_cascade_and_for_hierarchical_broadcasting(condition_expr)
-          end
-
-          # Otherwise compile normally
-          compile_expr(condition_expr)
-        end
-
-        def compile_cascade_and_for_hierarchical_broadcasting(condition_expr)
-          # Compile individual trait references
-          trait_fns = condition_expr.args.map { |arg| compile_expr(arg) }
-
-          lambda do |ctx|
-            # Evaluate all traits to get their array structures
-            trait_values = trait_fns.map { |fn| fn.call(ctx) }
-
-            fn = Kumi::Registry.fetch(:cascade_and)
-            result = fn.call(*trait_values)
-
-            result
-          end
-        end
-      end
-    end
-  end
-end

data/lib/kumi/core/compiler/function_invoker.rb

@@ -1,55 +0,0 @@
-module Kumi
-  module Core
-    module Compiler
-      module FunctionInvoker
-        private
-
-        def invoke_function(name, arg_fns, ctx, loc)
-          fn = Kumi::Registry.fetch(name)
-          values = arg_fns.map { |fn| fn.call(ctx) }
-
-          # REMOVED AUTO-FLATTENING: Let operations work on the structure they receive
-          # If flattening is needed, it should be handled by explicit operation modes
-          # in the InputElementReference compilation, not here.
-          fn.call(*values)
-        rescue StandardError => e
-          # Preserve original error class and backtrace while adding context
-          enhanced_message = "Error calling fn(:#{name}) at #{loc}: #{e.message}"
-
-          if e.is_a?(Kumi::Core::Errors::Error)
-            # Re-raise Kumi errors with enhanced message but preserve type
-            e.define_singleton_method(:message) { enhanced_message }
-            raise e
-          else
-            # For non-Kumi errors, wrap in RuntimeError but preserve original error info
-            runtime_error = Errors::RuntimeError.new(enhanced_message)
-            runtime_error.set_backtrace(e.backtrace)
-            runtime_error.define_singleton_method(:cause) { e }
-            raise runtime_error
-          end
-        end
-
-        def invoke_function_with_flattening(name, arg_fns, ctx, loc, _original_args, _flattening_info)
-          fn = Kumi::Registry.fetch(name)
-
-          # Use pre-computed flattening indices from analysis
-          compilation_meta = @analysis.state[:broadcasts]&.dig(:compilation_metadata, @current_declaration)
-          flatten_indices = compilation_meta&.dig(:function_call_strategy, :flatten_argument_indices) || []
-
-          values = arg_fns.map.with_index do |arg_fn, index|
-            value = arg_fn.call(ctx)
-            flatten_indices.include?(index) ? flatten_completely(value) : value
-          end
-
-          fn.call(*values)
-        rescue StandardError => e
-          enhanced_message = "Error calling fn(:#{name}) at #{loc}: #{e.message}"
-          runtime_error = Errors::RuntimeError.new(enhanced_message)
-          runtime_error.set_backtrace(e.backtrace)
-          runtime_error.define_singleton_method(:cause) { e }
-          raise runtime_error
-        end
-      end
-    end
-  end
-end

data/lib/kumi/core/compiler/path_traversal_compiler.rb

@@ -1,158 +0,0 @@
-module Kumi
-  module Core
-    module Compiler
-      module PathTraversalCompiler
-        private
-
-        def compile_element_field_reference(expr)
-          path = expr.path
-
-          # Check if we have nested paths metadata for this path
-          nested_paths = @analysis.state[:broadcasts]&.dig(:nested_paths)
-          unless nested_paths && nested_paths[path]
-            raise Errors::CompilationError, "Missing nested path metadata for #{path.inspect}. This indicates an analyzer bug."
-          end
-
-          # Determine operation mode based on context
-          operation_mode = determine_operation_mode_for_path(path)
-          path_metadata = nested_paths[path]
-          lambda do |ctx|
-            traverse_nested_path(ctx, path, operation_mode, path_metadata)
-          end
-
-          # ERROR: All nested paths should have metadata from the analyzer
-          # If we reach here, it means the BroadcastDetector didn't process this path
-        end
-
-        # Metadata-driven nested array traversal using the traversal algorithm from our design
-        def traverse_nested_path(data, path, operation_mode, path_metadata = nil)
-          access_mode = path_metadata&.dig(:access_mode) || :object
-
-          # Use specialized traversal for element access mode
-          result = if access_mode == :element
-                     traverse_element_path(data, path, operation_mode)
-                   else
-                     traverse_path_recursive(data, path, operation_mode, access_mode)
-                   end
-
-          # Post-process result based on operation mode
-          case operation_mode
-          when :flatten
-            # Completely flatten nested arrays for aggregation
-            flatten_completely(result)
-          else
-            result
-          end
-        end
-
-        # Specialized traversal for element access mode
-        # In element access, we need to extract the specific field from EvaluationWrapper
-        # then apply progressive traversal based on path depth
-        def traverse_element_path(data, path, _operation_mode)
-          # Handle EvaluationWrapper by extracting the specific field
-          if data.is_a?(Core::EvaluationWrapper)
-            field_name = path.first
-            array_data = data[field_name]
-
-            # Always apply progressive traversal based on path depth
-            # This gives us the structure at the correct nesting level for both
-            # broadcast operations and structure operations
-            if array_data.is_a?(Array) && path.length > 1
-              # Flatten exactly (path_depth - 1) levels to get the desired nesting level
-              array_data.flatten(path.length - 1)
-            else
-              array_data
-            end
-          else
-            data
-          end
-        end
-
-        def traverse_path_recursive(data, path, operation_mode, access_mode = :object, original_path_length = nil)
-          # Track original path length to determine traversal depth
-          original_path_length ||= path.length
-          current_depth = original_path_length - path.length
-
-          return data if path.empty?
-
-          field = path.first
-          remaining_path = path[1..]
-
-          if remaining_path.empty?
-            # Final field - extract based on operation mode
-            case operation_mode
-            when :broadcast, :flatten
-              # Extract field preserving array structure
-              extract_field_preserving_structure(data, field, access_mode, current_depth)
-            else
-              # Simple field access
-              if data.is_a?(Array)
-                data.map do |item|
-                  access_field(item, field, access_mode, current_depth)
-                end
-              else
-                access_field(data, field, access_mode, current_depth)
-              end
-            end
-          elsif data.is_a?(Array)
-            # Intermediate step - traverse deeper
-            # Array of items - traverse each item
-            data.map do |item|
-              traverse_path_recursive(access_field(item, field, access_mode, current_depth), remaining_path, operation_mode, access_mode,
-                                      original_path_length)
-            end
-          else
-            # Single item - traverse directly
-            traverse_path_recursive(access_field(data, field, access_mode, current_depth), remaining_path, operation_mode, access_mode,
-                                    original_path_length)
-          end
-        end
-
-        def extract_field_preserving_structure(data, field, access_mode = :object, depth = 0)
-          if data.is_a?(Array)
-            data.map { |item| extract_field_preserving_structure(item, field, access_mode, depth) }
-          else
-            access_field(data, field, access_mode, depth)
-          end
-        end
-
-        def access_field(data, field, access_mode, _depth = 0)
-          case access_mode
-          when :element
-            # Element access mode - for nested arrays, we need to traverse one level deeper
-            # This enables progressive path traversal like input.cube.layer.row.value
-            if data.is_a?(Core::EvaluationWrapper)
-              data[field]
-            elsif data.is_a?(Array)
-              # For element access, flatten one level to traverse deeper into nested structure
-              data.flatten(1)
-            else
-              # If not an array, return as-is (leaf level)
-              data
-            end
-          when :object
-            # Object access mode - normal hash/object field access
-            data[field]
-          else
-            # Default to object access
-            data[field]
-          end
-        end
-
-        def flatten_completely(data)
-          result = []
-          flatten_recursive(data, result)
-          result
-        end
-
-        def flatten_recursive(data, result)
-          if data.is_a?(Array)
-            data.each { |item| flatten_recursive(item, result) }
-          else
-            result << data
-          end
-        end
-      end
-    end
-  end
-end