kumi 0.0.15 → 0.0.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5405d7d0612a81e5154bd1d452fdfc150691b022137fc0ee132c47ede1a58e2e
-  data.tar.gz: '093cf7a6d305c02f92de600b06f62be39f8af90d798a0f93ed3ef59f539ada9b'
+  metadata.gz: 9f51be8774c472629e599ce3edd4ab02551edfd52fea8676e2ee93eed5198800
+  data.tar.gz: 26532cec84e5c59553031dc86d82abff310d2a79c857776e1d793af2b35e00ea
 SHA512:
-  metadata.gz: b3ea711bf465e0c11cc95fabb3809dd632ebbfcc8c36297b161fb1f179fffdda5df1e5c033968e837dd8ed3f983639416de08bd371f30be9f2cefd5543efe1ff
-  data.tar.gz: 0a63fe824fb604639b4efb9cfc2ce24a93429110adc75cfd3edc905c58b3501273ad07a3cf66415eabce4e099c5fd4d202bd2c4064875d7a73e0c2a26f0689cb
+  metadata.gz: 3ba28da3acbf5cd430902c29e34e5786562e7121f9c3851a2e9db3a04a13fc8b1a5e9729649f3daadb523ea55f2c97a1e4560139b9809aa636eff21e9716abbf
+  data.tar.gz: 6683358d25786a5e15cba975e785b3178ec0ea13f40a3eeb940e72f25886f5004c499f3e80ddded31fdd23f3981cbe5244db1ab0aec1405b3c52acb68c4b1e60

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 ## [Unreleased]
 
+## [0.0.16] – 2025-08-22
+
+### Performance
+- Input accessor code generation replaces nested lambda chains with compiled Ruby methods
+- Fix cache handling in Runtime - it was being recreated on updates
+- Add early shortcut for Analyzer Passes.
+
 ## [0.0.15] – 2025-08-21
 ### Added
 - (DX) Schema-aware VM profiling with multi-schema performance analysis

data/lib/kumi/analyzer.rb

@@ -3,6 +3,7 @@
 module Kumi
   module Analyzer
     Result = Struct.new(:definitions, :dependency_graph, :leaf_map, :topo_order, :decl_types, :state, keyword_init: true)
+    ERROR_THRESHOLD_PASS = Core::Analyzer::Passes::LowerToIRPass
 
     DEFAULT_PASSES = [
       Core::Analyzer::Passes::NameIndexer,                     # 1. Finds all names and checks for duplicates.
@@ -22,7 +23,9 @@ module Kumi
       Core::Analyzer::Passes::JoinReducePlanningPass,          # 16. Plans join/reduce operations (Generates IR Structs)
       Core::Analyzer::Passes::LowerToIRPass,                   # 17. Lowers the schema to IR (Generates IR Structs)
       Core::Analyzer::Passes::LoadInputCSE,                    # 18. Eliminates redundant load_input operations
-      Core::Analyzer::Passes::IRDependencyPass                 # 19. Extracts IR-level dependencies for VM execution optimization
+      Core::Analyzer::Passes::IRDependencyPass,                # 19. Extracts IR-level dependencies for VM execution optimization
+      Core::Analyzer::Passes::IRExecutionSchedulePass # 20. Builds a precomputed execution schedule.
+
     ].freeze
 
     def self.analyze!(schema, passes: DEFAULT_PASSES, **opts)
@@ -44,6 +47,8 @@ module Kumi
       skipping   = !!resume_at
 
       passes.each_with_index do |pass_class, idx|
+        raise handle_analysis_errors(errors) if (ERROR_THRESHOLD_PASS == pass_class) && !errors.empty?
+
         pass_name = pass_class.name.split("::").last
 
         if skipping

data/lib/kumi/core/analyzer/passes/ir_dependency_pass.rb

@@ -7,19 +7,19 @@ module Kumi
         # RESPONSIBILITY: Extract IR-level dependencies for VM execution optimization
         # DEPENDENCIES: :ir_module from LowerToIRPass
         # PRODUCES: :ir_dependencies - Hash mapping declaration names to referenced bindings
-        #           :name_index - Hash mapping stored binding names to producing declarations
+        #           :ir_name_index - Hash mapping stored binding names to producing declarations
         # INTERFACE: new(schema, state).run(errors)
-        # 
+        #
         # NOTE: This pass extracts actual IR-level dependencies by analyzing :ref operations
         # in the generated IR, providing the dependency information needed for optimized VM scheduling.
         class IRDependencyPass < PassBase
           def run(errors)
             ir_module = get_state(:ir_module, required: true)
-            
+
             ir_dependencies = build_ir_dependency_map(ir_module)
-            name_index = build_name_index(ir_module)
-            
-            state.with(:ir_dependencies, ir_dependencies).with(:name_index, name_index)
+            ir_name_index = build_ir_name_index(ir_module)
+
+            state.with(:ir_dependencies, ir_dependencies).with(:ir_name_index, ir_name_index)
           end
 
           private
@@ -27,41 +27,39 @@ module Kumi
           # Build a map of declaration -> [stored_bindings_it_references] from the IR
           def build_ir_dependency_map(ir_module)
             deps_map = {}
-            
+
             ir_module.decls.each do |decl|
               refs = []
               decl.ops.each do |op|
-                if op.tag == :ref
-                  refs << op.attrs[:name]
-                end
+                refs << op.attrs[:name] if op.tag == :ref
               end
               deps_map[decl.name] = refs
             end
-            
+
             deps_map.freeze
           end
 
           # Build name index to map stored binding names to their producing declarations
-          def build_name_index(ir_module)
-            name_index = {}
-            
+          def build_ir_name_index(ir_module)
+            ir_name_index = {}
+
             ir_module.decls.each do |decl|
               # Map the primary declaration name
-              name_index[decl.name] = decl
-              
+              ir_name_index[decl.name] = decl
+
               # Also map any vectorized twin names produced by this declaration
               decl.ops.each do |op|
                 if op.tag == :store
                   stored_name = op.attrs[:name]
-                  name_index[stored_name] = decl
+                  ir_name_index[stored_name] = decl
                 end
               end
             end
-            
-            name_index.freeze
+
+            ir_name_index.freeze
           end
         end
       end
     end
   end
-end
+end

data/lib/kumi/core/analyzer/passes/ir_execution_schedule_pass.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Core
+    module Analyzer
+      module Passes
+        # PRODUCES: :execution_schedules => { store_name(Symbol) => [Decl, ...] }
+        class IRExecutionSchedulePass < PassBase
+          def run(errors)
+            ir          = get_state(:ir_module, required: true)
+            deps        = get_state(:ir_dependencies, required: true)    # decl_name => [binding_name, ...]
+            name_index  = get_state(:ir_name_index, required: true)      # binding_name => Decl  (← use IR-specific index)
+
+            by_name     = ir.decls.to_h { |d| [d.name, d] }
+            pos         = ir.decls.each_with_index.to_h                  # for deterministic ordering
+
+            closure_cache = {}
+            visiting      = {}
+
+            visit = lambda do |dn|
+              return closure_cache[dn] if closure_cache.key?(dn)
+
+              raise Kumi::Core::Errors::TypeError, "cycle detected in IR at #{dn.inspect}" if visiting[dn]
+
+              visiting[dn] = true
+
+              # Resolve binding refs -> producing decl names
+              preds = Array(deps[dn]).filter_map { |b| name_index[b]&.name }.uniq
+
+              # Deterministic order: earlier IR decls first
+              preds.sort_by! { |n| pos[n] || Float::INFINITY }
+
+              order = []
+              preds.each do |p|
+                next if p == dn # guard against self-deps; treat as error if you prefer
+
+                order.concat(visit.call(p))
+              end
+              order << dn unless order.last == dn
+
+              visiting.delete(dn)
+              closure_cache[dn] = order.uniq.freeze
+            end
+
+            schedules = {}
+
+            ir.decls.each do |decl|
+              target_names = [decl.name] + decl.ops.select { _1.tag == :store }.map { _1.attrs[:name] }
+
+              seq = visit.call(decl.name).map { |dn| by_name.fetch(dn) }.freeze
+
+              target_names.each do |t|
+                if schedules.key?(t) && schedules[t] != seq
+                  raise Kumi::Core::Errors::TypeError,
+                        "duplicate schedule target #{t.inspect} produced by #{schedules[t].last.name} and #{decl.name}"
+                end
+                schedules[t] = seq
+              end
+            end
+
+            state.with(:ir_execution_schedules, schedules.freeze)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kumi/core/analyzer/passes/toposorter.rb

@@ -18,7 +18,7 @@ module Kumi
             # Create node index for later passes to use
             node_index = build_node_index(definitions)
             order = compute_topological_order(dependency_graph, definitions, errors)
-            
+
             state.with(:evaluation_order, order).with(:node_index, node_index)
           end
 
@@ -26,34 +26,32 @@ module Kumi
 
           def build_node_index(definitions)
             index = {}
-            
+
             # Walk all declarations and their expressions to index every node
             definitions.each_value do |decl|
               index_node_recursive(decl, index)
             end
-            
+
             index
           end
-          
+
           def index_node_recursive(node, index)
             return unless node
-            
+
             # Index this node by its object_id
             index[node.object_id] = {
               node: node,
-              type: node.class.name.split('::').last,
+              type: node.class.name.split("::").last,
               metadata: {}
             }
-            
+
             # Use the same approach as the visitor pattern - recursively index all children
-            if node.respond_to?(:children)
-              node.children.each { |child| index_node_recursive(child, index) }
-            end
-            
+            node.children.each { |child| index_node_recursive(child, index) } if node.respond_to?(:children)
+
             # Index expression for declaration nodes
-            if node.respond_to?(:expression)
-              index_node_recursive(node.expression, index)
-            end
+            return unless node.respond_to?(:expression)
+
+            index_node_recursive(node.expression, index)
           end
 
           def compute_topological_order(graph, definitions, errors)
@@ -96,7 +94,6 @@ module Kumi
             order.freeze
           end
 
-
           def report_unexpected_cycle(temp_marks, current_node, errors)
             cycle_path = temp_marks.to_a.join(" → ") + " → #{current_node}"
 

data/lib/kumi/core/compiler/access_builder.rb

@@ -2,18 +2,32 @@ module Kumi
   module Core
     module Compiler
       class AccessBuilder
+        class << self
+          attr_accessor :strategy
+        end
+
+        self.strategy = if defined?(RubyVM::YJIT) && RubyVM::YJIT.enabled?
+                          :interp
+                        else
+                          :codegen
+                        end
+
         def self.build(plans)
           accessors = {}
           plans.each_value do |variants|
             variants.each do |plan|
-              key = plan.respond_to?(:accessor_key) ? plan.accessor_key : "#{plan.path}:#{mode}"
-              accessors[key] = build_proc_for(
-                mode: plan.mode,
-                path_key: plan.path,
-                missing: (plan.on_missing || :error).to_sym,
-                key_policy: (plan.key_policy || :indifferent).to_sym,
-                operations: plan.operations
-              )
+              accessors[plan.accessor_key] =
+                case strategy
+                when :codegen then AccessCodegen.fetch_or_compile(plan)
+                else
+                  build_proc_for(
+                    mode: plan.mode,
+                    path_key: plan.path,
+                    missing: (plan.on_missing || :error).to_sym,
+                    key_policy: (plan.key_policy || :indifferent).to_sym,
+                    operations: plan.operations
+                  )
+                end
             end
           end
           accessors.freeze
@@ -25,7 +39,6 @@ module Kumi
           when :materialize then Accessors::MaterializeAccessor.build(operations, path_key, missing, key_policy)
           when :ravel       then Accessors::RavelAccessor.build(operations, path_key, missing, key_policy)
           when :each_indexed then Accessors::EachIndexedAccessor.build(operations, path_key, missing, key_policy, true)
-          when :each then Accessors::EachAccessor.build(operations, path_key, missing, key_policy)
           else
             raise "Unknown accessor mode: #{mode.inspect}"
           end

data/lib/kumi/core/compiler/access_codegen.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "digest/sha1"
+
+module Kumi
+  module Core
+    module Compiler
+      class AccessCodegen
+        CACHE = {}
+        CACHE_MUTEX = Mutex.new
+
+        def self.fetch_or_compile(plan)
+          key = Digest::SHA1.hexdigest(Marshal.dump([plan.mode, plan.operations, plan.on_missing, plan.key_policy, plan.path]))
+          CACHE_MUTEX.synchronize do
+            CACHE[key] ||= compile(plan).tap(&:freeze)
+          end
+        end
+
+        def self.compile(plan)
+          case plan.mode
+          when :read         then gen_read(plan)
+          when :materialize  then gen_materialize(plan)
+          when :ravel        then gen_ravel(plan)
+          when :each_indexed then gen_each_indexed(plan)
+          else
+            raise "Unknown accessor mode: #{plan.mode.inspect}"
+          end
+        end
+
+        private_class_method def self.gen_read(plan)
+          code = AccessEmit::Read.build(plan)
+          debug_code(code, plan, "READ") if ENV["DEBUG_CODEGEN"]
+          eval(code, TOPLEVEL_BINDING)
+        end
+
+        private_class_method def self.gen_materialize(plan)
+          code = AccessEmit::Materialize.build(plan)
+          debug_code(code, plan, "MATERIALIZE") if ENV["DEBUG_CODEGEN"]
+          eval(code, TOPLEVEL_BINDING)
+        end
+
+        private_class_method def self.gen_ravel(plan)
+          code = AccessEmit::Ravel.build(plan)
+          debug_code(code, plan, "RAVEL") if ENV["DEBUG_CODEGEN"]
+          eval(code, TOPLEVEL_BINDING)
+        end
+
+        private_class_method def self.gen_each_indexed(plan)
+          code = AccessEmit::EachIndexed.build(plan)
+          debug_code(code, plan, "EACH_INDEXED") if ENV["DEBUG_CODEGEN"]
+          eval(code, TOPLEVEL_BINDING)
+        end
+
+        private_class_method def self.debug_code(code, plan, mode_name)
+          puts "=== Generated #{mode_name} code for #{plan.path}:#{plan.mode} ==="
+          puts code
+        end
+      end
+    end
+  end
+end

data/lib/kumi/core/compiler/access_emit/base.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+module Kumi
+  module Core
+    module Compiler
+      module AccessEmit
+        module Base
+          module_function
+
+          # ---------- IR segmentation ----------
+          def segment_ops(ops)
+            segs, cur = [], []
+            i = 0
+            while i < ops.length
+              case ops[i][:type]
+              when :enter_hash
+                preview = (i + 1 < ops.length) && ops[i + 1][:type] == :enter_array
+                cur << [:enter_hash, ops[i][:key].to_s, preview]
+              when :enter_array
+                segs << cur unless cur.empty?
+                segs << :array
+                cur = []
+              else
+                raise "Unknown operation: #{ops[i].inspect}"
+              end
+              i += 1
+            end
+            segs << cur unless cur.empty?
+            segs
+          end
+
+          # ---------- codegen helpers ----------
+          def fetch_hash_code(node_var:, key:, key_policy:, preview_array:, mode:, policy:, path_key:, map_depth:)
+            effective_policy = preview_array ? :indifferent : (key_policy || :indifferent)
+            str = key.to_s.inspect
+            sym = key.to_sym.inspect
+
+            fetch =
+              case effective_policy
+              when :string
+                %(next_node = #{node_var}.key?(#{str}) ? #{node_var}[#{str}] : :__missing__)
+              when :symbol
+                %(next_node = #{node_var}.key?(#{sym}) ? #{node_var}[#{sym}] : :__missing__)
+              else # :indifferent
+                <<~RB.chomp
+                  next_node =
+                    if #{node_var}.key?(#{str}); #{node_var}[#{str}]
+                    elsif #{node_var}.key?(#{sym}); #{node_var}[#{sym}]
+                    elsif #{node_var}.key?(#{str}); #{node_var}[#{str}] # (string twice ok / predictable)
+                    else :__missing__
+                    end
+                RB
+              end
+
+            miss_action = build_miss_action(policy, mode, map_depth, preview_array, key: key, path_key: path_key)
+
+            <<~RB.chomp
+              raise TypeError, "Expected Hash at '#{path_key}' (#{mode})" unless #{node_var}.is_a?(Hash)
+              #{fetch}
+              if next_node == :__missing__
+                #{miss_action}
+              end
+              #{node_var} = next_node
+            RB
+          end
+
+          def array_guard_code(node_var:, mode:, policy:, path_key:, map_depth:)
+            miss_action = build_array_miss_action(policy, mode, map_depth, path_key)
+            <<~RB.chomp
+              if #{node_var}.nil?
+                #{miss_action}
+              end
+              unless #{node_var}.is_a?(Array)
+                raise TypeError, "Expected Array at '#{path_key}' (#{mode}); got \#{#{node_var}.class}"
+              end
+            RB
+          end
+
+          # ---------- missing behaviors ----------
+          def build_miss_action(policy, mode, map_depth, preview_array, key:, path_key:)
+            case policy
+            when :nil
+              if mode == :ravel
+                base = "out << nil"
+                cont = map_depth.positive? ? "next" : "return out"
+                "#{base}\n#{cont}"
+              elsif mode == :each_indexed
+                if map_depth.positive?
+                  <<~RB.chomp
+                    if block
+                      block.call(nil, idx_vec.dup)
+                      next
+                    else
+                      out << [nil, idx_vec.dup]
+                      next
+                    end
+                  RB
+                else
+                  <<~RB.chomp
+                    if block
+                      block.call(nil, idx_vec.dup)
+                      return nil
+                    else
+                      out << [nil, idx_vec.dup]
+                      return out
+                    end
+                  RB
+                end
+              else # :materialize, :read
+                # Important: for :materialize this is ALWAYS nil (never [])
+                return_val = 'nil'
+                map_depth.positive? ? "next #{return_val}" : "return #{return_val}"
+              end
+            when :skip
+              if mode == :materialize
+                return_val = preview_array ? '[]' : 'nil'
+                map_depth.positive? ? "next #{return_val}" : "return #{return_val}"
+              else
+                map_depth.positive? ? "next" : (mode == :each_indexed ? "if block; return nil; else; return out; end" : "return out")
+              end
+            else # :error
+              %(raise KeyError, "Missing key '#{key}' at '#{path_key}' (#{mode})")
+            end
+          end
+
+          def build_array_miss_action(policy, mode, map_depth, path_key)
+            case policy
+            when :nil
+              if mode == :materialize
+                map_depth.positive? ? "next nil" : "return nil"
+              elsif mode == :each_indexed
+                if map_depth.positive?
+                  <<~RB.chomp
+                    if block
+                      block.call(nil, idx_vec.dup)
+                      next
+                    else
+                      out << [nil, idx_vec.dup]
+                      next
+                    end
+                  RB
+                else
+                  <<~RB.chomp
+                    if block
+                      block.call(nil, idx_vec.dup)
+                      return nil
+                    else
+                      out << [nil, idx_vec.dup]
+                      return out
+                    end
+                  RB
+                end
+              else # :ravel / others
+                base = "out << nil"
+                cont = map_depth.positive? ? "next" : "return out"
+                "#{base}\n#{cont}"
+              end
+            when :skip
+              if mode == :materialize
+                map_depth.positive? ? "next []" : "return []"
+              elsif mode == :each_indexed
+                map_depth.positive? ? "next" : "if block; return nil; else; return out; end"
+              else # :ravel
+                map_depth.positive? ? "next" : "return out"
+              end
+            else
+              %(raise TypeError, "Missing array at '#{path_key}' (#{mode})")
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kumi/core/compiler/access_emit/each_indexed.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+module Kumi::Core::Compiler::AccessEmit
+  module EachIndexed
+    extend Base
+    module_function
+    def build(plan)
+      policy     = plan.on_missing || :error
+      key_policy = plan.key_policy || :indifferent
+      path_key   = plan.path
+      segs       = segment_ops(plan.operations)
+
+      code = +"lambda do |data, &block|\n"
+      code << "  out = []\n"
+      code << "  node0 = data\n"
+      code << "  idx_vec = []\n"
+      nodev, depth, loop_depth = "node0", 0, 0
+
+      segs.each do |seg|
+        if seg == :array
+          code << "  #{array_guard_code(node_var: nodev, mode: :each_indexed, policy: policy, path_key: path_key, map_depth: loop_depth)}\n"
+          code << "  ary#{loop_depth} = #{nodev}\n"
+          code << "  len#{loop_depth} = ary#{loop_depth}.length\n"
+          code << "  i#{loop_depth} = -1\n"
+          code << "  while (i#{loop_depth} += 1) < len#{loop_depth}\n"
+          code << "    idx_vec[#{loop_depth}] = i#{loop_depth}\n"
+          child = "node#{depth + 1}"
+          code << "    #{child} = ary#{loop_depth}[i#{loop_depth}]\n"
+          nodev = child; depth += 1; loop_depth += 1
+        else
+          seg.each do |(_, key, preview)|
+            code << fetch_hash_code(node_var: nodev, key: key, key_policy: key_policy,
+                                    preview_array: preview, mode: :each_indexed, policy: policy,
+                                    path_key: path_key, map_depth: loop_depth)
+            code << "\n"
+          end
+        end
+      end
+
+      code << "  if block\n"
+      code << "    block.call(#{nodev}, idx_vec.dup)\n"
+      code << "  else\n"
+      code << "    out << [#{nodev}, idx_vec.dup]\n"
+      code << "  end\n"
+
+      while loop_depth.positive?
+        code << "  end\n"
+        loop_depth -= 1
+        nodev = "node#{depth - 1}"
+        depth -= 1
+      end
+
+      code << "  block ? nil : out\nend\n"
+      code
+    end
+  end
+end

data/lib/kumi/core/compiler/access_emit/materialize.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+module Kumi::Core::Compiler::AccessEmit
+  module Materialize
+    extend Base
+    module_function
+    def build(plan)
+      policy     = plan.on_missing || :error
+      key_policy = plan.key_policy || :indifferent
+      path_key   = plan.path
+      segs       = segment_ops(plan.operations)
+
+      code = +"lambda do |data|\n"
+      nodev, depth, map_depth = "node0", 0, 0
+      code << "  #{nodev} = data\n"
+
+      segs.each do |seg|
+        if seg == :array
+          code << "  #{array_guard_code(node_var: nodev, mode: :materialize, policy: policy, path_key: path_key, map_depth: map_depth)}\n"
+          child = "node#{depth + 1}"
+          code << "  #{nodev} = #{nodev}.map do |__e#{depth}|\n"
+          code << "    #{child} = __e#{depth}\n"
+          nodev = child; depth += 1; map_depth += 1
+        else
+          seg.each do |(_, key, preview)|
+            code << "  "
+            code << fetch_hash_code(node_var: nodev, key: key, key_policy: key_policy,
+                                    preview_array: preview, mode: :materialize, policy: policy,
+                                    path_key: path_key, map_depth: map_depth)
+            code << "\n"
+          end
+        end
+      end
+
+      while map_depth.positive?
+        code << "  " * map_depth + "#{nodev}\n"
+        code << "  " * (map_depth - 1) + "end\n"
+        nodev = "node#{depth - 1}"
+        depth -= 1
+        map_depth -= 1
+      end
+      code << "  #{nodev}\nend\n"
+      code
+    end
+  end
+end