kumi 0.0.15 → 0.0.16

data/lib/kumi/core/compiler/access_emit/ravel.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+module Kumi::Core::Compiler::AccessEmit
+  module Ravel
+    extend Base
+    module_function
+    def build(plan)
+      policy     = plan.on_missing || :error
+      key_policy = plan.key_policy || :indifferent
+      path_key   = plan.path
+      segs       = segment_ops(plan.operations)
+
+      code = +"lambda do |data|\n"
+      code << "  out = []\n"
+      nodev, depth, loop_depth = "node0", 0, 0
+      code << "  #{nodev} = data\n"
+
+      segs.each do |seg|
+        if seg == :array
+          code << "  #{array_guard_code(node_var: nodev, mode: :ravel, policy: policy, path_key: path_key, map_depth: loop_depth)}\n"
+          code << "  ary#{loop_depth} = #{nodev}\n"
+          code << "  len#{loop_depth} = ary#{loop_depth}.length\n"
+          code << "  i#{loop_depth} = -1\n"
+          code << "  while (i#{loop_depth} += 1) < len#{loop_depth}\n"
+          child = "node#{depth + 1}"
+          code << "    #{child} = ary#{loop_depth}[i#{loop_depth}]\n"
+          nodev = child; depth += 1; loop_depth += 1
+        else
+          seg.each do |(_, key, preview)|
+            code << "  "
+            code << fetch_hash_code(node_var: nodev, key: key, key_policy: key_policy,
+                                    preview_array: preview, mode: :ravel, policy: policy,
+                                    path_key: path_key, map_depth: loop_depth)
+            code << "\n"
+          end
+        end
+      end
+
+      code << "  out << #{nodev}\n"
+      while loop_depth.positive?
+        code << "  end\n"
+        loop_depth -= 1
+        nodev = "node#{depth - 1}"
+        depth -= 1
+      end
+
+      code << "  out\nend\n"
+      code
+    end
+  end
+end

data/lib/kumi/core/compiler/access_emit/read.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+module Kumi::Core::Compiler::AccessEmit
+  module Read
+    extend Base
+    module_function
+    def build(plan)
+      policy     = plan.on_missing || :error
+      key_policy = plan.key_policy || :indifferent
+      path_key   = plan.path
+      ops        = plan.operations
+
+      body = ops.map do |op|
+        case op[:type]
+        when :enter_hash
+          fetch_hash_code(node_var: "node", key: op[:key], key_policy: key_policy,
+                          preview_array: false, mode: :read, policy: policy,
+                          path_key: path_key, map_depth: 0)
+        when :enter_array
+          %(raise TypeError, "Array encountered in :read accessor at '#{path_key}'")
+        end
+      end.join("\n      ")
+
+      <<~RUBY
+        lambda do |data|
+          node = data
+          #{body}
+          node
+        end
+      RUBY
+    end
+  end
+end

data/lib/kumi/core/ir/execution_engine/interpreter.rb

@@ -8,70 +8,34 @@ module Kumi
         module Interpreter
           PRODUCES_SLOT = %i[const load_input ref array map reduce lift align_to switch].freeze
           NON_PRODUCERS = %i[guard_push guard_pop assign store].freeze
+          EMPTY_ARY = [].freeze
 
-          def self.build_name_index(ir_module)
-            index = {}
-            ir_module.decls.each do |decl|
-              decl.ops.each do |op|
-                next unless op.tag == :store
-
-                name = op.attrs[:name]
-                index[name] = decl if name
-              end
-            end
-            index
-          end
-
-          def self.run(ir_module, ctx, accessors:, registry:)
-            # Validate registry is properly initialized
-            raise ArgumentError, "Registry cannot be nil" if registry.nil?
-            raise ArgumentError, "Registry must be a Hash, got #{registry.class}" unless registry.is_a?(Hash)
-
+          def self.run(schedule, input:, runtime:, accessors:, registry:)
+            prof = Profiler.enabled?
             # --- PROFILER: init per run (but not in persistent mode) ---
-            if Profiler.enabled?
-              schema_name = ctx[:schema_name] || "UnknownSchema"
-              if Profiler.persistent?
-                # In persistent mode, just update schema name without full reset
-                Profiler.set_schema_name(schema_name)
-              else
-                # Normal mode: full reset with schema name
-                Profiler.reset!(meta: { decls: ir_module.decls&.size || 0, schema_name: schema_name })
-              end
+            if prof
+              schema_name = runtime[:schema_name] || "UnknownSchema"
+              # In persistent mode, just update schema name without full reset
+              Profiler.set_schema_name(schema_name)
             end
 
             outputs = {}
-            target = ctx[:target]
+            target = runtime[:target]
             guard_stack = [true]
 
-            # Always ensure we have a declaration cache - either from caller or new for this VM run
-            declaration_cache = ctx[:declaration_cache] || {}
-
-            # Build name index for targeting by stored names
-            name_index = ctx[:name_index] || (target ? build_name_index(ir_module) : nil)
+            # Caches live in runtime (engine frame), not input
+            declaration_cache = runtime[:declaration_cache]
 
             # Choose declarations to execute - prefer explicit schedule if present
-            decls_to_run =
-              if ctx[:decls_to_run]
-                ctx[:decls_to_run] # array of decl objects
-              elsif target
-                # Prefer a decl that STORES the target (covers __vec twins)
-                d = name_index && name_index[target]
-                # Fallback: allow targeting by decl name (legacy behavior)
-                d ||= ir_module.decls.find { |dd| dd.name == target }
-                raise "Unknown target: #{target}" unless d
-
-                [d]
-              else
-                ir_module.decls
-              end
+            # decls_to_run = runtime[:decls_to_run] || ir_module.decls
 
-            decls_to_run.each do |decl|
+            schedule.each do |decl|
               slots = []
               guard_stack = [true] # reset per decl
 
               decl.ops.each_with_index do |op, op_index|
-                t0 = Profiler.enabled? ? Profiler.t0 : nil
-                cpu_t0 = Profiler.enabled? ? Profiler.cpu_t0 : nil
+                t0 = prof ? Profiler.t0 : nil
+                cpu_t0 = prof ? Profiler.cpu_t0 : nil
                 rows_touched = nil
                 if ENV["ASSERT_VM_SLOTS"] == "1"
                   expected = op_index
@@ -97,7 +61,7 @@ module Kumi
                                    false
                                  end
                   slots << nil # keep slot_id == op_index
-                  if t0
+                  if prof
                     Profiler.record!(decl: decl.name, idx: op_index, tag: op.tag, op: op, t0: t0, cpu_t0: cpu_t0, rows: 0,
                                      note: "enter")
                   end
@@ -122,92 +86,46 @@ module Kumi
 
                 case op.tag
 
-                when :assign
-                  dst = op.attrs[:dst]
-                  src = op.attrs[:src]
-                  raise "assign: dst/src OOB" if dst >= slots.length || src >= slots.length
-
-                  slots[dst] = slots[src]
-                  Profiler.record!(decl: decl.name, idx: op_index, tag: :assign, op: op, t0: t0, cpu_t0: cpu_t0, rows: 1) if t0
-
                 when :const
                   result = Values.scalar(op.attrs[:value])
-                  puts "DEBUG Const #{op.attrs[:value].inspect}: result=#{result}" if ENV["DEBUG_VM_ARGS"]
                   slots << result
                   Profiler.record!(decl: decl.name, idx: op_index, tag: :const, op: op, t0: t0, cpu_t0: cpu_t0, rows: 1) if t0
 
                 when :load_input
                   plan_id = op.attrs[:plan_id]
-                  scope = op.attrs[:scope] || []
+                  scope = op.attrs[:scope] || EMPTY_ARY
                   scalar = op.attrs[:is_scalar]
                   indexed = op.attrs[:has_idx]
 
-                  # NEW: consult runtime accessor cache
-                  acc_cache = ctx[:accessor_cache] || {}
-                  input_obj = ctx[:input] || ctx["input"]
-                  cache_key = [plan_id, input_obj.object_id]
+                  raw = accessors[plan_id].call(input) # <- memoized by ExecutionEngine
 
-                  if acc_cache.key?(cache_key)
-                    raw = acc_cache[cache_key]
-                    hit = true
-                  else
-                    raw = accessors.fetch(plan_id).call(input_obj)
-                    acc_cache[cache_key] = raw
-                    hit = false
-                  end
-
-                  puts "DEBUG LoadInput plan_id: #{plan_id} raw_values: #{raw.inspect} cache_hit: #{hit}" if ENV["DEBUG_VM_ARGS"]
                   slots << if scalar
                              Values.scalar(raw)
                            elsif indexed
-                             rows_touched = raw.respond_to?(:size) ? raw.size : raw.count
+                             rows_touched = prof && raw.respond_to?(:size) ? raw.size : raw.count
                              Values.vec(scope, raw.map { |v, idx| { v: v, idx: Array(idx) } }, true)
                            else
-                             rows_touched = raw.respond_to?(:size) ? raw.size : raw.count
+                             rows_touched = prof && raw.respond_to?(:size) ? raw.size : raw.count
                              Values.vec(scope, raw.map { |v| { v: v } }, false)
                            end
                   rows_touched ||= 1
-                  cache_note = hit ? "hit:#{plan_id}" : "miss:#{plan_id}"
                   if t0
                     Profiler.record!(decl: decl.name, idx: op_index, tag: :load_input, op: op, t0: t0, cpu_t0: cpu_t0,
-                                     rows: rows_touched, note: cache_note)
+                                     rows: rows_touched, note: "ok")
                   end
 
                 when :ref
                   name = op.attrs[:name]
-
-                  if outputs.key?(name)
-                    referenced = outputs[name]
-                    hit = :outputs
-                  elsif declaration_cache.key?(name)
-                    referenced = declaration_cache[name]
-                    hit = :cache
-                  else
-                    raise "unscheduled ref #{name}: producer not executed or dependency analysis failed"
-                  end
-
-                  if ENV["DEBUG_VM_ARGS"]
-                    puts "DEBUG Ref #{name}: #{referenced[:k] == :scalar ? "scalar(#{referenced[:v].inspect})" : "#{referenced[:k]}(#{referenced[:rows]&.size || 0} rows)"}"
-                  end
+                  referenced = outputs[name] { raise "unscheduled ref #{name}: producer not executed or dependency analysis failed" }
 
                   slots << referenced
                   rows_touched = referenced[:k] == :vec ? (referenced[:rows]&.size || 0) : 1
-                  if t0
+                  if prof
                     Profiler.record!(decl: decl.name, idx: op_index, tag: :ref, op: op, t0: t0, cpu_t0: cpu_t0,
                                      rows: rows_touched, note: hit)
                   end
 
                 when :array
-                  # Validate slot indices before accessing
-                  op.args.each do |slot_idx|
-                    if slot_idx >= slots.length
-                      raise "Array operation: slot index #{slot_idx} out of bounds (slots.length=#{slots.length})"
-                    elsif slots[slot_idx].nil?
-                      raise "Array operation: slot #{slot_idx} is nil " \
-                            "(available slots: #{slots.length}, non-nil slots: #{slots.compact.length})"
-                    end
-                  end
-
                   parts = op.args.map { |i| slots[i] }
                   if parts.all? { |p| p[:k] == :scalar }
                     slots << Values.scalar(parts.map { |p| p[:v] })
@@ -231,63 +149,43 @@ module Kumi
                   fn_name = op.attrs[:fn]
                   fn_entry = registry[fn_name] or raise "Function #{fn_name} not found in registry"
                   fn = fn_entry.fn
-                  puts "DEBUG Map #{fn_name}: args=#{op.args.inspect}" if ENV["DEBUG_VM_ARGS"]
 
                   # Validate slot indices before accessing
-                  op.args.each do |slot_idx|
-                    if slot_idx >= slots.length
-                      raise "Map operation #{fn_name}: slot index #{slot_idx} out of bounds (slots.length=#{slots.length})"
-                    elsif slots[slot_idx].nil?
-                      raise "Map operation #{fn_name}: slot #{slot_idx} is nil " \
-                            "(available slots: #{slots.length}, non-nil slots: #{slots.compact.length})"
-                    end
-                  end
+                  # op.args.each do |slot_idx|
+                  #   if slot_idx >= slots.length
+                  #     raise "Map operation #{fn_name}: slot index #{slot_idx} out of bounds (slots.length=#{slots.length})"
+                  #   elsif slots[slot_idx].nil?
+                  #     raise "Map operation #{fn_name}: slot #{slot_idx} is nil " \
+                  #           "(available slots: #{slots.length}, non-nil slots: #{slots.compact.length})"
+                  #   end
+                  # end
 
                   args = op.args.map { |slot_idx| slots[slot_idx] }
 
                   if args.all? { |a| a[:k] == :scalar }
-                    puts "DEBUG Scalar call #{fn_name}: args=#{args.map { |a| a[:v] }.inspect}" if ENV["DEBUG_VM_ARGS"]
                     scalar_args = args.map { |a| a[:v] }
                     result = fn.call(*scalar_args)
                     slots << Values.scalar(result)
                   else
                     base = args.find { |a| a[:k] == :vec } or raise "Map needs a vec carrier"
-                    puts "DEBUG Vec call #{fn_name}: base=#{base.inspect}" if ENV["DEBUG_VM_ARGS"]
                     # Preserve original order: broadcast scalars in-place
                     arg_vecs = args.map { |a| a[:k] == :scalar ? Combinators.broadcast_scalar(a, base) : a }
-                    puts "DEBUG Vec call #{fn_name}: arg_vecs=#{arg_vecs.inspect}" if ENV["DEBUG_VM_ARGS"]
                     scopes = arg_vecs.map { |v| v[:scope] }.uniq
-                    puts "DEBUG Vec call #{fn_name}: scopes=#{scopes.inspect}" if ENV["DEBUG_VM_ARGS"]
                     raise "Cross-scope Map without Join" unless scopes.size <= 1
 
                     zipped = Combinators.zip_same_scope(*arg_vecs)
 
-                    # if ENV["DEBUG_VM_ARGS"] && fn_name == :if
-                    #   puts "DEBUG Vec call #{fn_name}: zipped rows:"
-                    #   zipped[:rows].each_with_index do |row, i|
-                    #     puts "  [#{i}] args=#{Array(row[:v]).inspect}"
-                    #   end
-                    # end
-
-                    puts "DEBUG Vec call #{fn_name}: zipped rows=#{zipped[:rows].inspect}" if ENV["DEBUG_VM_ARGS"]
                     rows = zipped[:rows].map do |row|
                       row_args = Array(row[:v])
                       vr = fn.call(*row_args)
                       row.key?(:idx) ? { v: vr, idx: row[:idx] } : { v: vr }
                     end
-                    puts "DEBUG Vec call #{fn_name}: result rows=#{rows.inspect}" if ENV["DEBUG_VM_ARGS"]
 
                     slots << Values.vec(base[:scope], rows, base[:has_idx])
                   end
 
                 when :switch
                   chosen = op.attrs[:cases].find do |(cond_slot, _)|
-                    if cond_slot >= slots.length
-                      raise "Switch operation: condition slot #{cond_slot} out of bounds (slots.length=#{slots.length})"
-                    elsif slots[cond_slot].nil?
-                      raise "Switch operation: condition slot #{cond_slot} is nil (available slots: #{slots.length}, non-nil slots: #{slots.compact.length})"
-                    end
-
                     c = slots[cond_slot]
                     if c[:k] == :scalar
                       !!c[:v]
@@ -297,22 +195,12 @@ module Kumi
                     end
                   end
                   result_slot = chosen ? chosen[1] : op.attrs[:default]
-                  if result_slot >= slots.length
-                    raise "Switch operation: result slot #{result_slot} out of bounds (slots.length=#{slots.length})"
-                  elsif slots[result_slot].nil?
-                    raise "Switch operation: result slot #{result_slot} is nil (available slots: #{slots.length}, non-nil slots: #{slots.compact.length})"
-                  end
 
                   slots << slots[result_slot]
 
                 when :store
                   name = op.attrs[:name]
                   src  = op.args[0] or raise "store: missing source slot"
-                  if src >= slots.length
-                    raise "Store operation '#{name}': source slot #{src} out of bounds (slots.length=#{slots.length})"
-                  elsif slots[src].nil?
-                    raise "Store operation '#{name}': source slot #{src} is nil (available slots: #{slots.length}, non-nil slots: #{slots.compact.length})"
-                  end
 
                   result = slots[src]
                   outputs[name] = result
@@ -329,10 +217,8 @@ module Kumi
                   fn = fn_entry.fn
 
                   src = slots[op.args[0]]
-                  raise "Reduce expects Vec" unless src[:k] == :vec
-
-                  result_scope = Array(op.attrs[:result_scope] || [])
-                  axis         = Array(op.attrs[:axis] || [])
+                  result_scope = op.attrs[:result_scope]
+                  axis         = op.attrs[:axis]
 
                   if result_scope.empty?
                     # === GLOBAL REDUCE ===
@@ -340,12 +226,6 @@ module Kumi
                     vals = src[:rows].map { |r| r[:v] }
                     slots << Values.scalar(fn.call(vals))
                   else
-                    # === GROUPED REDUCE ===
-                    # Must have indices to group by prefix keys.
-                    unless src[:has_idx]
-                      raise "Grouped reduce requires indexed input (got ravel) for #{op.attrs[:fn]} at #{result_scope.inspect}"
-                    end
-
                     group_len = result_scope.length
 
                     # Preserve stable source order so zips with other @result_scope vecs line up.
@@ -368,39 +248,17 @@ module Kumi
 
                 when :lift
                   src_slot = op.args[0]
-                  if src_slot >= slots.length
-                    raise "Lift operation: source slot #{src_slot} out of bounds (slots.length=#{slots.length})"
-                  elsif slots[src_slot].nil?
-                    raise "Lift operation: source slot #{src_slot} is nil (available slots: #{slots.length}, non-nil slots: #{slots.compact.length})"
-                  end
 
                   v = slots[src_slot]
-                  to_scope = op.attrs[:to_scope] || []
+                  to_scope = op.attrs[:to_scope] || EMPTY_ARY
                   depth    = [to_scope.length, v[:rank] || v[:rows].first&.dig(:idx)&.length || 0].min
                   slots << Values.scalar(Combinators.group_rows(v[:rows], depth))
 
                 when :align_to
-                  tgt_slot = op.args[0]
-                  src_slot = op.args[1]
-
-                  if tgt_slot >= slots.length
-                    raise "AlignTo operation: target slot #{tgt_slot} out of bounds (slots.length=#{slots.length})"
-                  elsif slots[tgt_slot].nil?
-                    raise "AlignTo operation: target slot #{tgt_slot} is nil " \
-                          "(available slots: #{slots.length}, non-nil slots: #{slots.compact.length})"
-                  end
-
-                  if src_slot >= slots.length
-                    raise "AlignTo operation: source slot #{src_slot} out of bounds (slots.length=#{slots.length})"
-                  elsif slots[src_slot].nil?
-                    raise "AlignTo operation: source slot #{src_slot} is nil " \
-                          "(available slots: #{slots.length}, non-nil slots: #{slots.compact.length})"
-                  end
+                  tgt = slots[op.args[0]]
+                  src = slots[op.args[1]]
 
-                  tgt = slots[tgt_slot]
-                  src = slots[src_slot]
-
-                  to_scope = op.attrs[:to_scope] || []
+                  to_scope = op.attrs[:to_scope] || EMPTY_ARY
                   require_unique = op.attrs[:require_unique] || false
                   on_missing = op.attrs[:on_missing] || :error
 
@@ -409,9 +267,6 @@ module Kumi
                                                            on_missing: on_missing)
                   slots << aligned
 
-                when :join
-                  raise NotImplementedError, "Join not implemented yet"
-
                 else
                   raise "Unknown operation: #{op.tag}"
                 end

data/lib/kumi/core/ir/execution_engine/values.rb

@@ -13,14 +13,14 @@ module Kumi
 
           # Create a vector with scope and rows
           def self.vec(scope, rows, has_idx)
-            if has_idx
-              rank = rows.empty? ? 0 : rows.first[:idx].length
-              # TODO: > Make sure this is not costly
-              # raise if rows.any? { |r| r[:idx].length != rank }
-              rows = rows.sort_by { |r| r[:idx] } # one-time sort
-            else
-              rank = 0
-            end
+            rank = if has_idx
+                     rows.empty? ? 0 : rows.first[:idx].length
+                   # TODO: > Make sure this is not costly
+                   # raise if rows.any? { |r| r[:idx].length != rank }
+                   # rows = rows.sort_by { |r| r[:idx] } # one-time sort
+                   else
+                     0
+                   end
 
             { k: :vec, scope: scope, rows: rows, has_idx: has_idx, rank: rank }
           end

data/lib/kumi/core/ir/execution_engine.rb

@@ -41,29 +41,13 @@ module Kumi
       # - DEBUG_VM_ARGS=1 prints per-op execution and arguments.
       # - DEBUG_GROUP_ROWS=1 prints grouping decisions during Lift.
       module ExecutionEngine
-        def self.run(ir_module, ctx, accessors:, registry:)
-          # Use persistent accessor cache if available, otherwise create temporary one
-          memoized_accessors = Dev::Profiler.phase("engine.memoization") do
-            # Include input data in cache key to avoid cross-context pollution
-            input_key = ctx[:input]&.hash || ctx["input"]&.hash || 0
-            add_persistent_memoization(accessors, ctx[:accessor_cache], input_key)
-          end
+        def self.run(schedule, input:, accessors:, registry:, runtime: {})
+          runtime[:accessor_cache] ||= {}
 
           Dev::Profiler.phase("engine.interpreter") do
-            Interpreter.run(ir_module, ctx, accessors: memoized_accessors, registry: registry)
+            Interpreter.run(schedule, input: input, runtime: runtime, accessors: accessors, registry: registry)
           end
         end
-
-        private
-
-        def self.add_persistent_memoization(accessors, cache, input_key)
-          accessors.map do |plan_id, accessor_fn|
-            [plan_id, lambda do |input_data|
-              cache_key = [plan_id, input_key]
-              cache[cache_key] ||= accessor_fn.call(input_data)
-            end]
-          end.to_h
-        end
       end
     end
   end

data/lib/kumi/dev/parse.rb

@@ -35,20 +35,20 @@ module Kumi
         end
 
         # Report trace file if enabled
-        if opts[:trace] && res.respond_to?(:trace_file)
-          puts "Trace written to: #{res.trace_file}"
-        end
+        puts "Trace written to: #{res.trace_file}" if opts[:trace] && res.respond_to?(:trace_file)
 
         # Determine file extension and renderer
         extension = opts[:json] ? "json" : "txt"
-        golden_path = File.join(File.dirname(schema_path), "expected", "ir.#{extension}")
+
+        file_name = File.basename(schema_path)
+        golden_path = File.join(File.dirname(schema_path), "expected", "#{file_name}_ir.#{extension}")
 
         # Render IR
         rendered = if opts[:json]
-          Dev::IR.to_json(res.ir, pretty: true)
-        else
-          Dev::IR.to_text(res.ir)
-        end
+                     Dev::IR.to_json(res.ir, pretty: true)
+                   else
+                     Dev::IR.to_text(res.ir)
+                   end
 
         # Handle write mode
         if opts[:write]
@@ -71,7 +71,7 @@ module Kumi
           end
         end
 
-        # Handle no-diff mode  
+        # Handle no-diff mode
         if opts[:no_diff]
           puts rendered
           return true
@@ -84,7 +84,7 @@ module Kumi
           Tempfile.create(["actual", File.extname(golden_path)]) do |actual_file|
             actual_file.write(rendered)
             actual_file.flush
-            
+
             result = `diff -u --label=expected --label=actual #{golden_path} #{actual_file.path}`
             if result.empty?
               puts "No changes (#{golden_path})"
@@ -97,9 +97,9 @@ module Kumi
         else
           # No golden file exists, just print the output
           puts rendered
-          return true
+          true
         end
       end
     end
   end
-end
+end