kumi 0.0.12 → 0.0.14

data/lib/kumi/core/ir/execution_engine.rb

@@ -42,7 +42,36 @@ module Kumi
       # - DEBUG_GROUP_ROWS=1 prints grouping decisions during Lift.
       module ExecutionEngine
         def self.run(ir_module, ctx, accessors:, registry:)
-          Interpreter.run(ir_module, ctx, accessors: accessors, registry: registry)
+          # Use persistent accessor cache if available, otherwise create temporary one
+          if ctx[:accessor_cache]
+            # Include input data in cache key to avoid cross-context pollution
+            input_key = ctx[:input]&.hash || ctx["input"]&.hash || 0
+            memoized_accessors = add_persistent_memoization(accessors, ctx[:accessor_cache], input_key)
+          else
+            memoized_accessors = add_temporary_memoization(accessors)
+          end
+          
+          Interpreter.run(ir_module, ctx, accessors: memoized_accessors, registry: registry)
+        end
+
+        private
+
+        def self.add_persistent_memoization(accessors, cache, input_key)
+          accessors.map do |plan_id, accessor_fn|
+            [plan_id, lambda do |input_data|
+              cache_key = [plan_id, input_key]
+              cache[cache_key] ||= accessor_fn.call(input_data)
+            end]
+          end.to_h
+        end
+
+        def self.add_temporary_memoization(accessors)
+          cache = {}
+          accessors.map do |plan_id, accessor_fn|
+            [plan_id, lambda do |input_data|
+              cache[plan_id] ||= accessor_fn.call(input_data)
+            end]
+          end.to_h
         end
       end
     end

data/lib/kumi/dev/ir.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Kumi
+  module Dev
+    module IR
+      module_function
+
+      def to_text(ir_module)
+        raise "nil IR" unless ir_module
+
+        lines = []
+        lines << "IR Module"
+        lines << "decls: #{ir_module.decls.size}"
+        
+        ir_module.decls.each_with_index do |decl, i|
+          lines << "decl[#{i}] #{decl.kind}:#{decl.name} shape=#{decl.shape} ops=#{decl.ops.size}"
+          
+          decl.ops.each_with_index do |op, j|
+            # Sort attribute keys for deterministic output
+            sorted_attrs = op.attrs.keys.sort.map { |k| "#{k}=#{format_value(op.attrs[k])}" }.join(" ")
+            args_str = op.args.inspect
+            lines << "  #{j}: #{op.tag} #{sorted_attrs} #{args_str}".rstrip
+          end
+        end
+        
+        lines.join("\n") + "\n"
+      end
+
+      private
+
+      def self.format_value(val)
+        case val
+        when true, false
+          val.to_s
+        when Symbol
+          ":#{val}"
+        when Array
+          val.inspect
+        else
+          val.to_s
+        end
+      end
+
+      def to_json(ir_module, pretty: true)
+        raise "nil IR" unless ir_module
+
+        data = {
+          inputs: ir_module.inputs,
+          decls: ir_module.decls.map do |decl|
+            {
+              name: decl.name,
+              kind: decl.kind,
+              shape: decl.shape,
+              ops: decl.ops.map do |op|
+                {
+                  tag: op.tag,
+                  attrs: op.attrs,
+                  args: op.args
+                }
+              end
+            }
+          end
+        }
+
+        if pretty
+          JSON.pretty_generate(data)
+        else
+          JSON.generate(data)
+        end
+      end
+    end
+  end
+end

data/lib/kumi/dev/parse.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Kumi
+  module Dev
+    module Parse
+      module_function
+
+      def run(schema_path, opts = {})
+        # Load schema via text frontend
+        begin
+          schema, _inputs = Kumi::Frontends::Text.load(path: schema_path)
+        rescue LoadError => e
+          puts "Error: kumi-parser gem not available. Install: gem install kumi-parser"
+          return false
+        rescue StandardError => e
+          puts "Parse error: #{e.message}"
+          return false
+        end
+
+        # Run analyzer
+        runner_opts = opts.slice(:trace, :snap, :snap_dir, :resume_from, :resume_at, :stop_after)
+        res = Dev::Runner.run(schema, runner_opts)
+
+        unless res.ok?
+          puts "Analysis errors:"
+          res.errors.each { |err| puts "  #{err}" }
+          return false
+        end
+
+        unless res.ir
+          puts "Error: No IR generated"
+          return false
+        end
+
+        # Report trace file if enabled
+        if opts[:trace] && res.respond_to?(:trace_file)
+          puts "Trace written to: #{res.trace_file}"
+        end
+
+        # Determine file extension and renderer
+        extension = opts[:json] ? "json" : "txt"
+        golden_path = File.join(File.dirname(schema_path), "expected", "ir.#{extension}")
+
+        # Render IR
+        rendered = if opts[:json]
+          Dev::IR.to_json(res.ir, pretty: true)
+        else
+          Dev::IR.to_text(res.ir)
+        end
+
+        # Handle write mode
+        if opts[:write]
+          FileUtils.mkdir_p(File.dirname(golden_path))
+          File.write(golden_path, rendered)
+          puts "Wrote: #{golden_path}"
+          return true
+        end
+
+        # Handle update mode (write only if different)
+        if opts[:update]
+          if File.exist?(golden_path) && File.read(golden_path) == rendered
+            puts "No changes (#{golden_path})"
+            return true
+          else
+            FileUtils.mkdir_p(File.dirname(golden_path))
+            File.write(golden_path, rendered)
+            puts "Updated: #{golden_path}"
+            return true
+          end
+        end
+
+        # Handle no-diff mode  
+        if opts[:no_diff]
+          puts rendered
+          return true
+        end
+
+        # Default: diff mode (same as write but show diff instead)
+        if File.exist?(golden_path)
+          # Use diff directly with the golden file path
+          require "tempfile"
+          Tempfile.create(["actual", File.extname(golden_path)]) do |actual_file|
+            actual_file.write(rendered)
+            actual_file.flush
+            
+            result = `diff -u --label=expected --label=actual #{golden_path} #{actual_file.path}`
+            if result.empty?
+              puts "No changes (#{golden_path})"
+              return true
+            else
+              puts result.chomp
+              return false
+            end
+          end
+        else
+          # No golden file exists, just print the output
+          puts rendered
+          return true
+        end
+      end
+    end
+  end
+end

data/lib/kumi/dev/runner.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Dev
+    module Runner
+      Result = Struct.new(:state, :ir, :errors, keyword_init: true) do
+        def ok?
+          errors.empty?
+        end
+      end
+
+      module_function
+
+      def run(schema, opts = {})
+        # Set ENV vars for debug/checkpoint based on opts
+        setup_env_vars(opts)
+
+        state = Core::Analyzer::AnalysisState.new
+        errors = []
+
+        begin
+          final_state = Kumi::Analyzer.run_analysis_passes(schema, Kumi::Analyzer::DEFAULT_PASSES, state, errors)
+          ir = final_state[:ir_module]
+          
+          result = Result.new(
+            state: final_state,
+            ir: ir,
+            errors: errors
+          )
+          
+          # Report trace file if enabled
+          if opts[:trace] && defined?(@trace_file) && @trace_file
+            trace_file_path = @trace_file
+            result.define_singleton_method(:trace_file) { trace_file_path }
+          end
+          
+          result
+        rescue StandardError => e
+          # Convert exception to error if not already captured
+          errors << e.message unless errors.include?(e.message)
+          Result.new(
+            state: state,
+            ir: nil,
+            errors: errors
+          )
+        end
+      end
+
+      private
+
+      def self.setup_env_vars(opts)
+        if opts[:trace]
+          ENV["KUMI_DEBUG_STATE"] = "1"
+          trace_file = ENV["KUMI_DEBUG_FILE"] || "tmp/state_trace.jsonl"
+          ENV["KUMI_DEBUG_FILE"] = trace_file
+          
+          # Store for later reporting
+          @trace_file = trace_file
+        end
+
+        if opts[:snap]
+          ENV["KUMI_CHECKPOINT_PHASES"] = opts[:snap]
+        end
+
+        if opts[:snap_dir]
+          ENV["KUMI_CHECKPOINT_DIR"] = opts[:snap_dir]
+        end
+
+        if opts[:resume_from]
+          ENV["KUMI_CHECKPOINT_RESUME_FROM"] = opts[:resume_from]
+        end
+
+        if opts[:resume_at]
+          ENV["KUMI_CHECKPOINT_RESUME_AT"] = opts[:resume_at]
+        end
+
+        if opts[:stop_after]
+          ENV["KUMI_CHECKPOINT_STOP_AFTER"] = opts[:stop_after]
+        end
+      end
+    end
+  end
+end

data/lib/kumi/frontends/ruby.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Frontends
+    module Ruby
+      module_function
+      
+      def load(path:, inputs: {})
+        mod = Module.new
+        mod.extend(Kumi::Schema)
+        mod.module_eval(File.read(path), path)
+        
+        # Extract just the syntax tree AST (same as Text frontend)
+        schema_ast = if mod.const_defined?(:GoldenSchema)
+          golden = mod.const_get(:GoldenSchema)
+          golden.build if golden.respond_to?(:build)
+          golden.__syntax_tree__
+        elsif mod.__syntax_tree__
+          mod.__syntax_tree__
+        else
+          raise "No schema AST found. Make sure the .rb file calls 'schema do...end'"
+        end
+        
+        [schema_ast, inputs]
+      end
+    end
+  end
+end

data/lib/kumi/frontends/text.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Frontends
+    module Text
+      module_function
+      
+      def load(path:, inputs: {})
+        src = File.read(path)
+        
+        begin
+          require "kumi-parser"
+          ast = Kumi::Parser::TextParser.parse(src)
+          Core::Analyzer::Debug.info(:parse, kind: :text, file: path, ok: true) if Core::Analyzer::Debug.enabled?
+          [ast, inputs]
+        rescue LoadError
+          raise "kumi-parser gem not available. Install: gem install kumi-parser"
+        rescue StandardError => e
+          loc = (e.respond_to?(:location) && e.location) || {}
+          line, col = loc.values_at(:line, :column)
+          snippet = self.code_frame(src, line, col)
+          raise StandardError, "#{path}:#{line || '?'}:#{col || '?'}: #{e.message}\n#{snippet}"
+        end
+      end
+      
+      private
+      
+      def self.code_frame(src, line, col, context: 2)
+        return "" unless line
+        
+        lines = src.lines
+        from = [line - 1 - context, 0].max
+        to = [line - 1 + context, lines.length - 1].min
+        out = []
+        
+        (from..to).each do |i|
+          prefix = (i + 1 == line) ? "➤" : " "
+          out << "#{prefix} %4d | %s" % [i + 1, lines[i].rstrip]
+          out << "       | %s^" % (" " * (col - 1)) if i + 1 == line && col
+        end
+        
+        out.join("\n")
+      end
+    end
+  end
+end

data/lib/kumi/frontends.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Frontends
+    def self.load(path:, inputs: {})
+      mode = (ENV["KUMI_PARSER"] || "auto") # auto|text|ruby
+      ext  = File.extname(path)
+
+      # Explicit mode selection
+      return Text.load(path:, inputs:) if mode == "text"
+      return Ruby.load(path:, inputs:) if mode == "ruby"
+      
+      # Auto mode: prefer .kumi if present
+      if mode == "auto" && ext == ".rb"
+        kumi_path = path.sub(/\.rb\z/, ".kumi")
+        if File.exist?(kumi_path)
+          return Text.load(path: kumi_path, inputs: inputs)
+        end
+      end
+      
+      # File extension based selection
+      return Text.load(path:, inputs:) if ext == ".kumi"
+      return Ruby.load(path:, inputs:) if ext == ".rb"
+      
+      # Default fallback
+      Ruby.load(path:, inputs:)
+    end
+  end
+end

data/lib/kumi/kernels/ruby/aggregate_core.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Kernels
+    module Ruby
+      module AggregateCore
+        module_function
+
+        def kumi_sum(enum, skip_nulls: true, min_count: 0)
+          total = 0
+          count = 0
+          enum.each do |x|
+            next if skip_nulls && x.nil?
+
+            total += x
+            count += 1
+          end
+          return nil if count < min_count
+
+          total
+        end
+
+        def kumi_min(enum, skip_nulls: true, min_count: 0)
+          best = nil
+          count = 0
+          enum.each do |x|
+            next if skip_nulls && x.nil?
+
+            best = x if best.nil? || x < best
+            count += 1
+          end
+          return nil if count < min_count
+
+          best
+        end
+
+        def kumi_max(enum, skip_nulls: true, min_count: 0)
+          best = nil
+          count = 0
+          enum.each do |x|
+            next if skip_nulls && x.nil?
+
+            best = x if best.nil? || x > best
+            count += 1
+          end
+          return nil if count < min_count
+
+          best
+        end
+
+        def kumi_mean(enum, skip_nulls: true, min_count: 0)
+          total = 0.0
+          count = 0
+          enum.each do |x|
+            next if skip_nulls && x.nil?
+
+            total += x
+            count += 1
+          end
+          return nil if count < [min_count, 1].max
+
+          total / count
+        end
+
+        def kumi_any(enum, skip_nulls: true, min_count: 0)
+          count = 0
+          enum.each do |x|
+            next if skip_nulls && x.nil?
+
+            return true if x
+            count += 1
+          end
+          return nil if count < min_count
+
+          false
+        end
+
+        def kumi_all(enum, skip_nulls: true, min_count: 0)
+          count = 0
+          enum.each do |x|
+            next if skip_nulls && x.nil?
+
+            return false unless x
+            count += 1
+          end
+          return nil if count < min_count
+
+          true
+        end
+
+        def kumi_count(enum, skip_nulls: true, min_count: 0)
+          count = 0
+          enum.each do |x|
+            next if skip_nulls && x.nil?
+
+            count += 1
+          end
+          return nil if count < min_count
+
+          count
+        end
+      end
+    end
+  end
+end

data/lib/kumi/kernels/ruby/datetime_scalar.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "date"
+
+module Kumi
+  module Kernels
+    module Ruby
+      module DatetimeScalar
+        module_function
+
+        def dt_add_days(d, n)
+          d + n
+        end
+
+        def dt_diff_days(d1, d2)
+          (d1 - d2).to_i
+        end
+      end
+    end
+  end
+end

data/lib/kumi/kernels/ruby/mask_scalar.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Kernels
+    module Ruby
+      module MaskScalar
+        module_function
+
+        def where(condition, if_true, if_false)
+          condition ? if_true : if_false
+        end
+      end
+    end
+  end
+end

data/lib/kumi/kernels/ruby/scalar_core.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Kernels
+    module Ruby
+      module ScalarCore
+        module_function
+
+        def kumi_add(a, b)
+          a + b
+        end
+
+        def kumi_sub(a, b)
+          a - b
+        end
+
+        def kumi_mul(a, b)
+          a * b
+        end
+
+        def kumi_div(a, b)
+          a / b.to_f
+        end
+
+        def kumi_eq(a, b)
+          a == b
+        end
+
+        def kumi_gt(a, b)
+          a > b
+        end
+
+        def kumi_gte(a, b)
+          a >= b
+        end
+
+        def kumi_lt(a, b)
+          a < b
+        end
+
+        def kumi_lte(a, b)
+          a <= b
+        end
+
+        def kumi_ne(a, b)
+          a != b
+        end
+
+        def kumi_and(a, b)
+          a && b
+        end
+
+        def kumi_or(a, b)
+          a || b
+        end
+
+        def kumi_not(a)
+          !a
+        end
+      end
+    end
+  end
+end

data/lib/kumi/kernels/ruby/string_scalar.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Kernels
+    module Ruby
+      module StringScalar
+        module_function
+
+        def str_concat(a, b)
+          "#{a}#{b}"
+        end
+
+        def str_length(s)
+          s&.length
+        end
+      end
+    end
+  end
+end

data/lib/kumi/kernels/ruby/vector_struct.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Kernels
+    module Ruby
+      module VectorStruct
+        module_function
+
+        def size(vec)
+          vec&.size
+        end
+
+        def join_zip(left, right)
+          raise NotImplementedError, "join operations should be implemented in IR/VM"
+        end
+
+        def join_product(left, right)
+          raise NotImplementedError, "join operations should be implemented in IR/VM"
+        end
+
+        def align_to(vec, target_axes)
+          raise NotImplementedError, "align_to should be implemented in IR/VM"
+        end
+
+        def lift(vec, indices)
+          raise NotImplementedError, "lift should be implemented in IR/VM"
+        end
+
+        def flatten(*args)
+          raise NotImplementedError, "flatten should be implemented in IR/VM"
+        end
+
+        def take(values, indices)
+          raise NotImplementedError, "take should be implemented in IR/VM"
+        end
+      end
+    end
+  end
+end