kumi 0.0.18 → 0.0.19

data/lib/kumi/core/ir/execution_engine/profiler.rb

@@ -13,37 +13,37 @@ module Kumi
           class << self
             def enabled? = ENV["KUMI_PROFILE"] == "1"
             def ops_enabled? = ENV.fetch("KUMI_PROFILE_OPS", "1") == "1"
-            def sample_rate = (ENV["KUMI_PROFILE_SAMPLE"]&.to_i || 1)
+            def sample_rate = ENV["KUMI_PROFILE_SAMPLE"]&.to_i || 1
             def persistent? = ENV["KUMI_PROFILE_PERSISTENT"] == "1"
-            
+
             def set_schema_name(name)
               @schema_name = name
-              
+
               # Ensure profiler is initialized in persistent mode
-              unless @initialized
-                @events = []
-                @meta = {}
-                @file = ENV["KUMI_PROFILE_FILE"] || "tmp/profile.jsonl"
-                @run_id ||= 1
-                @op_seq ||= 0
-                @aggregated_stats ||= Hash.new { |h, k| h[k] = { count: 0, total_ms: 0.0, total_cpu_ms: 0.0, rows: 0, runs: Set.new } }
-                
-                # Truncate file if needed
-                if ENV["KUMI_PROFILE_TRUNCATE"] == "1" && !@persistent_initialized
-                  FileUtils.mkdir_p(File.dirname(@file))
-                  File.write(@file, "")
-                  @aggregated_stats.clear
-                  @persistent_initialized = true
-                end
-                
-                @initialized = true
+              return if @initialized
+
+              @events = []
+              @meta = {}
+              @file = ENV["KUMI_PROFILE_FILE"] || "tmp/profile.jsonl"
+              @run_id ||= 1
+              @op_seq ||= 0
+              @aggregated_stats ||= Hash.new { |h, k| h[k] = { count: 0, total_ms: 0.0, total_cpu_ms: 0.0, rows: 0, runs: Set.new } }
+
+              # Truncate file if needed
+              if ENV["KUMI_PROFILE_TRUNCATE"] == "1" && !@persistent_initialized
+                FileUtils.mkdir_p(File.dirname(@file))
+                File.write(@file, "")
+                @aggregated_stats.clear
+                @persistent_initialized = true
               end
+
+              @initialized = true
             end
 
             def reset!(meta: {})
               set_schema_name(meta[:schema_name]) if meta[:schema_name]
               return unless enabled?
-              
+
               # In persistent mode, don't reset aggregated stats or increment run_id
               # This allows profiling across multiple schema creations
               if persistent?
@@ -54,7 +54,7 @@ module Kumi
                 @run_id ||= 1
                 @op_seq ||= 0
                 @aggregated_stats ||= Hash.new { |h, k| h[k] = { count: 0, total_ms: 0.0, total_cpu_ms: 0.0, rows: 0, runs: Set.new } }
-                
+
                 # Only truncate on very first reset in persistent mode
                 if ENV["KUMI_PROFILE_TRUNCATE"] == "1" && !@persistent_initialized
                   FileUtils.mkdir_p(File.dirname(@file))
@@ -70,8 +70,10 @@ module Kumi
                 @file   = ENV["KUMI_PROFILE_FILE"] || "tmp/profile.jsonl"
                 @run_id = (@run_id || 0) + 1
                 @op_seq = 0
-                @aggregated_stats = (@aggregated_stats || Hash.new { |h, k| h[k] = { count: 0, total_ms: 0.0, total_cpu_ms: 0.0, rows: 0, runs: Set.new } })
-                
+                @aggregated_stats ||= Hash.new do |h, k|
+                  h[k] = { count: 0, total_ms: 0.0, total_cpu_ms: 0.0, rows: 0, runs: Set.new }
+                end
+
                 if ENV["KUMI_PROFILE_TRUNCATE"] == "1"
                   FileUtils.mkdir_p(File.dirname(@file))
                   File.write(@file, "")
@@ -84,7 +86,7 @@ module Kumi
             def t0
               Process.clock_gettime(Process::CLOCK_MONOTONIC)
             end
-            
+
             # CPU time start (process + thread)
             def cpu_t0
               Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID)
@@ -93,83 +95,87 @@ module Kumi
             # Phase timing for coarse-grained operations
             def phase(name, tags = {})
               return yield unless enabled?
-              p0 = t0; c0 = cpu_t0
+
+              p0 = t0
+              c0 = cpu_t0
               result = yield
               wall_ms = (t0 - p0) * 1000.0
               cpu_ms = (cpu_t0 - c0) * 1000.0
               stream({
-                ts: Time.now.utc.iso8601(3),
-                kind: "phase",
-                name: name,
-                wall_ms: wall_ms.round(3),
-                cpu_ms: cpu_ms.round(3),
-                tags: tags,
-                run: @run_id
-              })
+                       ts: Time.now.utc.iso8601(3),
+                       kind: "phase",
+                       name: name,
+                       wall_ms: wall_ms.round(3),
+                       cpu_ms: cpu_ms.round(3),
+                       tags: tags,
+                       run: @run_id
+                     })
               result
             end
 
             # Memory snapshot with GC statistics
             def memory_snapshot(label, extra: {})
               return unless enabled?
+
               s = GC.stat
               stream({
-                ts: Time.now.utc.iso8601(3),
-                kind: "mem",
-                label: label,
-                heap_live: s[:heap_live_slots],
-                old_objects: s[:old_objects],
-                minor_gc: s[:minor_gc_count],
-                major_gc: s[:major_gc_count],
-                rss_mb: read_rss_mb,
-                run: @run_id,
-                **extra
-              })
+                       ts: Time.now.utc.iso8601(3),
+                       kind: "mem",
+                       label: label,
+                       heap_live: s[:heap_live_slots],
+                       old_objects: s[:old_objects],
+                       minor_gc: s[:minor_gc_count],
+                       major_gc: s[:major_gc_count],
+                       rss_mb: read_rss_mb,
+                       run: @run_id,
+                       **extra
+                     })
             end
 
             def read_rss_mb
-              ((File.read("/proc/#{$$}/status")[/VmRSS:\s+(\d+)\skB/, 1].to_i) / 1024.0).round(2)
-            rescue
+              (File.read("/proc/#{$$}/status")[/VmRSS:\s+(\d+)\skB/, 1].to_i / 1024.0).round(2)
+            rescue StandardError
               nil
             end
 
             # Per-op record with both wall time and CPU time (with sampling support)
             def record!(decl:, idx:, tag:, op:, t0:, cpu_t0: nil, rows: nil, note: nil)
               return unless enabled? && ops_enabled?
+
               @op_seq += 1
               return unless sample_rate <= 1 || (@op_seq % sample_rate).zero?
-              
+
               wall_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0) * 1000.0)
               cpu_ms = cpu_t0 ? ((Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID) - cpu_t0) * 1000.0) : wall_ms
-              
+
               ev = {
-                ts:     Time.now.utc.iso8601(3),
-                run:    @run_id,
-                schema: @schema_name,  # schema identifier for multi-schema differentiation
-                decl:   decl,     # decl name (string/symbol)
-                i:      idx,      # op index
-                tag:    tag,      # op tag (symbol)
+                ts: Time.now.utc.iso8601(3),
+                run: @run_id,
+                schema: @schema_name, # schema identifier for multi-schema differentiation
+                decl: decl, # decl name (string/symbol)
+                i: idx, # op index
+                tag: tag, # op tag (symbol)
                 wall_ms: wall_ms.round(4),
-                cpu_ms:  cpu_ms.round(4),
-                rows:   rows,
-                note:   note,
-                key:    op_key(decl, idx, tag, op),      # stable key for grep/diff
-                attrs:  compact_attrs(op.attrs)
+                cpu_ms: cpu_ms.round(4),
+                rows: rows,
+                note: note,
+                key: op_key(decl, idx, tag, op), # stable key for grep/diff
+                attrs: compact_attrs(op.attrs)
               }
-              
+
               # Aggregate stats for multi-run averaging
               op_key = "#{decl}@#{idx}:#{tag}"
               agg = @aggregated_stats[op_key]
               agg[:count] += 1
               agg[:total_ms] += wall_ms
               agg[:total_cpu_ms] += cpu_ms
-              agg[:rows] += (rows || 0)
+              agg[:rows] += rows || 0
               agg[:runs] << @run_id
               agg[:decl] = decl
               agg[:tag] = tag
               agg[:idx] = idx
               agg[:note] = note if note
-              
+
               (@events ||= []) << ev
               stream(ev) if ENV["KUMI_PROFILE_STREAM"] == "1"
               ev
@@ -177,43 +183,43 @@ module Kumi
 
             def summary(top: 20)
               return {} unless enabled?
-              
+
               # Current run summary (legacy format)
               current_agg = Hash.new { |h, k| h[k] = { count: 0, ms: 0.0, rows: 0 } }
               (@events || []).each do |e|
                 k = [e[:decl], e[:tag]]
                 a = current_agg[k]
                 a[:count] += 1
-                a[:ms]    += (e[:wall_ms] || e[:ms] || 0)
-                a[:rows]  += (e[:rows] || 0)
+                a[:ms]    += e[:wall_ms] || e[:ms] || 0
+                a[:rows]  += e[:rows] || 0
               end
-              current_ranked = current_agg.map { |(decl, tag), v|
+              current_ranked = current_agg.map do |(decl, tag), v|
                 { decl: decl, tag: tag, count: v[:count], ms: v[:ms].round(3), rows: v[:rows],
                   rps: v[:rows] > 0 ? (v[:rows] / v[:ms]).round(1) : nil }
-              }.sort_by { |h| -h[:ms] }.first(top)
+              end.sort_by { |h| -h[:ms] }.first(top)
 
               { meta: @meta || {}, top: current_ranked,
-                total_ms: ((@events || []).sum { |e| e[:wall_ms] || e[:ms] || 0 }).round(3),
+                total_ms: (@events || []).sum { |e| e[:wall_ms] || e[:ms] || 0 }.round(3),
                 op_count: (@events || []).size,
                 run_id: @run_id }
             end
-            
+
             # Multi-run averaged analysis
             def averaged_analysis(top: 20)
               return {} unless enabled? && @aggregated_stats&.any?
-              
+
               # Convert aggregated stats to averaged metrics
               averaged = @aggregated_stats.map do |op_key, stats|
                 num_runs = stats[:runs].size
                 avg_wall_ms = stats[:total_ms] / stats[:count]
-                avg_cpu_ms = stats[:total_cpu_ms] / stats[:count] 
+                avg_cpu_ms = stats[:total_cpu_ms] / stats[:count]
                 total_wall_ms = stats[:total_ms]
                 total_cpu_ms = stats[:total_cpu_ms]
-                
+
                 {
                   op_key: op_key,
                   decl: stats[:decl],
-                  idx: stats[:idx], 
+                  idx: stats[:idx],
                   tag: stats[:tag],
                   runs: num_runs,
                   total_calls: stats[:count],
@@ -227,28 +233,28 @@ module Kumi
                   note: stats[:note]
                 }
               end.sort_by { |s| -s[:total_wall_ms] }.first(top)
-              
+
               {
                 meta: @meta || {},
-                total_runs: (@aggregated_stats.values.map { |s| s[:runs].size }.max || 0),
+                total_runs: @aggregated_stats.values.map { |s| s[:runs].size }.max || 0,
                 averaged_ops: averaged,
                 total_operations: @aggregated_stats.size
               }
             end
-            
+
             # Identify potential cache overhead operations
             def cache_overhead_analysis
               return {} unless enabled? && @aggregated_stats&.any?
-              
+
               # Look for operations that might be cache-related
               cache_ops = @aggregated_stats.select do |op_key, stats|
                 op_key.include?("ref") || op_key.include?("load_input") || stats[:note]&.include?("cache")
               end
-              
+
               cache_analysis = cache_ops.map do |op_key, stats|
-                num_runs = stats[:runs].size
+                stats[:runs].size
                 avg_wall_ms = stats[:total_ms] / stats[:count]
-                
+
                 {
                   op_key: op_key,
                   decl: stats[:decl],
@@ -259,7 +265,7 @@ module Kumi
                   overhead_per_call: avg_wall_ms.round(6)
                 }
               end.sort_by { |s| -s[:total_time_ms] }
-              
+
               {
                 cache_operations: cache_analysis,
                 total_cache_time: cache_analysis.sum { |op| op[:total_time_ms] }.round(3)
@@ -268,47 +274,50 @@ module Kumi
 
             def emit_summary!
               return unless enabled?
+
               stream({ ts: Time.now.utc.iso8601(3), kind: "summary", data: summary })
             end
 
             def init_persistent!
               return unless enabled? && persistent?
+
               @persistent_initialized = false
               reset!
             end
 
             def finalize!
               return unless enabled?
-              
+
               # Emit final aggregated summary
               if @aggregated_stats&.any?
                 stream({
-                  ts: Time.now.utc.iso8601(3),
-                  kind: "final_summary",
-                  data: averaged_analysis
-                })
+                         ts: Time.now.utc.iso8601(3),
+                         kind: "final_summary",
+                         data: averaged_analysis
+                       })
               end
-              
+
               # Emit cache analysis if available
               cache_analysis = cache_overhead_analysis
-              if cache_analysis[:cache_operations]&.any?
-                stream({
-                  ts: Time.now.utc.iso8601(3),
-                  kind: "cache_analysis", 
-                  data: cache_analysis
-                })
-              end
+              return unless cache_analysis[:cache_operations]&.any?
+
+              stream({
+                       ts: Time.now.utc.iso8601(3),
+                       kind: "cache_analysis",
+                       data: cache_analysis
+                     })
             end
 
             # Stable textual key for "match ops one by one"
             def op_key(decl, idx, tag, op)
               attrs = compact_attrs(op.attrs)
               args  = op.args
-              "#{decl}@#{idx}:#{tag}|#{attrs.keys.sort_by(&:to_s).map { |k| "#{k}=#{attrs[k].inspect}" }.join(",")}|args=#{args.inspect}"
+              "#{decl}@#{idx}:#{tag}|#{attrs.keys.sort_by(&:to_s).map { |k| "#{k}=#{attrs[k].inspect}" }.join(',')}|args=#{args.inspect}"
             end
 
             def compact_attrs(h)
               return {} unless h
+
               h.transform_values do |v|
                 case v
                 when Array, Hash, Symbol, String, Numeric, TrueClass, FalseClass, NilClass then v
@@ -319,6 +328,7 @@ module Kumi
 
             def stream(obj)
               return unless @file
+
               FileUtils.mkdir_p(File.dirname(@file))
               File.open(@file, "a") { |f| f.puts(obj.to_json) }
             end
@@ -327,4 +337,4 @@ module Kumi
       end
     end
   end
-end
+end

data/lib/kumi/core/ir.rb

@@ -27,7 +27,11 @@ module Kumi
 
     module IR::Ops
       def self.Const(v)                      = IR::Op.new(tag: :const, attrs: { value: v }, args: [])
-      def self.LoadInput(plan_id, scope: [], is_scalar: false, has_idx: false) = IR::Op.new(tag: :load_input, attrs: { plan_id: plan_id, scope: scope, is_scalar: is_scalar, has_idx: has_idx }, args: [])
+
+      def self.LoadInput(plan_id, scope: [], is_scalar: false,
+                         has_idx: false) = IR::Op.new(tag: :load_input,
+                                                      attrs: { plan_id: plan_id, scope: scope, is_scalar: is_scalar, has_idx: has_idx }, args: [])
+
       def self.Ref(name)                     = IR::Op.new(tag: :ref, attrs: { name: name }, args: [])
       def self.Map(fn, argc, *slots)         = IR::Op.new(tag: :map, attrs: { fn: fn, argc: argc }, args: slots)
       def self.Array(count, *slots)          = IR::Op.new(tag: :array, attrs: { count: count }, args: slots)
@@ -38,7 +42,7 @@ module Kumi
       def self.Store(name, slot)             = IR::Op.new(tag: :store, attrs: { name: name }, args: [slot])
       def self.Lift(to_scope, slot)          = IR::Op.new(tag: :lift, attrs: { to_scope: to_scope }, args: [slot])
       def self.Join(*slots)                  = IR::Op.new(tag: :join, attrs: {}, args: slots)
-      
+
       # Up-sample `source` to the scope (and order) of `target` by index-prefix.
       # Policies: :error | :nil for missing; require_unique: true enforces 1:1 on prefix.
       def self.AlignTo(target_slot, source_slot, to_scope:, on_missing: :error, require_unique: true)

data/lib/kumi/core/irv2/builder.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Kumi
+  module Core
+    module IRV2
+      class Builder
+        attr_reader :values, :exports
+
+        def initialize
+          @next_id = 0
+          @values  = []
+          @exports  = []
+        end
+
+        def load_input(path, stamp: nil) = emit(:LoadInput, [path], {}, stamp: stamp)
+        def load_declaration(name, stamp: nil) = emit(:LoadDeclaration, [name], {}, stamp: stamp)
+        def load_decl(name, stamp: nil)      = emit(:LoadDecl, [name], {}, stamp: stamp)
+        def const(lit, stamp: nil)           = emit(:Const, [lit], {}, stamp: stamp)
+        def align_to(v, target_axes, stamp: nil) = emit(:AlignTo, [v], { target_axes: Array(target_axes).map(&:to_s) }, stamp: stamp)
+        def select(cond, then_v, else_v, stamp: nil) = emit(:Select, [cond, then_v, else_v], {}, stamp: stamp)
+        def map(func, *values, stamp: nil)       = emit(:Map, values, { fn: func }, stamp: stamp)
+        def reduce(func, val, axis, stamp: nil)  = emit(:Reduce, [val], { fn: func, axis: axis.to_s }, stamp: stamp)
+        def construct_tuple(*vs, stamp: nil) = emit(:ConstructTuple, vs, {}, stamp: stamp)
+        def tuple_get(v, index, stamp: nil) = emit(:TupleGet, [v], { index: index }, stamp: stamp)
+
+        def store(name, v)
+          (@exports << [name, v]
+           nil)
+        end
+
+        def dump
+          [*values.map(&:to_s), *exports.map { |(n, v)| "Store #{n}, %#{v.id}" }].join("\n")
+        end
+
+        private
+
+        def emit(op, args, attrs, stamp: nil, elem_stamps: nil)
+          v = Value.new(@next_id, op, args, attrs, stamp: stamp, elem_stamps: elem_stamps)
+          @values << v
+          @next_id += 1
+          v
+        end
+      end
+    end
+  end
+end

data/lib/kumi/core/irv2/declaration.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Kumi
+  module Core
+    module IRV2
+      class Declaration
+        attr_reader :name, :operations, :result, :parameters
+
+        def initialize(name, operations, result, parameters = [])
+          @name = name
+          @operations = operations
+          @result = result
+          @parameters = parameters
+        end
+
+        def inputs
+          @parameters.select { |p| p[:type] == :input }
+        end
+
+        def dependencies
+          @parameters.select { |p| p[:type] == :dependency }.map { |p| p[:source] }
+        end
+      end
+    end
+  end
+end

data/lib/kumi/core/irv2/module.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Kumi
+  module Core
+    module IRV2
+      class Module
+        attr_reader :declarations, :metadata
+
+        def initialize(declarations, metadata = {})
+          @declarations = declarations
+          @metadata = metadata
+        end
+
+        def to_s
+          format_declaration_based
+        end
+
+        private
+
+        def format_declaration_based
+          output = []
+          output << "; — Module: Declaration-Based IR"
+          output << ""
+
+          @declarations.each do |name, decl|
+            output << "Declaration #{name} {"
+
+            unless decl.parameters.empty?
+              output << "  params:"
+              decl.parameters.each do |param|
+                case param[:type]
+                when :input
+                  output << "    #{param[:name]} : View(#{param[:dtype]}, axes=#{param[:axes]})"
+                when :dependency
+                  output << "    #{param[:name]} : View(#{param[:dtype]}, axes=#{param[:axes]})  ; #{param[:source]}"
+                end
+              end
+            end
+
+            output << "  operations: ["
+            decl.operations.each do |op|
+              comment = format_operation_comment(op)
+              op_str = format_operation(op)
+              padding = [50 - op_str.length, 1].max
+              output << "    #{op_str}#{' ' * padding}; #{comment}"
+            end
+            output << "  ]"
+            output << "  result: %#{decl.result.id}"
+            output << "}"
+            output << ""
+          end
+
+          # Add canonical inputs if available
+          if @metadata.dig("analysis", "inputs")
+            output << "; — Canonical Inputs"
+            output << ""
+            @metadata["analysis"]["inputs"].each do |input|
+              path_str = input["path"].join(".")
+              scope_str = input["scope"].empty? ? "[]" : "[#{input['scope'].join(', ')}]"
+              output << "#{path_str}: #{input['dtype']} (scope=#{scope_str})"
+            end
+            output << ""
+          end
+
+          output.join("\n")
+        end
+
+        def format_operation(val)
+          case val.op
+          when :LoadInput
+            "%#{val.id} = LoadInput #{val.args.first.inspect}"
+          when :LoadDeclaration
+            "%#{val.id} = LoadDeclaration #{val.args.first.inspect}"
+          when :LoadDecl
+            "%#{val.id} = LoadDecl #{val.args.first.inspect}"
+          when :Map
+            args_str = val.args.map { |a| "%#{a.id}" }.join(", ")
+            op_name = val.attrs[:fn] || "unknown"
+            "%#{val.id} = Map(#{op_name}, #{args_str})"
+          when :Reduce
+            op_name = val.attrs[:fn] || "unknown"
+            last_axis = val.attrs[:axis]
+            "%#{val.id} = Reduce(#{op_name}, %#{val.args.first.id}, #{last_axis.inspect})"
+          when :AlignTo
+            axes_str = val.attrs[:target_axes].map(&:inspect).join(",")
+            "%#{val.id} = AlignTo(%#{val.args.first.id}, [#{axes_str}])"
+          when :ConstructTuple
+            args_str = val.args.map { |a| "%#{a.id}" }.join(", ")
+            "%#{val.id} = ConstructTuple(#{args_str})"
+          else
+            val.to_s
+          end
+        end
+
+        def format_operation_comment(val)
+          # Extract dimensional information from metadata if available
+          scope = metadata.dig(:operation_scopes, val.id) || []
+          dtype = metadata.dig(:operation_types, val.id) || :unknown
+
+          scope_str = scope.empty? ? "[]" : "[#{scope.map(&:inspect).join(',')}]"
+          "#{scope_str}, #{dtype}"
+        end
+      end
+    end
+  end
+end

data/lib/kumi/core/irv2/value.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Kumi
+  module Core
+    module IRV2
+      class Value
+        attr_reader :id, :op, :args, :attrs, :stamp, :elem_stamps
+
+        def initialize(id, op, args, attrs, stamp: nil, elem_stamps: nil)
+          @id = id
+          @op = op
+          @args = args
+          @attrs = attrs
+          @stamp = stamp
+          @elem_stamps = elem_stamps
+        end
+
+        def to_s
+          a = args.map { |x| x.is_a?(Value) ? "%#{x.id}" : x.inspect }.join(", ")
+          attrs_s = attrs.empty? ? "" : " " + attrs.map { |k, v| "#{k}: #{v.inspect}" }.join(", ")
+          format("%%%d = %s(%s)%s", id, op, a, attrs_s)
+        end
+      end
+    end
+  end
+end