kumi 0.0.18 → 0.0.20

data/lib/kumi/configuration.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+
+module Kumi
+  # Holds the configuration state for the Kumi compiler and runtime.
+  # This object is yielded to the user in the `Kumi.configure` block.
+  class Configuration
+    # The directory where compiled schemas are stored as cached Ruby files.
+    # On file-based systems, this is crucial for performance.
+    attr_accessor :cache_path
+
+    # The compilation strategy.
+    #   :jit (Just-in-Time): Compiles schemas on-the-fly at boot time if the
+    #     source has changed. Ideal for development.
+    #   :aot (Ahead-of-Time): Expects schemas to be precompiled via a build
+    #     task. Raises an error at runtime if a compiled file is missing.
+    #     Ideal for production and test environments.
+    attr_accessor :compilation_mode
+
+    # A master switch to bypass the cache and force recompilation on every run.
+    # Useful for debugging the compiler itself.
+    attr_accessor :force_recompile
+
+    def initialize
+      # Set smart, environment-aware defaults.
+      @cache_path = default_cache_path
+      @compilation_mode = default_compilation_mode
+      @force_recompile = false
+    end
+
+    private
+
+    # Provides a sensible default cache path.
+    # It prefers the Rails cache directory if available, otherwise uses the
+    # system's temporary directory.
+    def default_cache_path
+      if defined?(Rails) && Rails.root
+        Rails.root.join("tmp", "cache", "kumi")
+      else
+        File.join(Dir.tmpdir, "kumi_cache")
+      end
+    end
+
+    # Determines the best compilation mode based on the environment.
+    # The convention is to use JIT for development for a seamless "save and
+    # refresh" experience, and AOT for production to ensure fast boots
+    # and catch precompilation errors in CI.
+    def default_compilation_mode
+      # RACK_ENV is a common standard, but Rails.env is more specific.
+      env = defined?(Rails) ? Rails.env.to_s : ENV.fetch("RACK_ENV", nil)
+
+      if env == "development"
+        :jit
+      else
+        :aot
+      end
+    end
+  end
+end

data/lib/kumi/core/analyzer/binder.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require "json"
+require "digest"
+
+module Kumi
+  module Core
+    module Analyzer
+      module Binder
+        OPS_WITH_KERNELS = %i[KernelCall Accumulate Fold].freeze
+
+        module_function
+
+        def bind(lir_decls, registry, target:)
+          bindings = []
+          target_sym = target.to_sym
+
+          lir_decls.each do |decl_name, decl|
+            Array(decl[:operations]).each do |op|
+              opcode = op.opcode
+              next unless OPS_WITH_KERNELS.include?(opcode)
+
+              fn_name = op.attributes[:fn]
+              raise "KernelCall at #{op.location} is missing :fn attribute" unless fn_name
+
+              function = registry.function(fn_name)
+              fn_id = function.id
+
+              kernel = registry.kernel_for(fn_id, target: target_sym)
+              id = kernel.id
+              impl = kernel.impl
+              inline = kernel.inline
+              fold_inline = kernel.fold_inline
+              dtype = op.stamp&.dig("dtype")
+
+              attrs = {}
+
+              attrs["inline"] = inline if inline
+              attrs["fold_inline"] = fold_inline if fold_inline
+
+              # TODO: Make this logic clear, we are mixing reducers functions that works over scalars collections and eachwise, this is confusing.
+              if function.reduce? && opcode != :Fold
+                case function.reduction_strategy
+                when :identity
+
+                  identity = registry.kernel_identity_for(fn_id, dtype: dtype, target: target_sym)
+                  attrs["identity"] = identity if identity
+                when :first_element
+                  attrs["first_element"] = true
+                end
+              end
+
+              fname = fn_id.split(".").join("_")
+
+              bindings << {
+                "decl" => decl_name.to_s,
+                "op_result_reg" => op.result_register,
+                "fn" => fn_name, # Original function name/alias
+                "fn_id" => fn_id, # Resolved canonical function ID
+                "fname" => fname, # Sanitized function name for use in Ruby method names
+                "id" => id,
+                "impl" => impl,
+                "attrs" => attrs
+              }
+            end
+          end
+
+          deduplicated = deduplicate_kernel_impls(bindings)
+
+          {
+            "lir_ref" => sha256_lir_ref(lir_decls),
+            "target" => target.to_s,
+            "registry_ref" => registry.registry_ref,
+            "bindings" => deduplicated[:bindings],
+            "kernels" => deduplicated[:kernels]
+          }
+        end
+
+        def deduplicate_kernel_impls(bindings)
+          kernels = {}
+          deduplicated_bindings = []
+
+          bindings.each do |binding|
+            id = binding["id"]
+            impl = binding["impl"]
+            attrs = binding["attrs"]
+
+            if kernels.key?(id)
+              raise "Inconsistent impl for #{id}" unless kernels[id]["impl"] == impl
+
+              kernels[id]["attrs"].merge!(attrs)
+            else
+              kernels[id] = {
+                "id" => id,
+                "fn_id" => binding["fn_id"],
+                "impl" => impl,
+                "attrs" => attrs
+              }
+            end
+
+            deduplicated_bindings << binding.reject { |k, _| %w[impl attrs].include?(k) }
+          end
+
+          {
+            bindings: deduplicated_bindings,
+            kernels: kernels.values
+          }
+        end
+
+        def sha256_lir_ref(lir_decls)
+          canonical_decls = lir_decls.transform_values do |decl|
+            decl[:operations].map(&:to_h)
+          end
+
+          json = JSON.generate(canonical_decls, { ascii_only: true, array_nl: "", object_nl: "", indent: "", space: "", space_before: "" })
+          "sha256:#{Digest::SHA256.hexdigest(json)}"
+        end
+      end
+    end
+  end
+end

data/lib/kumi/core/analyzer/checkpoint.rb

@@ -9,6 +9,8 @@ module Kumi
         class << self
           # ===== Interface shape matches Debug =====
           def enabled?
+            return false if ENV["KUMI_CHECKPOINT"] == "0"
+
             ENV["KUMI_CHECKPOINT"] == "1" ||
               !resume_from.nil? || !resume_at.nil? || !stop_after.nil?
           end
@@ -17,26 +19,32 @@ module Kumi
           def dir         = ENV["KUMI_CHECKPOINT_DIR"]    || "tmp/analysis_snapshots"
           def phases      = (ENV["KUMI_CHECKPOINT_PHASE"] || "before,after").split(",").map! { _1.strip.downcase.to_sym }
           def formats     = (ENV["KUMI_CHECKPOINT_FORMAT"] || "marshal").split(",").map! { _1.strip.downcase } # marshal|json|both
-          def resume_from = ENV["KUMI_RESUME_FROM"] # file path (.msh or .json)
-          def resume_at   = ENV["KUMI_RESUME_AT"]   # pass short name
-          def stop_after  = ENV["KUMI_STOP_AFTER"]  # pass short name
+          def resume_from = ENV.fetch("KUMI_RESUME_FROM", nil) # file path (.msh or .json)
+          def resume_at   = ENV.fetch("KUMI_RESUME_AT", nil)   # pass short name
+          def stop_after  = ENV.fetch("KUMI_STOP_AFTER", nil)  # pass short name
 
           # ===== Lifecycle (called by analyzer) =====
           def load_initial_state(default_state)
             path = resume_from
             return default_state unless path && File.exist?(path)
+
             data = File.binread(path)
-            path.end_with?(".msh") ? StateSerde.load_marshal(data)
-                                   : StateSerde.load_json(data)
+            if path.end_with?(".msh")
+              StateSerde.load_marshal(data)
+            else
+              StateSerde.load_json(data)
+            end
           end
 
           def entering(pass_name:, idx:, state:)
             return unless enabled?
+
             snapshot(pass_name:, idx:, phase: :before, state:) if phases.include?(:before)
           end
 
           def leaving(pass_name:, idx:, state:)
             return unless enabled?
+
             snapshot(pass_name:, idx:, phase: :after, state:) if phases.include?(:after)
           end
 
@@ -59,9 +67,7 @@ module Kumi
             end
 
             # Fold checkpoint info into the same per-pass logs the Debugger uses.
-            if Core::Analyzer::Debug.enabled?
-              Core::Analyzer::Debug.info(:checkpoint, phase:, idx:, files:)
-            end
+            Core::Analyzer::Debug.info(:checkpoint, phase:, idx:, files:) if Core::Analyzer::Debug.enabled?
 
             files
           end
@@ -69,4 +75,4 @@ module Kumi
       end
     end
   end
-end
+end

data/lib/kumi/core/analyzer/constant_evaluator.rb

@@ -4,54 +4,49 @@ module Kumi
   module Core
     module Analyzer
       class ConstantEvaluator
-        include Syntax
+        NAST = Kumi::Core::NAST
 
-        def initialize(definitions)
-          @definitions = definitions
-          @memo = {}
+        def self.evaluate(call_node, registry, known_constants = {})
+          new(call_node, registry, known_constants).evaluate
         end
 
-        OPERATORS = {
-          add: :+,
-          subtract: :-,
-          multiply: :*,
-          divide: :/
-        }.freeze
-
-        def evaluate(node, visited = Set.new)
-          return :unknown unless node
-          return @memo[node] if @memo.key?(node)
-          return node.value if node.is_a?(Literal)
-
-          result = case node
-                   when DeclarationReference then evaluate_binding(node, visited)
-                   when CallExpression then evaluate_call_expression(node, visited)
-                   else :unknown
-                   end
-
-          @memo[node] = result unless result == :unknown
-          result
+        def initialize(call_node, registry, known_constants)
+          @node = call_node
+          @registry = registry
+          @known_constants = known_constants
         end
 
-        private
+        def evaluate
+          arg_values = @node.args.map { |arg| resolve_constant_value(arg) }
 
-        def evaluate_binding(node, visited)
-          return :unknown if visited.include?(node.name)
+          return nil if arg_values.any?(&:nil?)
 
-          visited << node.name
-          definition = @definitions[node.name]
-          return :unknown unless definition
+          func = @registry.function(@node.fn)
 
-          evaluate(definition.expression, visited)
-        end
+          return nil unless func.respond_to?(:folding_class_method) && func.folding_class_method
 
-        def evaluate_call_expression(node, visited)
-          return :unknown unless OPERATORS.key?(node.fn_name)
+          method_name = func.folding_class_method.to_sym
 
-          args = node.args.map { |arg| evaluate(arg, visited) }
-          return :unknown if args.any?(:unknown)
+          ConstantFoldingHelpers.send(method_name, *arg_values)
+        rescue StandardError
+          nil
+        end
+
+        private
 
-          args.reduce(OPERATORS[node.fn_name])
+        def resolve_constant_value(node)
+          case node
+          when NAST::Const
+            node.value
+          when NAST::Ref
+            const_node = @known_constants[node.name]
+            resolve_constant_value(const_node) if const_node
+          when NAST::Tuple
+            values = node.args.map { |arg| resolve_constant_value(arg) }
+            values.all? ? values : nil
+          else
+            nil
+          end
         end
       end
     end

data/lib/kumi/core/analyzer/constant_folding_helpers.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Core
+    module Analyzer
+      # This module provides pure Ruby implementations of primitive functions
+      # specifically for the ConstantEvaluator to use at compile time.
+      # This approach avoids all use of `eval` and is fast and secure.
+      # It is references by the functions data yamls, as folding_method
+      module ConstantFoldingHelpers
+        module_function
+
+        # :agg :booleans
+        def all?(collection) = collection.all?
+        def any?(collection) = collection.any?
+
+        # :agg :numeric
+        def sum(collection) = collection.sum
+        def count(collection) = collection.size
+        def avg(collection) = collection.sum.to_f / collection.size
+        def min(collection) = collection.min
+        def max(collection) = collection.max
+
+        # :core :numeric
+        def add(a, b) = a + b
+        def sub(a, b) = a - b
+        def mul(a, b) = a * b
+        def div(a, b) = a.to_f / b
+        def mod(a, b) = a % b
+        def at(collection, index) = collection[index]
+
+        # :core :booleans
+        def and(a, b) = a && b
+        def or(a, b)  = a || b
+        def not?(a) = !a
+
+        # :core :comparisons
+        def lt?(a, b) = a < b
+        def gt?(a, b) = a > b
+        def gte?(a, b) = a >= b
+        def eq?(a, b)  = a == b
+        def neq?(a, b) = a != b
+
+        # :core :constructor
+        def length(collection) = collection.size
+        def tuple(*args) = args
+
+        # :core :select
+        def select(condition_mask, value_when_true, value_when_false)
+          condition_mask ? value_when_true : value_when_false
+        end
+      end
+    end
+  end
+end

data/lib/kumi/core/analyzer/debug.rb

@@ -8,14 +8,14 @@ module Kumi
     module Analyzer
       module Debug
         KEY = :kumi_debug_log
-        
+
         class << self
           def enabled?
             ENV["KUMI_DEBUG_STATE"] == "1"
           end
 
           def output_path
-            ENV["KUMI_DEBUG_OUTPUT_PATH"]
+            ENV.fetch("KUMI_DEBUG_OUTPUT_PATH", nil)
           end
 
           def max_depth
@@ -65,21 +65,20 @@ module Kumi
 
           def trace(id, **start_fields)
             t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-            info("#{id}_start".to_sym, **start_fields)
-            yield.tap do |ret|
+            info(:"#{id}_start", **start_fields)
+            yield.tap do |_ret|
               dt = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0) * 1000).round(2)
-              info("#{id}_finish".to_sym, ms: dt)
-              ret
+              info(:"#{id}_finish", ms: dt)
             end
-          rescue => e
-            log(level: :error, id: "#{id}_error".to_sym, error: e.class.name, message: e.message)
+          rescue StandardError => e
+            log(level: :error, id: :"#{id}_error", error: e.class.name, message: e.message)
             raise
           end
 
           # State diffing
           def diff_state(before, after)
             changes = {}
-            
+
             all_keys = (before.keys + after.keys).uniq
             all_keys.each do |key|
               if !before.key?(key)
@@ -94,7 +93,7 @@ module Kumi
                 }
               end
             end
-            
+
             changes
           end
 
@@ -107,7 +106,7 @@ module Kumi
               diff: diff,
               logs: logs
             }
-            
+
             if output_path && !output_path.empty?
               File.open(output_path, "a") { |f| f.puts(JSON.dump(payload)) }
             else
@@ -120,7 +119,7 @@ module Kumi
 
           def truncate(value, depth = max_depth)
             return value if depth <= 0
-            
+
             case value
             when Hash
               if value.size > max_items
@@ -144,7 +143,6 @@ module Kumi
               value
             end
           end
-
         end
 
         # Mixin for passes
@@ -157,11 +155,11 @@ module Kumi
             Debug.debug(id, method: __method__, **fields)
           end
 
-          def trace(id, **fields, &block)
-            Debug.trace(id, **fields, &block)
+          def trace(id, **fields, &)
+            Debug.trace(id, **fields, &)
           end
         end
       end
     end
   end
-end
+end

data/lib/kumi/core/analyzer/fn_aliases.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Core
+    module Analyzer
+      module FnAliases
+        # Canonical names use simple domains:
+        # - core.*  : elementwise / structural ops
+        # - agg.*   : reductions (sum, etc.)
+        #
+        # NOTE: keep this lexical-only. Overload resolution (e.g. length on string vs array)
+        #       should happen in a later resolver pass.
+
+        MAP = {
+          :+ => :"core.add",
+          :- => :"core.sub",
+          :* => :"core.mul",
+          :/ => :"core.div",
+          :% => :"core.mod",
+          :** => :"core.pow",
+          :"==" => :"core.eq",
+          :"!=" => :"core.neq",
+          :">" => :"core.gt",
+          :">=" => :"core.gte",
+          :"<" => :"core.lt",
+          :"<=" => :"core.lte",
+          :"&" => :"core.and",
+          :"!" => :"core.not"
+        }.freeze
+
+        # Public: normalize any fn identifier (String/Symbol) to canonical Symbol.
+        # Unknown names are returned unchanged (as Symbol).
+        def self.canonical(fn)
+          key =
+            case fn
+            when Symbol then fn
+            when String then fn.strip.downcase.to_sym
+            else fn.to_s.strip.downcase.to_sym
+            end
+
+          MAP.fetch(key, key)
+        end
+      end
+    end
+  end
+end

data/lib/kumi/core/analyzer/folder.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Core
+    module Analyzer
+      class Folder
+        def initialize(pass, nast_module, order, registry)
+          @nast_module = nast_module
+          @order = order
+          @registry = registry
+          @pass = pass
+          @known_constants = {}
+          @changed = false
+        end
+
+        def debug(msg)
+          @pass.method(:debug).call(msg)
+        end
+
+        def fold
+          folded_decls = {}
+          @order.each do |decl_name|
+            decl = @nast_module.decls[decl_name]
+            next unless decl
+
+            folded_decls[decl_name] = fold_declaration(decl)
+          end
+
+          [NAST::Module.new(decls: folded_decls), @changed]
+        end
+
+        def fold_declaration(decl)
+          debug "[FOLD]   Folding declaration :#{decl.name}"
+          new_body = fold_node(decl.body)
+          @changed ||= !new_body.equal?(decl.body)
+
+          # *** FIX ***: A node is constant if it's a Const OR a Tuple of only Consts.
+          is_constant = new_body.is_a?(NAST::Const) ||
+                        (new_body.is_a?(NAST::Tuple) && new_body.args.all? { |arg| arg.is_a?(NAST::Const) })
+
+          if is_constant
+            @known_constants[decl.name] = new_body
+            debug "[FOLD]     - Identified :#{decl.name} as a constant value."
+          end
+
+          NAST::Declaration.new(name: decl.name, body: new_body, loc: decl.loc, meta: decl.meta)
+        end
+
+        def fold_node(node)
+          case node
+          when NAST::Call
+            fold_call(node)
+          when NAST::Ref
+            resolved_node = @known_constants.fetch(node.name, node)
+            unless resolved_node.equal?(node)
+              debug "[FOLD]     - Propagating constant for ref :#{node.name}"
+              @changed = true
+            end
+            resolved_node
+          when NAST::Tuple
+            new_args = node.args.map { |arg| fold_node(arg) }
+            new_args.map(&:object_id) == node.args.map(&:object_id) ? node : NAST::Tuple.new(args: new_args, loc: node.loc)
+          else
+            node
+          end
+        end
+
+        def fold_call(node)
+          folded_args = node.args.map { |arg| fold_node(arg) }
+          folded_call = if folded_args.map(&:object_id) == node.args.map(&:object_id)
+                          node
+                        else
+                          NAST::Call.new(fn: node.fn,
+                                         args: folded_args,
+                                         opts: node.opts,
+                                         loc: node.loc)
+                        end
+
+          debug "[FOLD]     - Attempting to evaluate call to :#{folded_call.fn}"
+          evaluated_value = ConstantEvaluator.evaluate(folded_call, @registry, @known_constants)
+
+          if evaluated_value.nil?
+            debug "[FOLD]       - Could not evaluate. Keeping call."
+            folded_call
+          else
+            debug "[FOLD]       - SUCCESS! Folded call to :#{folded_call.fn} -> #{evaluated_value.inspect}"
+            @changed = true
+            NAST::Const.new(value: evaluated_value, loc: node.loc)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kumi/core/analyzer/macro_expander.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Core
+    module Analyzer
+      # Takes a function's `expand` template and pre-normalized NAST arguments,
+      # and recursively builds a new NAST tree representing the expansion.
+      class MacroExpander
+        NAST = Kumi::Core::NAST
+
+        def self.expand(func, normalized_args, loc, errors)
+          new(func, normalized_args, loc, errors).expand
+        end
+
+        def initialize(func, normalized_args, loc, errors)
+          @func = func
+          @normalized_args = normalized_args
+          @loc = loc
+          @errors = errors
+        end
+
+        def expand
+          build_nast_node(@func.expand)
+        end
+
+        private
+
+        def build_nast_node(template_part)
+          case template_part
+          when Hash
+            if template_part.key?("fn")
+              build_call_node(template_part)
+            elsif template_part.key?("const")
+              NAST::Const.new(value: template_part["const"], loc: @loc)
+            else
+              raise "Invalid expansion template part: #{template_part}"
+            end
+          when String
+            build_argument_node(template_part)
+          else
+            raise "Invalid expansion template part: #{template_part}"
+          end
+        end
+
+        def build_call_node(template_hash)
+          new_args = template_hash.fetch("args", []).map do |arg_template|
+            build_nast_node(arg_template)
+          end
+
+          NAST::Call.new(
+            fn: template_hash.fetch("fn"),
+            args: new_args,
+            loc: @loc
+          )
+        end
+
+        def build_argument_node(arg_str)
+          raise "Invalid argument placeholder: #{arg_str}" unless arg_str.start_with?("$")
+
+          index = arg_str[1..].to_i - 1
+          @normalized_args.fetch(index) do
+            @errors << "Expansion for #{@func.id} requires at least #{index + 1} arguments."
+            NAST::Const.new(value: nil, loc: @loc)
+          end
+        end
+      end
+    end
+  end
+end