kumi 0.0.13 → 0.0.14

data/lib/kumi/analyzer.rb

@@ -15,11 +15,13 @@ module Kumi
       Core::Analyzer::Passes::BroadcastDetector,               # 9. Detects which operations should be broadcast over arrays.
       Core::Analyzer::Passes::TypeInferencerPass,              # 10. Infers types for all declarations (uses vectorization metadata).
       Core::Analyzer::Passes::TypeConsistencyChecker,          # 11. Validates declared vs inferred type consistency.
-      Core::Analyzer::Passes::TypeChecker,                     # 12. Validates types using inferred information.
-      Core::Analyzer::Passes::InputAccessPlannerPass,          # 13. Plans access strategies for input fields.
-      Core::Analyzer::Passes::ScopeResolutionPass,             # 14. Plans execution scope and lifting needs for declarations.
-      Core::Analyzer::Passes::JoinReducePlanningPass,          # 15. Plans join/reduce operations (Generates IR Structs)
-      Core::Analyzer::Passes::LowerToIRPass # 16. Lowers the schema to IR (Generates IR Structs)
+      Core::Analyzer::Passes::FunctionSignaturePass,           # 12. Resolves NEP-20 signatures for function calls.
+      Core::Analyzer::Passes::TypeChecker,                     # 13. Validates types using inferred information.
+      Core::Analyzer::Passes::InputAccessPlannerPass,          # 14. Plans access strategies for input fields.
+      Core::Analyzer::Passes::ScopeResolutionPass,             # 15. Plans execution scope and lifting needs for declarations.
+      Core::Analyzer::Passes::JoinReducePlanningPass,          # 16. Plans join/reduce operations (Generates IR Structs)
+      Core::Analyzer::Passes::LowerToIRPass,                   # 17. Lowers the schema to IR (Generates IR Structs)
+      Core::Analyzer::Passes::LoadInputCSE                     # 18. Eliminates redundant load_input operations
     ].freeze
 
     def self.analyze!(schema, passes: DEFAULT_PASSES, **opts)
@@ -32,33 +34,91 @@ module Kumi
     end
 
     def self.run_analysis_passes(schema, passes, state, errors)
-      passes.each do |pass_class|
+      # Resume from a saved state if configured
+      state = Core::Analyzer::Checkpoint.load_initial_state(state)
+
+      debug_on = Core::Analyzer::Debug.enabled?
+      resume_at  = Core::Analyzer::Checkpoint.resume_at
+      stop_after = Core::Analyzer::Checkpoint.stop_after
+      skipping   = !!resume_at
+
+      passes.each_with_index do |pass_class, idx|
+        pass_name = pass_class.name.split("::").last
+
+        if skipping
+          skipping = false if pass_name == resume_at
+          next if skipping
+        end
+
+        Core::Analyzer::Checkpoint.entering(pass_name:, idx:, state:)
+
+        before = state.to_h if debug_on
+        Core::Analyzer::Debug.reset_log(pass: pass_name) if debug_on
+
+        t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         pass_instance = pass_class.new(schema, state)
         begin
           state = pass_instance.run(errors)
         rescue StandardError => e
           # TODO: - GREATLY improve this, need to capture the context of the error
           # and the pass that failed and line number if relevant
-          pass_name = pass_class.name.split("::").last
           message = "Error in Analysis Pass(#{pass_name}): #{e.message}"
           errors << Core::ErrorReporter.create_error(message, location: nil, type: :semantic, backtrace: e.backtrace)
 
+          if debug_on
+            logs = Core::Analyzer::Debug.drain_log
+            elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0) * 1000).round(2)
+
+            Core::Analyzer::Debug.emit(
+              pass: pass_name,
+              diff: {},
+              elapsed_ms: elapsed_ms,
+              logs: logs + [{ level: :error, id: :exception, message: e.message, error_class: e.class.name }]
+            )
+          end
+
           raise
         end
+        elapsed_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0) * 1000).round(2)
+
+        if debug_on
+          after = state.to_h
+
+          # Optional immutability guard
+          if ENV["KUMI_DEBUG_REQUIRE_FROZEN"] == "1"
+            (after || {}).each do |k, v|
+              if v.nil? || v.is_a?(Numeric) || v.is_a?(Symbol) || v.is_a?(TrueClass) || v.is_a?(FalseClass) || (v.is_a?(String) && v.frozen?)
+                next
+              end
+              raise "State[#{k}] not frozen: #{v.class}" unless v.frozen?
+            end
+          end
+
+          diff = Core::Analyzer::Debug.diff_state(before, after)
+          logs = Core::Analyzer::Debug.drain_log
+
+          Core::Analyzer::Debug.emit(
+            pass: pass_name,
+            diff: diff,
+            elapsed_ms: elapsed_ms,
+            logs: logs
+          )
+        end
+
+        Core::Analyzer::Checkpoint.leaving(pass_name:, idx:, state:)
+
+        break if stop_after && pass_name == stop_after
       end
       state
     end
 
     def self.handle_analysis_errors(errors)
       type_errors = errors.select { |e| e.type == :type }
-      semantic_errors = errors.select { |e| e.type == :semantic }
       first_error_location = errors.first.location
 
       raise Errors::TypeError.new(format_errors(errors), first_error_location) if type_errors.any?
 
-      raise Errors::SemanticError.new(format_errors(errors), first_error_location) if first_error_location || semantic_errors
-
-      raise Errors::AnalysisError.new(format_errors(errors))
+      raise Errors::SemanticError.new(format_errors(errors), first_error_location)
     end
 
     def self.create_analysis_result(state)
@@ -76,16 +136,7 @@ module Kumi
     def self.format_errors(errors)
       return "" if errors.empty?
 
-      backtrace = errors.first.backtrace
-
-      message = errors.map(&:to_s).join("\n")
-
-      message.tap do |msg|
-        if backtrace && !backtrace.empty?
-          msg << "\n\nBacktrace:\n"
-          msg << backtrace[0..10].join("\n") # Limit to first 10 lines for readability
-        end
-      end
+      errors.map(&:to_s).join("\n")
     end
   end
 end

data/lib/kumi/core/analyzer/checkpoint.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Kumi
+  module Core
+    module Analyzer
+      module Checkpoint
+        class << self
+          # ===== Interface shape matches Debug =====
+          def enabled?
+            ENV["KUMI_CHECKPOINT"] == "1" ||
+              !resume_from.nil? || !resume_at.nil? || !stop_after.nil?
+          end
+
+          # ---- Config (ENV) ----
+          def dir         = ENV["KUMI_CHECKPOINT_DIR"]    || "tmp/analysis_snapshots"
+          def phases      = (ENV["KUMI_CHECKPOINT_PHASE"] || "before,after").split(",").map! { _1.strip.downcase.to_sym }
+          def formats     = (ENV["KUMI_CHECKPOINT_FORMAT"] || "marshal").split(",").map! { _1.strip.downcase } # marshal|json|both
+          def resume_from = ENV["KUMI_RESUME_FROM"] # file path (.msh or .json)
+          def resume_at   = ENV["KUMI_RESUME_AT"]   # pass short name
+          def stop_after  = ENV["KUMI_STOP_AFTER"]  # pass short name
+
+          # ===== Lifecycle (called by analyzer) =====
+          def load_initial_state(default_state)
+            path = resume_from
+            return default_state unless path && File.exist?(path)
+            data = File.binread(path)
+            path.end_with?(".msh") ? StateSerde.load_marshal(data)
+                                   : StateSerde.load_json(data)
+          end
+
+          def entering(pass_name:, idx:, state:)
+            return unless enabled?
+            snapshot(pass_name:, idx:, phase: :before, state:) if phases.include?(:before)
+          end
+
+          def leaving(pass_name:, idx:, state:)
+            return unless enabled?
+            snapshot(pass_name:, idx:, phase: :after, state:) if phases.include?(:after)
+          end
+
+          # ===== Implementation =====
+          def snapshot(pass_name:, idx:, phase:, state:)
+            FileUtils.mkdir_p(dir)
+            base = File.join(dir, "%03d_#{pass_name}_#{phase}" % idx)
+            files = []
+
+            if formats.include?("marshal") || formats.include?("both")
+              path = "#{base}.msh"
+              File.binwrite(path, StateSerde.dump_marshal(state))
+              files << path
+            end
+
+            if formats.include?("json") || formats.include?("both")
+              path = "#{base}.json"
+              File.write(path, StateSerde.dump_json(state, pretty: true))
+              files << path
+            end
+
+            # Fold checkpoint info into the same per-pass logs the Debugger uses.
+            if Core::Analyzer::Debug.enabled?
+              Core::Analyzer::Debug.info(:checkpoint, phase:, idx:, files:)
+            end
+
+            files
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kumi/core/analyzer/debug.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+require "json"
+require "time"
+
+module Kumi
+  module Core
+    module Analyzer
+      module Debug
+        KEY = :kumi_debug_log
+        
+        class << self
+          def enabled?
+            ENV["KUMI_DEBUG_STATE"] == "1"
+          end
+
+          def output_path
+            ENV["KUMI_DEBUG_OUTPUT_PATH"]
+          end
+
+          def max_depth
+            (ENV["KUMI_DEBUG_MAX_DEPTH"] || "5").to_i
+          end
+
+          def max_items
+            (ENV["KUMI_DEBUG_MAX_ITEMS"] || "100").to_i
+          end
+
+          # Log buffer management
+          def reset_log(pass:)
+            Thread.current[KEY] = { pass: pass, events: [] }
+          end
+
+          def drain_log
+            stash = Thread.current[KEY]
+            Thread.current[KEY] = nil
+            (stash && stash[:events]) || []
+          end
+
+          def log(level:, id:, method: nil, **fields)
+            buf = Thread.current[KEY]
+            return unless buf
+
+            loc = caller_locations(1, 1)&.first
+            meth = method || loc&.base_label
+            buf[:events] << {
+              ts: Time.now.utc.iso8601,
+              pass: buf[:pass],
+              level: level,
+              id: id,
+              method: meth,
+              file: loc&.path,
+              line: loc&.lineno,
+              **fields
+            }
+          end
+
+          def info(id, **fields)
+            log(level: :info, id: id, **fields)
+          end
+
+          def debug(id, **fields)
+            log(level: :debug, id: id, **fields)
+          end
+
+          def trace(id, **start_fields)
+            t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            info("#{id}_start".to_sym, **start_fields)
+            yield.tap do |ret|
+              dt = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0) * 1000).round(2)
+              info("#{id}_finish".to_sym, ms: dt)
+              ret
+            end
+          rescue => e
+            log(level: :error, id: "#{id}_error".to_sym, error: e.class.name, message: e.message)
+            raise
+          end
+
+          # State diffing
+          def diff_state(before, after)
+            changes = {}
+            
+            all_keys = (before.keys + after.keys).uniq
+            all_keys.each do |key|
+              if !before.key?(key)
+                changes[key] = { type: :added, value: truncate(after[key]) }
+              elsif !after.key?(key)
+                changes[key] = { type: :removed, value: truncate(before[key]) }
+              elsif before[key] != after[key]
+                changes[key] = {
+                  type: :changed,
+                  before: truncate(before[key]),
+                  after: truncate(after[key])
+                }
+              end
+            end
+            
+            changes
+          end
+
+          # Emit debug event
+          def emit(pass:, diff:, elapsed_ms:, logs:)
+            payload = {
+              ts: Time.now.utc.iso8601,
+              pass: pass,
+              elapsed_ms: elapsed_ms,
+              diff: diff,
+              logs: logs
+            }
+            
+            if output_path && !output_path.empty?
+              File.open(output_path, "a") { |f| f.puts(JSON.dump(payload)) }
+            else
+              $stdout.puts "\n=== STATE #{pass} (#{elapsed_ms}ms) ==="
+              $stdout.puts JSON.pretty_generate(payload)
+            end
+          end
+
+          private
+
+          def truncate(value, depth = max_depth)
+            return value if depth <= 0
+            
+            case value
+            when Hash
+              if value.size > max_items
+                truncated = value.first(max_items).to_h
+                truncated[:__truncated__] = "... #{value.size - max_items} more"
+                truncated.transform_values { |v| truncate(v, depth - 1) }
+              else
+                value.transform_values { |v| truncate(v, depth - 1) }
+              end
+            when Array
+              if value.size > max_items
+                truncated = value.first(max_items).dup
+                truncated << "... #{value.size - max_items} more"
+                truncated.map { |v| truncate(v, depth - 1) }
+              else
+                value.map { |v| truncate(v, depth - 1) }
+              end
+            when String
+              value
+            else
+              value
+            end
+          end
+
+        end
+
+        # Mixin for passes
+        module Loggable
+          def log_info(id, **fields)
+            Debug.info(id, method: __method__, **fields)
+          end
+
+          def log_debug(id, **fields)
+            Debug.debug(id, method: __method__, **fields)
+          end
+
+          def trace(id, **fields, &block)
+            Debug.trace(id, **fields, &block)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kumi/core/analyzer/passes/broadcast_detector.rb

@@ -40,9 +40,7 @@ module Kumi
               result = analyze_value_vectorization(name, decl.expression, array_fields, nested_paths, vectorized_values, errors,
                                                    definitions)
 
-              if ENV["DEBUG_BROADCAST_CLEAN"]
-                puts "#{name}: #{result[:type]} #{format_broadcast_info(result)}"
-              end
+              puts "#{name}: #{result[:type]} #{format_broadcast_info(result)}" if ENV["DEBUG_BROADCAST_CLEAN"]
 
               case result[:type]
               when :vectorized

data/lib/kumi/core/analyzer/passes/function_signature_pass.rb

@@ -0,0 +1,199 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Core
+    module Analyzer
+      module Passes
+        # RESPONSIBILITY: Apply NEP-20 signature resolution to function calls
+        # DEPENDENCIES: :node_index from Toposorter, :broadcast_metadata (optional)
+        # PRODUCES: Signature metadata in node_index for CallExpression nodes
+        # INTERFACE: new(schema, state).run(errors)
+        class FunctionSignaturePass < PassBase
+          def run(errors)
+            node_index = get_state(:node_index, required: true)
+
+            # Process all CallExpression nodes in the index
+            node_index.each do |object_id, entry|
+              next unless entry[:type] == "CallExpression"
+
+              resolve_function_signature(entry, object_id, errors)
+            end
+
+            state # Node index is modified in-place
+          end
+
+          private
+
+          def resolve_function_signature(entry, object_id, errors)
+            node = entry[:node]
+
+            # 1) Gather candidate signatures from current registry
+            sig_strings = get_function_signatures(node)
+            return if sig_strings.empty?
+
+            begin
+              sigs = parse_signatures(sig_strings)
+            rescue Kumi::Core::Functions::SignatureError => e
+              report_error(errors, "Invalid signature for function `#{node.fn_name}`: #{e.message}",
+                           location: node.loc, type: :type)
+              return
+            end
+
+            # 2) Build arg_shapes from current node context
+            arg_shapes = build_argument_shapes(node, object_id)
+
+            # 3) Resolve signature
+            begin
+              plan = Kumi::Core::Functions::SignatureResolver.choose(signatures: sigs, arg_shapes: arg_shapes)
+            rescue Kumi::Core::Functions::SignatureMatchError => e
+              report_error(errors, 
+                          "Signature mismatch for `#{node.fn_name}` with args #{format_shapes(arg_shapes)}. Candidates: #{format_sigs(sig_strings)}. #{e.message}",
+                          location: node.loc, type: :type)
+              return
+            end
+
+            # 4) Attach metadata to node index entry
+            attach_signature_metadata(entry, plan)
+          end
+
+          def get_function_signatures(node)
+            # Use RegistryV2 if enabled, otherwise fall back to legacy registry
+            if registry_v2_enabled?
+              registry_v2_signatures(node)
+            else
+              legacy_registry_signatures(node)
+            end
+          end
+
+          def registry_v2_signatures(node)
+            registry_v2.get_function_signatures(node.fn_name)
+          rescue => e
+            # If RegistryV2 fails, fall back to legacy
+            legacy_registry_signatures(node)
+          end
+
+          def legacy_registry_signatures(node)
+            # Try to get signatures from the current registry
+            # For now, we'll create basic signatures from the current registry format
+
+            meta = Kumi::Registry.signature(node.fn_name)
+
+            # Check if the function already has NEP-20 signatures
+            return meta[:signatures] if meta[:signatures] && meta[:signatures].is_a?(Array)
+
+            # Otherwise, create a basic signature from arity
+            # This is a bridge until we have full NEP-20 signatures in the registry
+            create_basic_signature(meta[:arity])
+          rescue Kumi::Errors::UnknownFunction
+            # For now, return empty array - function existence will be caught by TypeChecker
+            []
+          end
+
+          def create_basic_signature(arity)
+            return [] if arity.nil? || arity < 0 # Variable arity - skip for now
+
+            case arity
+            when 0
+              ["()->()"] # Scalar function
+            when 1
+              ["()->()", "(i)->(i)"] # Scalar or element-wise
+            when 2
+              ["(),()->()", "(i),(i)->(i)"] # Scalar or element-wise binary
+            else
+              # For higher arity, just provide scalar signature
+              args = (["()"] * arity).join(",")
+              ["#{args}->()"]
+            end
+          end
+
+          def build_argument_shapes(node, object_id)
+            # Build argument shapes from current analysis context
+            node.args.map do |arg|
+              axes = get_broadcast_metadata(arg.object_id)
+              normalize_shape(axes)
+            end
+          end
+
+          def normalize_shape(axes)
+            case axes
+            when nil
+              [] # scalar
+            when Array
+              axes.map { |d| d.is_a?(Integer) ? d : d.to_sym }
+            else
+              [] # defensive fallback
+            end
+          end
+
+          def get_broadcast_metadata(arg_object_id)
+            # Try to get broadcast metadata from existing analysis state
+            broadcast_meta = get_state(:broadcast_metadata, required: false)
+            return nil unless broadcast_meta
+
+            # Look up by node object_id
+            broadcast_meta[arg_object_id]&.dig(:axes)
+          end
+
+          def parse_signatures(sig_strings)
+            @sig_cache ||= {}
+            sig_strings.map do |s|
+              @sig_cache[s] ||= Kumi::Core::Functions::SignatureParser.parse(s)
+            end
+          end
+
+          def format_shapes(shapes)
+            shapes.map { |ax| "(#{ax.join(',')})" }.join(', ')
+          end
+
+          def format_sigs(sig_strings)
+            sig_strings.join(" | ")
+          end
+
+          def attach_signature_metadata(entry, plan)
+            # Attach signature resolution results to the node index entry
+            # This way other passes can access the metadata via the node index
+            metadata = entry[:metadata]
+
+            attach_core_signature_data(metadata, plan)
+            attach_shape_contract(metadata, plan)
+          end
+
+          def attach_core_signature_data(metadata, plan)
+            metadata[:signature]           = plan[:signature]
+            metadata[:result_axes]         = plan[:result_axes]        # e.g., [:i, :j]
+            metadata[:join_policy]         = plan[:join_policy]        # nil | :zip | :product
+            metadata[:dropped_axes]        = plan[:dropped_axes]       # e.g., [:j] for reductions
+            metadata[:effective_signature] = plan[:effective_signature] # Normalized for lowering
+            metadata[:dim_env]             = plan[:env]                # Dimension bindings (for matmul)
+            metadata[:signature_score]     = plan[:score]              # Match quality
+          end
+
+          def attach_shape_contract(metadata, plan)
+            # Attach shape contract for lowering convenience
+            metadata[:shape_contract] = {
+              in:   plan[:effective_signature][:in_shapes],
+              out:  plan[:effective_signature][:out_shape],
+              join: plan[:effective_signature][:join_policy]
+            }
+          end
+
+          def registry_v2_enabled?
+            ENV["KUMI_FN_REGISTRY_V2"] == "1"
+          end
+
+          def registry_v2
+            @registry_v2 ||= Kumi::Core::Functions::RegistryV2.load_from_file
+          end
+
+          def nep20_flex_enabled?
+            ENV["KUMI_ENABLE_FLEX"] == "1"
+          end
+
+          def nep20_bcast1_enabled?
+            ENV["KUMI_ENABLE_BCAST1"] == "1"
+          end
+        end
+      end
+    end
+  end
+end