kumi 0.0.18 → 0.0.20

data/lib/kumi/core/analyzer/passes/assemble_irv2_pass.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Core
+    module Analyzer
+      module Passes
+        # Assembles the final IRv2 JSON structure from analyzer state
+        #
+        # Input:  state[:irv2_module], state[:ir_input_plans]
+        # Output: state[:irv2] (complete JSON structure ready for serialization)
+        class AssembleIRV2Pass < PassBase
+          def run(errors)
+            irv2_module = get_state(:irv2_module, required: true)
+            input_plans = get_state(:ir_input_plans, required: true)
+
+            irv2_structure = build_irv2_structure(irv2_module, input_plans, errors)
+            debug "Assembled IRv2 structure with #{irv2_structure['declarations'].size} declarations"
+
+            state.with(:irv2, irv2_structure.freeze)
+          end
+
+          private
+
+          def build_irv2_structure(irv2_module, input_plans, _errors)
+            {
+              "version" => IR_SCHEMA_VERSION,
+              "module" => determine_module_name(irv2_module),
+              "declarations" => build_declarations(irv2_module.declarations),
+              "analysis" => build_analysis_section(input_plans, irv2_module.metadata)
+            }
+          end
+
+          def determine_module_name(irv2_module)
+            irv2_module.metadata.dig("module_name") || "schema_module"
+          end
+
+          def build_declarations(declarations_hash)
+            result = {}
+            declarations_hash.each do |name, declaration|
+              result[name.to_s] = build_declaration_structure(declaration)
+            end
+            result
+          end
+
+          def build_declaration_structure(declaration)
+            {
+              "name" => declaration.name.to_s,
+              "parameters" => build_parameters(declaration.parameters),
+              "operations" => build_operations(declaration.operations, declaration.name),
+              "result" => declaration.result.id
+            }
+          end
+
+          def build_parameters(parameters)
+            parameters.map do |param|
+              case param[:type]
+              when :input
+                { "kind" => "input", "path" => param[:source] }
+              when :dependency
+                { "kind" => "dependency", "source" => param[:source] }
+              else
+                param
+              end
+            end
+          end
+
+          def build_operations(operations, _current_decl_name)
+            operations.map do |op|
+              h = {
+                "id" => op.id,
+                "op" => op.op.to_s,
+                "args" => build_operation_args(op.args)
+              }
+              h["stamp"]       = op.stamp if op.stamp
+              h["elem_stamps"] = op.elem_stamps if op.elem_stamps
+              attrs = build_operation_attrs(op.attrs)
+              h["attrs"] = attrs unless attrs.empty?
+              h
+            end
+          end
+
+          def build_operation_args(args)
+            args.map { |arg| arg.respond_to?(:id) ? arg.id : arg }
+          end
+
+          def build_operation_attrs(attrs)
+            attrs.each_with_object({}) { |(k, v), h| h[k.to_s] = v }
+          end
+
+          def build_analysis_section(input_plans, _metadata)
+            {
+              "defaults" => {
+                "key_policy" => "indifferent",
+                "on_missing" => "error"
+              },
+              "inputs" => build_canonical_inputs(input_plans)
+            }
+          end
+
+          def build_canonical_inputs(input_plans)
+            input_plans.map do |plan|
+              result = {
+                "path_fqn" => plan.path_fqn,
+                "axes" => plan.axes.map(&:to_s),
+                "dtype" => plan.dtype.to_s,
+                "navigation_steps" => plan.navigation_steps,
+                "key_policy" => plan.key_policy.to_s,
+                "on_missing" => plan.missing_policy.to_s
+              }
+              stringify_deep(result)
+            end
+          end
+
+          def stringify_deep(obj)
+            case obj
+            when Hash
+              obj.each_with_object({}) { |(k, v), h| h[k.to_s] = stringify_deep(v) }
+            when Array
+              obj.map { |v| stringify_deep(v) }
+            when Symbol
+              obj.to_s
+            else
+              obj
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kumi/core/analyzer/passes/attach_anchors_pass.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Core
+    module Analyzer
+      module Passes
+        class AttachAnchorsPass < PassBase
+          NAST = Kumi::Core::NAST
+
+          def run(_errors)
+            @snast = get_state(:snast_module, required: true)
+
+            out = {}
+            @snast.decls.each do |name, decl|
+              out[name] ||= {}
+
+              wanted = axes_of(decl)
+              wanted = axes_of(decl.body) if wanted.empty?
+
+              # No anchors needed for scalars (rank-0)
+              next if wanted.empty?
+
+              out[name][wanted] = pick_anchor_fqn(decl.body, wanted)
+            end
+
+            state.with(:anchor_by_decl, out.freeze)
+          end
+
+          private
+
+          def pick_anchor_fqn(node, wanted_axes)
+            return nil if Array(wanted_axes).empty?
+
+            found = nil
+            walk = lambda do |x|
+              case x
+              when NAST::InputRef
+                ax = axes_of(x)
+                found ||= ir_fqn(x) if prefix?(wanted_axes, ax)
+              when NAST::Ref
+                decl = @snast.decls.fetch(x.name) { raise "unknown declaration #{x.name}" }
+                walk.call(decl.body)
+              when NAST::Select
+                walk.call(x.cond)
+                walk.call(x.on_true)
+                walk.call(x.on_false)
+              when NAST::Reduce, NAST::Fold
+                walk.call(x.arg)
+              when NAST::Call, NAST::Tuple
+                x.args.each { walk.call(_1) }
+              when NAST::Hash
+                x.pairs.each { walk.call(_1) }
+              when NAST::Pair
+                walk.call(x.value)
+              when NAST::IndexRef
+                found ||= x.input_fqn
+              end
+            end
+
+            walk.call(node)
+            found or raise "no anchor for axes #{wanted_axes.inspect}"
+          end
+
+          def axes_of(n) = Array(n.meta[:stamp]&.dig(:axes))
+          def prefix?(pre, full) = pre.each_with_index.all? { |tok, i| full[i] == tok }
+          def ir_fqn(n) = n.instance_variable_get(:@fqn) || n.path.join(".")
+        end
+      end
+    end
+  end
+end

data/lib/kumi/core/analyzer/passes/attach_terminal_info_pass.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Kumi
+  module Core
+    module Analyzer
+      module Passes
+        # AttachTerminalInfoPass with debug tracing
+        class AttachTerminalInfoPass < PassBase
+          NAST = Kumi::Core::NAST
+
+          def run(_errors)
+            @dbg = ENV["KUMI_DEBUG_TERMINAL"] == "1"
+            plans  = get_state(:input_table, required: true) # Array<InputPlan>
+            by_fqn = plans.each_with_object({}) { |p, h| h[p.path_fqn.to_s] = p }
+            mod    = get_state(:snast_module, required: true)
+            annotate!(mod, by_fqn)
+            state
+          end
+
+          private
+
+          def annotate!(node, by_fqn)
+            case node
+            when NAST::Module
+              node.decls.each_value { |d| annotate!(d, by_fqn) }
+            when NAST::Declaration
+              annotate!(node.body, by_fqn)
+            when NAST::InputRef
+              annotate_input_ref!(node, by_fqn)
+            when NAST::Tuple, NAST::Call
+              node.args.each { |a| annotate!(a, by_fqn) }
+            when NAST::Hash
+              node.pairs.each { |p| annotate!(p.value, by_fqn) }
+            when NAST::Pair
+              annotate!(node.value, by_fqn)
+            when NAST::Select
+              annotate!(node.cond, by_fqn)
+              annotate!(node.on_true, by_fqn)
+              annotate!(node.on_false, by_fqn)
+            when NAST::Reduce, NAST::Fold
+              annotate!(node.arg, by_fqn)
+            when NAST::IndexRef
+              # no-op
+            end
+          end
+
+          def annotate_input_ref!(node, by_fqn)
+            plan  = by_fqn.fetch(node.path_fqn.to_s) { raise KeyError, "No InputPlan for #{node.path_fqn.inspect}" }
+            steps = normalize_steps(plan.navigation_steps) # do not mutate original
+
+            # Compute from original steps
+            base_sym, full_chain = detect_base_and_full_chain(steps)
+            tail = tail_after_last_loop(steps)
+            element_terminal = tail.any? { |s| s[:kind] == "element_access" }
+
+            # Compute key_chain based on context
+            key_chain =
+              if !element_terminal && tail.empty? && !full_chain.empty?
+                # Non-element-terminal array reference (e.g., fn(:array_size, input.x))
+                # Use full property chain to load from root
+                full_chain
+              else
+                # Element access or other cases: collect from tail
+                collect_tail_props(tail)
+              end
+
+            # Attach
+            node.instance_variable_set(:@fqn,               plan.path_fqn.to_s)
+            node.instance_variable_set(:@base_sym,          base_sym)
+            node.instance_variable_set(:@base_is_root,      base_sym == :__root)
+            node.instance_variable_set(:@full_chain,        full_chain)
+            node.instance_variable_set(:@key_chain,         key_chain)
+            node.instance_variable_set(:@element_terminal,  element_terminal)
+
+            return unless debug_enabled?
+
+            dbg_dump(
+              title: "InputRef #{plan.path_fqn}",
+              open_axis: plan.respond_to?(:open_axis) ? plan.open_axis : nil,
+              steps: steps,
+              tail: tail,
+              base_sym: base_sym,
+              full_chain: full_chain,
+              key_chain: key_chain,
+              element_terminal: element_terminal
+            )
+          end
+
+          # ---------- helpers ----------
+
+          def normalize_steps(raw)
+            Array(raw).map do |s|
+              { kind: (s[:kind] || s["kind"]).to_s, key: s[:key] || s["key"] }
+            end
+          end
+
+          def array_loop?(s)      = s && s[:kind] == "array_loop"
+          def property_access?(s) = s && s[:kind] == "property_access"
+          def element_access?(s)  = s && s[:kind] == "element_access"
+
+          # base + full property chain
+          def detect_base_and_full_chain(steps)
+            return [:__root, []] if steps.empty?
+
+            first_loop_idx = steps.index { |s| array_loop?(s) } || steps.length
+            pre            = steps[0...first_loop_idx]
+
+            base_sym =
+              if property_access?(pre.first)
+                pre.first[:key].to_sym
+              else
+                :__root
+              end
+
+            props = steps.select { |s| property_access?(s) }.map { |s| s[:key].to_sym }
+            props = [base_sym] + props if base_sym != :__root && props.first != base_sym
+            [base_sym, props]
+          end
+
+          # steps strictly after the last array_loop; if none, the entire steps
+          def tail_after_last_loop(steps)
+            last_loop_idx = steps.rindex { |s| array_loop?(s) } || -1
+            steps[(last_loop_idx + 1)..-1] || []
+          end
+
+          # property_access keys in tail:
+          # - if an element_access exists, collect properties AFTER the first element_access
+          # - otherwise collect all property_access in tail
+          def collect_tail_props(tail)
+            elem_idx = tail.index { |s| s && s[:kind] == "element_access" }
+
+            range =
+              if elem_idx
+                (elem_idx + 1)..-1       # after the element access
+              else
+                0..-1                    # no element access, take all
+              end
+
+            tail[range].to_a
+                       .select { |s| s[:kind] == "property_access" }
+                       .map    { |s| s[:key].to_sym }
+          end
+
+          # pretty trace
+          def dbg_dump(title:, open_axis:, steps:, tail:, base_sym:, full_chain:, key_chain:, element_terminal:)
+            puts "[AttachTerminalInfo] #{title}"
+            puts "  open_axis: #{open_axis.inspect}"
+            puts "  steps:"
+            steps.each_with_index { |s, i| puts "    #{i}: #{fmt_step(s)}" }
+            puts "  tail:"
+            tail.each_with_index { |s, i| puts "    #{i}: #{fmt_step(s)}" }
+            puts "  base_sym: #{base_sym.inspect}"
+            puts "  full_chain: #{full_chain.inspect}"
+            puts "  key_chain: #{key_chain.inspect}"
+            puts "  element_terminal: #{element_terminal}"
+            # quick sanity checks
+            if key_chain.empty? && steps.any? { |s| property_access?(s) } && !element_terminal
+              puts "  WARN: tail lost property keys? check last array_loop split"
+            end
+            if element_terminal && !steps.any? { |s| element_access?(s) }
+              puts "  WARN: element_terminal=true but no element_access in steps"
+            end
+            puts
+          end
+
+          def fmt_step(s)
+            k = s[:kind]
+            case k
+            when "property_access" then "property_access(#{s[:key].inspect})"
+            when "element_access"  then "element_access"
+            when "array_loop"      then "array_loop(#{s[:key].inspect})"
+            else                        "#{k}(#{s[:key].inspect})"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kumi/core/analyzer/passes/codegen/js/declaration_emitter.rb

@@ -0,0 +1,317 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "json"
+
+module Kumi
+  module Core
+    module Analyzer
+      module Passes
+        module Codegen
+          module Js
+            class DeclarationEmitter
+              extend Forwardable
+
+              def_delegators :@buffer, :write, :indent!, :dedent!, :last_line, :rewrite_line
+
+              def initialize(buffer, binds, kernels)
+                @buffer = buffer
+                @binds = binds
+                @kernels = kernels
+                @stack = []
+                @out_containers = []
+                @aliases = {}
+              end
+
+              def emit(name, ops)
+                @ops = ops
+                @yield_depth = find_yield_depth(ops)
+                has_yield = @ops.any? { |op| op.opcode == :Yield }
+                @aliases.clear
+
+                # Exported, pure function per declaration
+                write "export function _#{name}(input) {"
+                indent!
+
+                write "let out = [];" if has_yield && @yield_depth.positive?
+                @out_containers = ["out"]
+
+                @ops.each_with_index { |ins, i| emit_ins(ins, i) }
+
+                write "return out;" if has_yield && @yield_depth.positive?
+
+                dedent!
+                write "}\n"
+              end
+
+              private
+
+              def emit_ins(ins, i)
+                emitter_method = "emit_#{ins.opcode.to_s.downcase}"
+                if respond_to?(emitter_method, true)
+                  send(emitter_method, ins, i)
+                else
+                  warn "No JS emitter found for opcode: #{ins.opcode}"
+                end
+              end
+
+              # --- Loop and Yield Logic ---
+
+              def emit_loopstart(ins, i)
+                coll = areg(ins.inputs.first)
+                el = to_local(ins.attributes[:as_element])
+                ix = to_local(ins.attributes[:as_index])
+
+                write "#{coll}.forEach((#{el}, #{ix}) => {"
+                indent!
+                @stack.push({ el: el, ix: ix })
+
+                return unless loop_contain_yield?(i) && @stack.length < @yield_depth
+
+                container = "out_#{@stack.length}"
+                @out_containers.push(container)
+                write "let #{container} = [];"
+              end
+
+              def emit_loopend(ins, i)
+                start_index = find_loop_start_for_end(i)
+                is_yield_container_loop = @stack.length < @yield_depth && loop_contain_yield?(start_index)
+
+                if is_yield_container_loop
+                  child = @out_containers.pop
+                  parent = @out_containers.last
+                  write "#{parent}.push(#{child});"
+                end
+
+                @stack.pop
+                dedent!
+                write "});"
+              end
+
+              def emit_yield(ins, i)
+                v = operands_for(ins).first
+                current_depth = calculate_depth_at(i)
+
+                if current_depth.zero?
+                  # Optimization: if the last statement was the assignment,
+                  # we can just make it a return statement.
+                  if last_line&.strip&.match?(/^let #{v} = /)
+                    new_line = last_line.sub("let #{v} = ", "return ")
+                    rewrite_line(new_line.strip.chomp(";"))
+                  else
+                    write "return #{v};"
+                  end
+                else
+                  write "#{@out_containers.last}.push(#{v});"
+                end
+              end
+
+              def emit_kernelcall(ins, _i)
+                kernel = kernel_for(ins.result_register)
+                args = operands_for(ins)
+
+                if (template = kernel[:attrs]["js_inline"] || kernel[:attrs]["inline"])
+                  inlined_code = _inline_kernel(template, args)
+                  if template.start_with?("=") || template.include?("$0")
+                    write "let #{vreg(ins)} #{inlined_code};"
+                  else
+                    write "#{vreg(ins)} #{inlined_code};"
+                  end
+                else
+                  fn_name = kernel_method_name(kernel[:fn_id])
+                  write "let #{vreg(ins)} = #{fn_name}(#{args.join(', ')});"
+                end
+              end
+
+              def emit_fold(ins, _i)
+                kernel = kernel_for(ins.result_register)
+                args = operands_for(ins)
+
+                if (template = kernel[:attrs]["js_fold_inline"] || kernel[:attrs]["fold_inline"])
+                  inlined_code = _inline_kernel(template, args)
+                  write "let #{vreg(ins)} #{inlined_code};"
+                else
+                  raise "JS Emitter: Can't fold - no 'js_fold_inline' or 'fold_inline' template defined for kernel #{kernel[:fn_id]}"
+                end
+              end
+
+              def emit_accumulate(ins, _i)
+                kernel = kernel_for(ins.result_register)
+                args = [vreg(ins), operands_for(ins).first]
+
+                if kernel[:attrs]["first_element"]
+                  write "if (#{args[0]} === null || #{args[0]} === undefined) {"
+                  indent!
+                  write "#{args[0]} = #{args[1]};"
+                  dedent!
+                  write "} else {"
+                  indent!
+                end
+
+                if (template = kernel[:attrs]["js_inline"] || kernel[:attrs]["inline"])
+                  inlined_code = _inline_kernel(template, args)
+                  write "#{args[0]} #{inlined_code};"
+                else
+                  fn_name = kernel_method_name(kernel[:fn_id])
+                  write "#{args[0]} = #{fn_name}(#{args.join(', ')});"
+                end
+
+                return unless kernel[:attrs]["first_element"]
+
+                dedent!
+                write "}"
+              end
+
+              def emit_constant(ins, _i)
+                write "const #{vreg(ins)} = #{lit(ins.immediates.first)};"
+              end
+
+              def emit_loadinput(ins, _i)
+                key = imm_key(ins)
+                write "let #{vreg(ins)} = input[#{key.to_json}];"
+              end
+
+              def emit_loadfield(ins, _i)
+                obj = areg(ins.inputs.first)
+                key = imm_key(ins)
+
+                write "let #{vreg(ins)} = #{obj}[#{key.to_json}];"
+              end
+
+              def emit_select(ins, _i)
+                c, t, f = operands_for(ins)
+                write "let #{vreg(ins)} = #{c} ? #{t} : #{f};"
+              end
+
+              def emit_declareaccumulator(ins, _i)
+                kernel = kernel_for(ins.result_register)
+                identity_attrs = kernel[:attrs]["identity"]
+                identity = identity_attrs.is_a?(Hash) ? identity_attrs["integer"] || identity_attrs["any"] : identity_attrs
+                write "let #{vreg(ins)} = #{identity.to_json};"
+              end
+
+              def emit_loadaccumulator(ins, _i)
+                acc_reg = ins.inputs.first or raise "No accumulator bound"
+                # This is a no-op for codegen; it creates an alias that `areg` will resolve.
+                @aliases[vreg(ins)] = areg(acc_reg)
+              end
+
+              def emit_loaddeclaration(ins, _i)
+                write "let #{vreg(ins)} = _$#{ins.immediates.first.value}(input);"
+              end
+
+              def emit_maketuple(ins, _i)
+                elements = operands_for(ins).join(", ")
+                write "let #{vreg(ins)} = [#{elements}];"
+              end
+
+              def emit_makeobject(ins, _i)
+                values, keys = operands_for(ins, split: true)
+                write "let #{vreg(ins)} = {"
+                indent!
+                keys.each_with_index do |key, idx|
+                  postfix = idx == keys.size - 1 ? "" : ","
+                  # Keys in JS objects from variables don't need quotes if they are valid identifiers
+                  write "#{key}: #{values[idx]}#{postfix}"
+                end
+                dedent!
+                write "};"
+              end
+
+              def emit_length(ins, _i)
+                arr = ins.inputs[0]
+                write "let #{vreg(ins)} = #{arr}.length;"
+              end
+
+              def emit_gather(ins, _)
+                arr = ins.inputs[0]
+                idx = ins.inputs[1]
+                write "let #{vreg(ins)} = #{arr}[#{idx}];"
+              end
+
+              def find_matching_loop_end(start_index)
+                depth = 1; (start_index + 1...@ops.length).each do |i|
+                  op = @ops[i].opcode
+                  depth += 1 if op == :LoopStart
+                  depth -= 1 if op == :LoopEnd
+                  return i if depth.zero?
+                end
+                raise "Unbalanced LoopStart at index #{start_index}"
+              end
+
+              def find_loop_start_for_end(end_index)
+                depth = 1; (end_index - 1).downto(0) do |i|
+                  op = @ops[i].opcode
+                  depth += 1 if op == :LoopEnd
+                  depth -= 1 if op == :LoopStart
+                  return i if depth.zero?
+                end
+                raise "Unbalanced LoopEnd at index #{end_index}"
+              end
+
+              def loop_contain_yield?(loop_start_index)
+                end_index = find_matching_loop_end(loop_start_index)
+                @ops[(loop_start_index + 1)...end_index].any? { |ins| ins.opcode == :Yield }
+              end
+
+              def calculate_depth_at(index)
+                depth = 0
+                @ops[0...index].each do |op|
+                  depth += 1 if op.opcode == :LoopStart
+                  depth -= 1 if op.opcode == :LoopEnd
+                end
+                depth
+              end
+
+              def find_yield_depth(ops)
+                yield_op_index = ops.index { _1.opcode == :Yield }
+                return 0 unless yield_op_index
+
+                calculate_depth_at(yield_op_index)
+              end
+
+              def operands_for(ins, split: false)
+                immediates = Array(ins.immediates).map { |imm| lit(imm) }
+                inputs = Array(ins.inputs).map do |input|
+                  if input == :__immediate_placeholder__
+                    immediates.shift
+                  else
+                    areg(input)
+                  end
+                end
+                return [inputs, immediates] if split
+
+                inputs + immediates
+              end
+
+              def lit(l) = l.value.to_json
+              def imm_key(ins) = ins.immediates&.first&.value
+              def to_local(sym) = sym.to_s.delete("%:")
+
+              def areg(r)
+                return "null" if r.nil?
+
+                resolved = to_local(r)
+                # Follow the alias chain to find the original source register.
+                while (aliased_to = @aliases[resolved])
+                  resolved = aliased_to
+                end
+                resolved
+              end
+
+              def vreg(ins) = to_local(ins.result_register)
+              def kernel_method_name(fn_id) = "__#{fn_id.tr('.', '_')}"
+              def kernel_for(reg) = @kernels.fetch(@binds.fetch(reg).fn_id)
+
+              def _inline_kernel(template, args)
+                inlined = template.dup
+                args.each_with_index { |arg, i| inlined.gsub!("$#{i}", arg.to_s) }
+                inlined
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end