rubik 0.1.0

data/lib/rubik/ast/rust_extractor.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require_relative "symbol"
+require_relative "body_loader"
+
+module Rubik
+  module AST
+    class RustExtractor
+      include BodyLoader
+
+      def self.extract(file_path, content = nil)
+        new(file_path, content).extract
+      end
+
+      def initialize(file_path, content = nil)
+        @file_path = file_path
+        @content = content || (File.exist?(file_path) ? File.read(file_path) : "")
+        @lines = @content.lines
+      end
+
+      def extract
+        symbols = []
+        return symbols if @content.empty?
+
+        current_impl = nil
+        impl_end_line = 0
+        i = 0
+
+        while i < @lines.size
+          line = @lines[i]
+          line_num = i + 1
+          stripped = line.strip
+
+          current_impl = nil if line_num > impl_end_line
+
+          if stripped.start_with?("use ")
+            symbols << Symbol.new(
+              name: stripped.sub(/;$/, ""),
+              kind: :import,
+              file: @file_path,
+              line_start: line_num,
+              line_end: line_num,
+              signature: stripped
+            )
+          elsif (match = stripped.match(/^(?:pub(?:\(crate\))?\s+)?(struct|enum|trait)\s+([A-Za-z0-9_]+)/))
+            kind = match[1].to_sym
+            name = match[2]
+            end_line = stripped.end_with?(";") ? line_num : find_closing_brace(i)
+            symbols << Symbol.new(
+              name: name,
+              kind: kind,
+              file: @file_path,
+              line_start: line_num,
+              line_end: end_line,
+              signature: line.split("{").first.to_s.strip,
+              body_loader: body_loader_for(line_num, end_line)
+            )
+          elsif (match = stripped.match(/^impl(?:\s*<.*?>)?\s+([A-Za-z0-9_]+(?:\s+for\s+[A-Za-z0-9_]+)?)/))
+            current_impl = match[1].split(" for ").last.to_s.strip
+            impl_end_line = find_closing_brace(i)
+          elsif (match = stripped.match(/^(?:pub(?:\(crate\))?\s+)?(?:async\s+)?fn\s+([A-Za-z0-9_]+)\s*(?:<.*?>)?\s*\((.*?)\)/))
+            fn_name = match[1]
+            full_name = current_impl ? "#{current_impl}::#{fn_name}" : fn_name
+            end_line = stripped.end_with?(";") ? line_num : find_closing_brace(i)
+            body_for_calls = extract_lines(line_num, end_line)
+            calls = extract_calls(body_for_calls, fn_name)
+
+            # De-duplicate: the same logical symbol must be emitted exactly once.
+            # Without this guard an `impl Trait for X` block that also matched
+            # the generic `fn` rule earlier could re-emit a method as a free
+            # function, producing a duplicate-symbol collision in the graph.
+            next if symbols.any? { |s| s.name == full_name }
+
+            symbols << Symbol.new(
+              name: full_name,
+              kind: :function,
+              file: @file_path,
+              line_start: line_num,
+              line_end: end_line,
+              signature: line.split("{").first.to_s.strip,
+              calls: calls,
+              body_loader: body_loader_for(line_num, end_line)
+            )
+          end
+
+          i += 1
+        end
+
+        symbols
+      end
+
+      private
+
+      def extract_calls(body, self_name)
+        calls = body.scan(/\b([A-Za-z0-9_]+)(?:!|\s*\()/).flatten.uniq
+        calls.reject { |c| c == self_name || ["fn", "pub", "async", "if", "for", "while", "loop", "match", "return", "let", "mut"].include?(c) }
+      end
+
+      def find_closing_brace(start_idx)
+        brace_count = 0
+        found_open = false
+
+        (start_idx..(@lines.size - 1)).each do |idx|
+          line = @lines[idx]
+          line.chars.each do |char|
+            if char == "{"
+              brace_count += 1
+              found_open = true
+            elsif char == "}"
+              brace_count -= 1
+            end
+          end
+
+          return idx + 1 if found_open && brace_count <= 0
+        end
+
+        start_idx + 1
+      end
+
+      def extract_lines(start_line, end_line)
+        s_idx = [start_line - 1, 0].max
+        e_idx = [end_line - 1, @lines.size - 1].min
+        @lines[s_idx..e_idx].join
+      end
+    end
+  end
+end

data/lib/rubik/ast/symbol.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Rubik
+  module AST
+    class Symbol
+      attr_reader :name, :kind, :file, :line_start, :line_end, :signature, :calls, :docstring
+
+      # All string fields are scrubbed to valid UTF-8 at construction: a
+      # broken-encoding source file must never leak invalid bytes into JSON
+      # payloads (request bodies, cache writes) — they raise
+      # JSON::GeneratorError far away from the read that caused them.
+      def initialize(name:, kind:, file:, line_start:, line_end:, signature: nil, calls: [], body: nil, docstring: nil, body_loader: nil)
+        scrub = self.class.method(:scrub_if_invalid)
+        @name = scrub.call(name)
+        @kind = kind.to_sym
+        @file = file
+        @line_start = line_start
+        @line_end = line_end
+        @signature = scrub.call(signature || name)
+        @calls = (calls || []).map { |c| scrub.call(c) }
+        @body = scrub.call(body)
+        @body_loader = body_loader
+        @docstring = scrub.call(docstring)
+        @body_mutex = Mutex.new
+      end
+
+      # In-memory symbols carry the extracted body; cache-loaded symbols do
+      # not (bodies are excluded from the persistent index to keep it small).
+      # Extractors may also defer the body slice via `body_loader:` — the
+      # closure runs at most once, on first access, synchronized via @body_mutex.
+      # For symbols with neither, read the recorded line range from source on demand.
+      def body
+        return @body if @body
+
+        (@body_mutex ||= Mutex.new).synchronize do
+          return @body if @body
+
+          if @body_loader
+            loader = @body_loader
+            @body_loader = nil
+            @body = self.class.scrub_if_invalid(loader.call)
+          end
+          @body ||= self.class.read_source_range(@file, @line_start, @line_end) if @file && !@line_start.nil?
+          @body
+        end
+      end
+
+      def self.read_source_range(file, l_start, l_end)
+        return nil unless file && l_start && File.exist?(file)
+
+        lines = File.readlines(file)
+        selected = lines[(l_start - 1)..(l_end - 1)] if l_start >= 1
+        scrub_if_invalid(selected&.join)
+      rescue Errno::ENOENT, Errno::EACCES, Errno::EISDIR
+        nil
+      end
+
+      # JSON-safe view of a string: nil-safe. ASCII-8BIT (BINARY) tags pass
+      # `valid_encoding?` for ANY bytes yet still break JSON.dump, so they are
+      # re-tagged UTF-8 first; genuinely invalid sequences are then scrubbed.
+      def self.scrub_if_invalid(str)
+        return str if str.nil?
+
+        str = str.dup.force_encoding(Encoding::UTF_8) if str.encoding == Encoding::ASCII_8BIT
+        str.valid_encoding? ? str : str.scrub
+      end
+
+      # `include_body: false` skips the lazy source read — used by the cache
+      # writer, which never persists bodies (reading every indexed file just
+      # to drop the result would be pure waste).
+      def to_h(include_body: true)
+        {
+          name: @name,
+          kind: @kind,
+          file: @file,
+          line_start: @line_start,
+          line_end: @line_end,
+          line_range: [@line_start, @line_end],
+          signature: @signature,
+          calls: @calls,
+          body: include_body ? body : nil,
+          docstring: @docstring
+        }
+      end
+
+      def self.from_h(hash, default_file = nil)
+        l_start = hash["line_start"] || hash[:line_start] || (hash["line_range"] || hash[:line_range])&.first || 1
+        l_end = hash["line_end"] || hash[:line_end] || (hash["line_range"] || hash[:line_range])&.last || l_start
+        new(
+          name: hash["name"] || hash[:name],
+          kind: (hash["kind"] || hash[:kind] || :function).to_sym,
+          file: hash["file"] || hash[:file] || default_file,
+          line_start: l_start,
+          line_end: l_end,
+          signature: hash["signature"] || hash[:signature],
+          calls: hash["calls"] || hash[:calls] || [],
+          body: hash["body"] || hash[:body],
+          docstring: hash["docstring"] || hash[:docstring]
+        )
+      end
+    end
+  end
+end

data/lib/rubik/ast/treesitter/base_walker.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require_relative "../symbol"
+require_relative "../body_loader"
+
+module Rubik
+  module AST
+    module TreeSitter
+      class BaseWalker
+        include BodyLoader
+
+        attr_reader :file_path, :code
+
+        def initialize(file_path, code)
+          @file_path = file_path
+          @code = code || ""
+          @lines = nil
+        end
+
+        def lines
+          @lines ||= @code.lines
+        end
+
+        def extract_lines(start_line, end_line)
+          return "" if lines.empty?
+
+          s_idx = [start_line - 1, 0].max
+          e_idx = [end_line - 1, lines.size - 1].min
+          lines[s_idx..e_idx]&.join || ""
+        end
+
+        def text_for(node)
+          return "" unless node
+
+          @code.byteslice(node.start_byte...node.end_byte) || ""
+        end
+
+        def clean_signature(node, max_length: 300)
+          return "" unless node
+
+          first_line = text_for(node).lines.first.to_s.strip
+          clean = first_line.sub(/\{$/, "").strip
+          clean.length > max_length ? "#{clean[0...max_length]}..." : clean
+        end
+
+        def line_start(node)
+          node ? node.start_position.row + 1 : 1
+        end
+
+        def line_end(node)
+          node ? node.end_position.row + 1 : 1
+        end
+
+        def build_symbol(name:, kind:, node:, signature: nil, calls: [], docstring: nil)
+          l_start = line_start(node)
+          l_end = line_end(node)
+          sig = (signature || name).to_s
+          sig = "#{sig[0...300]}..." if sig.length > 300
+          Symbol.new(
+            name: name,
+            kind: kind,
+            file: @file_path,
+            line_start: l_start,
+            line_end: l_end,
+            signature: sig,
+            calls: calls,
+            body_loader: body_loader_for(l_start, l_end),
+            docstring: docstring
+          )
+        end
+
+        # Traverses an AST subtree iteratively to extract called function/method identifiers
+        def extract_calls(node, call_kinds = ["call", "call_expression", "invocation"])
+          calls = []
+          return calls unless node
+
+          stack = [node]
+          while (n = stack.pop)
+            if call_kinds.include?(n.kind)
+              fn = n.child_by_field_name("function") || n.child_by_field_name("name") || n.named_child(0)
+              if fn
+                fn_text = text_for(fn).strip
+                # Extract clean identifier name (e.g. "fmt.Println" -> "Println", "obj.method" -> "method")
+                fn_name = fn_text.split(".").last.to_s.split("::").last.to_s.strip
+                calls << fn_name if fn_name && !fn_name.empty?
+              end
+            end
+            (0...n.named_child_count).reverse_each { |i| stack.push(n.named_child(i)) }
+          end
+          calls.uniq
+        end
+
+        # Helper to extract leading comment or docstring
+        def docstring_before(node)
+          parent = node&.parent
+          return nil unless parent
+
+          idx = (0...parent.named_child_count).find { |i| parent.named_child(i) == node }
+          return nil unless idx&.positive?
+
+          prev = parent.named_child(idx - 1)
+          return nil unless prev&.kind&.include?("comment")
+
+          text_for(prev).strip
+        end
+      end
+    end
+  end
+end

data/lib/rubik/ast/treesitter/c_walker.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require_relative "base_walker"
+
+module Rubik
+  module AST
+    module TreeSitter
+      class CWalker < BaseWalker
+        def walk(root_node)
+          symbols = []
+          return symbols unless root_node
+
+          walk_children(root_node, symbols)
+          symbols
+        end
+
+        private
+
+        def walk_children(node, symbols)
+          (0...node.named_child_count).each do |i|
+            child = node.named_child(i)
+            case child.kind
+            when "preproc_include"
+              walk_include(child, symbols)
+            when "type_definition"
+              walk_typedef(child, symbols)
+            when "struct_specifier"
+              walk_struct(child, symbols)
+            when "function_definition"
+              walk_function(child, symbols)
+            else
+              walk_children(child, symbols)
+            end
+          end
+        end
+
+        def walk_include(node, symbols)
+          raw = text_for(node).strip
+          symbols << build_symbol(
+            name: raw,
+            kind: :import,
+            node: node,
+            signature: raw
+          )
+        end
+
+        def walk_typedef(node, symbols)
+          declarator = node.child_by_field_name("declarator")
+          name = if declarator
+                   text_for(declarator).strip
+                 else
+                   type_node = node.child_by_field_name("type")
+                   text_for(type_node&.child_by_field_name("name")).strip
+                 end
+          return if name.empty?
+
+          sig = clean_signature(node)
+          symbols << build_symbol(
+            name: name,
+            kind: :struct,
+            node: node,
+            signature: sig.empty? ? "typedef struct #{name}" : sig
+          )
+        end
+
+        def walk_struct(node, symbols)
+          name_node = node.child_by_field_name("name")
+          return unless name_node
+
+          name = text_for(name_node).strip
+          return if name.empty?
+
+          sig = "struct #{name}"
+          symbols << build_symbol(
+            name: name,
+            kind: :struct,
+            node: node,
+            signature: sig
+          )
+        end
+
+        def walk_function(node, symbols)
+          decl = node.child_by_field_name("declarator")
+          return unless decl
+
+          fn_name = extract_function_name(decl)
+          return if fn_name.nil? || fn_name.empty?
+
+          body = node.child_by_field_name("body")
+          calls = extract_calls(body, ["call_expression"])
+
+          sig = clean_signature(node)
+          symbols << build_symbol(
+            name: fn_name,
+            kind: :function,
+            node: node,
+            signature: sig.empty? ? "#{fn_name}()" : sig,
+            calls: calls
+          )
+        end
+
+        def extract_function_name(declarator_node)
+          current = declarator_node
+          while current
+            case current.kind
+            when "function_declarator", "pointer_declarator"
+              current = current.child_by_field_name("declarator")
+            when "identifier"
+              return text_for(current).strip
+            else
+              # Try first identifier child
+              ident = current.children.find { |c| c.kind == "identifier" }
+              return text_for(ident).strip if ident
+
+              return text_for(current).strip
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rubik/ast/treesitter/go_walker.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require_relative "base_walker"
+
+module Rubik
+  module AST
+    module TreeSitter
+      class GoWalker < BaseWalker
+        def walk(root_node)
+          symbols = []
+          return symbols unless root_node
+
+          walk_children(root_node, symbols)
+          symbols
+        end
+
+        private
+
+        def walk_children(node, symbols)
+          (0...node.named_child_count).each do |i|
+            child = node.named_child(i)
+            case child.kind
+            when "type_declaration"
+              walk_type_declaration(child, symbols)
+            when "type_spec"
+              walk_type_spec(child, symbols)
+            when "method_declaration"
+              walk_method(child, symbols)
+            when "function_declaration"
+              walk_function(child, symbols)
+            when "import_declaration"
+              walk_import(child, symbols)
+            else
+              walk_children(child, symbols)
+            end
+          end
+        end
+
+        def walk_type_declaration(node, symbols)
+          (0...node.named_child_count).each do |i|
+            child = node.named_child(i)
+            walk_type_spec(child, symbols) if child.kind == "type_spec"
+          end
+        end
+
+        def walk_type_spec(node, symbols)
+          name_node = node.child_by_field_name("name")
+          return unless name_node
+
+          name = text_for(name_node)
+          type_node = node.child_by_field_name("type")
+          kind = case type_node&.kind
+                 when "struct_type" then :struct
+                 when "interface_type" then :interface
+                 else :type
+                 end
+
+          sig = "type #{name} #{kind}"
+          symbols << build_symbol(
+            name: name,
+            kind: kind,
+            node: node,
+            signature: sig
+          )
+        end
+
+        def walk_method(node, symbols)
+          name_node = node.child_by_field_name("name")
+          recv_node = node.child_by_field_name("receiver")
+          return unless name_node && recv_node
+
+          method_name = text_for(name_node)
+          recv_text = text_for(recv_node).strip
+          full_name = "#{recv_text}.#{method_name}"
+
+          body = node.child_by_field_name("body")
+          calls = extract_calls(body, ["call_expression"])
+
+          sig_line = clean_signature(node)
+          symbols << build_symbol(
+            name: full_name,
+            kind: :function,
+            node: node,
+            signature: sig_line.empty? ? full_name : sig_line,
+            calls: calls
+          )
+        end
+
+        def walk_function(node, symbols)
+          name_node = node.child_by_field_name("name")
+          return unless name_node
+
+          fn_name = text_for(name_node)
+          body = node.child_by_field_name("body")
+          calls = extract_calls(body, ["call_expression"])
+
+          sig_line = clean_signature(node)
+          symbols << build_symbol(
+            name: fn_name,
+            kind: :function,
+            node: node,
+            signature: sig_line.empty? ? "func #{fn_name}()" : sig_line,
+            calls: calls
+          )
+        end
+
+        def walk_import(node, symbols)
+          raw = text_for(node).strip
+          name = raw.include?("(") ? "import (...)" : raw.lines.first.to_s.strip
+          symbols << build_symbol(
+            name: name,
+            kind: :import,
+            node: node,
+            signature: name
+          )
+        end
+      end
+    end
+  end
+end

data/lib/rubik/ast/treesitter/python_walker.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require_relative "base_walker"
+
+module Rubik
+  module AST
+    module TreeSitter
+      class PythonWalker < BaseWalker
+        def walk(root_node)
+          symbols = []
+          return symbols unless root_node
+
+          walk_children(root_node, symbols, nil)
+          symbols
+        end
+
+        private
+
+        def walk_children(node, symbols, current_class, depth: 0)
+          return if depth > 50
+
+          (0...node.named_child_count).each do |i|
+            child = node.named_child(i)
+            case child.kind
+            when "class_definition"
+              walk_class(child, symbols, depth: depth + 1)
+            when "function_definition"
+              walk_function(child, symbols, current_class)
+            when "import_statement", "import_from_statement"
+              walk_import(child, symbols)
+            else
+              walk_children(child, symbols, current_class, depth: depth + 1)
+            end
+          end
+        end
+
+        def walk_class(node, symbols, depth: 0)
+          name_node = node.child_by_field_name("name")
+          return unless name_node
+
+          class_name = text_for(name_node)
+          sig = text_for(node).lines.first.to_s.strip.sub(/:$/, "")
+          docstring = extract_docstring(node.child_by_field_name("body"))
+
+          symbols << build_symbol(
+            name: class_name,
+            kind: :class,
+            node: node,
+            signature: sig,
+            docstring: docstring
+          )
+
+          body = node.child_by_field_name("body")
+          walk_children(body, symbols, class_name, depth: depth + 1) if body
+        end
+
+        def walk_function(node, symbols, current_class)
+          name_node = node.child_by_field_name("name")
+          return unless name_node
+
+          fn_name = text_for(name_node)
+          full_name = current_class ? "#{current_class}.#{fn_name}" : fn_name
+          params = node.child_by_field_name("parameters")
+          sig = "def #{fn_name}#{text_for(params)}"
+          body = node.child_by_field_name("body")
+          calls = extract_calls(body, ["call"])
+          docstring = extract_docstring(body)
+
+          symbols << build_symbol(
+            name: full_name,
+            kind: :function,
+            node: node,
+            signature: sig,
+            calls: calls,
+            docstring: docstring
+          )
+        end
+
+        def walk_import(node, symbols)
+          text = text_for(node).strip
+          symbols << build_symbol(
+            name: text,
+            kind: :import,
+            node: node,
+            signature: text
+          )
+        end
+
+        def extract_docstring(body_node)
+          return nil unless body_node&.named_child_count&.positive?
+
+          first_stmt = body_node.named_child(0)
+          return nil unless first_stmt
+
+          str_node = if first_stmt.kind == "string"
+                       first_stmt
+                     elsif first_stmt.kind == "expression_statement" && first_stmt.named_child(0)&.kind == "string"
+                       first_stmt.named_child(0)
+                     end
+          return nil unless str_node
+
+          text_for(str_node).gsub(/^["']{3}|["']{3}$/, "").strip
+        end
+      end
+    end
+  end
+end