rubik 0.1.0

data/lib/rubik/ast/c_extractor.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require_relative "symbol"
+require_relative "body_loader"
+
+module Rubik
+  module AST
+    class CExtractor
+      include BodyLoader
+
+      EXCLUDED_CALLS = Set.new(["if", "for", "while", "switch", "return", "sizeof", "malloc", "free", "printf"]).freeze
+
+      def self.extract(file_path, content = nil)
+        new(file_path, content).extract
+      end
+
+      def initialize(file_path, content = nil)
+        @file_path = file_path
+        @content = content || (File.exist?(file_path) ? File.read(file_path) : "")
+        @lines = @content.lines
+      end
+
+      def extract
+        symbols = []
+        return symbols if @content.empty?
+
+        i = 0
+        while i < @lines.size
+          line = @lines[i]
+          line_num = i + 1
+          stripped = line.strip
+
+          if stripped.start_with?("#include")
+            symbols << Symbol.new(
+              name: stripped,
+              kind: :import,
+              file: @file_path,
+              line_start: line_num,
+              line_end: line_num,
+              signature: stripped
+            )
+          elsif (match = stripped.match(/^(?:typedef\s+)?(struct|union|enum)\s+([A-Za-z0-9_]+)?\s*\{?/))
+            name = match[2] || "anonymous_#{kind}_#{line_num}"
+            end_line = stripped.include?(";") && !stripped.include?("{") ? line_num : find_closing_brace(i)
+            symbols << Symbol.new(
+              name: name,
+              kind: match[1].to_sym,
+              file: @file_path,
+              line_start: line_num,
+              line_end: end_line,
+              signature: line.split("{").first.to_s.strip,
+              body_loader: body_loader_for(line_num, end_line)
+            )
+          elsif (match = stripped.match(/^([A-Za-z0-9_*\s]+)\s+([A-Za-z0-9_]+)\s*\((.*?)\)\s*\{/))
+            fn_name = match[2]
+            unless ["if", "for", "while", "switch"].include?(fn_name)
+              end_line = find_closing_brace(i)
+              body = extract_lines(line_num, end_line)
+              calls = extract_calls(body, fn_name)
+
+              symbols << Symbol.new(
+                name: fn_name,
+                kind: :function,
+                file: @file_path,
+                line_start: line_num,
+                line_end: end_line,
+                signature: line.split("{").first.to_s.strip,
+                calls: calls,
+                body_loader: body_loader_for(line_num, end_line)
+              )
+            end
+          end
+
+          i += 1
+        end
+
+        symbols
+      end
+
+      private
+
+      def extract_calls(body, self_name)
+        body.scan(/\b[A-Za-z0-9_]+(?=\s*\()/).uniq.delete_if { |c| c == self_name || EXCLUDED_CALLS.include?(c) }
+      end
+
+      def find_closing_brace(start_idx)
+        brace_count = 0
+        found_open = false
+
+        (start_idx..(@lines.size - 1)).each do |idx|
+          line = @lines[idx]
+          line.each_char do |char|
+            if char == "{"
+              brace_count += 1
+              found_open = true
+            elsif char == "}"
+              brace_count -= 1
+            end
+          end
+
+          return idx + 1 if found_open && brace_count <= 0
+        end
+
+        start_idx + 1
+      end
+
+      def extract_lines(start_line, end_line)
+        s_idx = [start_line - 1, 0].max
+        e_idx = [end_line - 1, @lines.size - 1].min
+        @lines[s_idx..e_idx].join
+      end
+    end
+  end
+end

data/lib/rubik/ast/cache_store.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+require "digest"
+require_relative "symbol"
+require_relative "../platform"
+
+module Rubik
+  module AST
+    class CacheStore
+      LANGUAGE_MAP = {
+        ".rb" => "ruby",
+        ".py" => "python",
+        ".go" => "go",
+        ".rs" => "rust",
+        ".zig" => "zig",
+        ".c" => "c",
+        ".h" => "c",
+        ".cpp" => "cpp",
+        ".hpp" => "cpp",
+        ".cc" => "cpp",
+        ".ts" => "typescript",
+        ".tsx" => "typescript",
+        ".js" => "javascript",
+        ".jsx" => "javascript"
+      }.freeze
+
+      attr_reader :root_dir, :home_dir
+
+      def initialize(root_dir: Dir.pwd, home_dir: nil)
+        @root_dir = File.expand_path(root_dir)
+        @home_dir = home_dir ? File.expand_path(home_dir) : Rubik::Platform.home_dir
+      end
+
+      def cache_path
+        rubik_dir = File.join(@root_dir, ".rubik")
+        primary = File.join(rubik_dir, "cache", "ast_index.json")
+        return primary if File.directory?(rubik_dir) && (File.writable?(rubik_dir) || (File.exist?(primary) && File.writable?(primary)))
+
+        project_hash = Digest::SHA256.hexdigest(@root_dir)[0..11]
+        File.join(@home_dir, ".rubik", "cache", project_hash, "ast_index.json")
+      end
+
+      def save(file_symbols_hash)
+        return false unless file_symbols_hash.is_a?(Hash)
+
+        data = {
+          "version" => 1,
+          "last_scan" => Time.now.to_i,
+          "files" => {}
+        }
+
+        file_symbols_hash.each do |file_path, syms|
+          next if syms.nil? || syms.empty?
+
+          full_path = File.expand_path(file_path, @root_dir)
+
+          stat = begin
+            File.stat(full_path)
+          rescue Errno::ENOENT, Errno::EACCES
+            nil
+          end
+          next unless stat&.file?
+
+          norm_root = Rubik::Platform.normalize_path_raw(@root_dir)
+          norm_full = Rubik::Platform.normalize_path_raw(full_path)
+          rel_path = if norm_full.start_with?("#{norm_root}/")
+                       norm_full.delete_prefix("#{norm_root}/")
+                     else
+                       file_path.to_s.tr("\\", "/")
+                     end
+
+          ext = File.extname(full_path).downcase
+          lang = LANGUAGE_MAP[ext] || "unknown"
+
+          digest = begin
+            Digest::SHA1.file(full_path).hexdigest
+          rescue Errno::ENOENT, Errno::EACCES
+            nil
+          end
+
+          # Bodies are intentionally NOT persisted: they dominate the cache
+          # size and are re-extracted lazily from source via Symbol#body.
+          data["files"][rel_path] = {
+            "language" => lang,
+            "mtime" => stat.mtime.to_i,
+            "mtime_f" => stat.mtime.to_f,
+            "size" => stat.size,
+            "digest" => digest,
+            "symbols" => syms.map { |sym| sym.to_h(include_body: false) }
+          }
+        end
+
+        target_path = cache_path
+        Rubik::Platform.atomic_write(target_path, JSON.generate(data))
+        true
+      rescue StandardError => e
+        Rubik::Log.warn(:ast, "cache save failed (#{e.class}: #{e.message}); next index rebuild starts from scratch")
+        false
+      end
+
+      def load
+        target_path = cache_path
+        return {} unless File.exist?(target_path)
+
+        content = File.read(target_path)
+        data = JSON.parse(content)
+        return {} unless data.is_a?(Hash) && data["files"].is_a?(Hash)
+
+        result = {}
+        data["files"].each do |rel_path, cached_info|
+          full_path = File.expand_path(rel_path, @root_dir)
+          next unless valid_entry?(full_path, cached_info)
+
+          symbols_raw = cached_info["symbols"] || []
+          symbols = symbols_raw.map do |s|
+            line_start = s["line_start"] || (s["line_range"] && s["line_range"][0]) || 1
+            line_end = s["line_end"] || (s["line_range"] && s["line_range"][1]) || line_start
+            Symbol.new(
+              name: s["name"],
+              kind: (s["kind"] || :function).to_sym,
+              file: full_path,
+              line_start: line_start,
+              line_end: line_end,
+              signature: s["signature"],
+              calls: s["calls"] || [],
+              body: s["body"],
+              docstring: s["docstring"]
+            )
+          end
+          result[full_path] = symbols
+        end
+
+        result
+      rescue JSON::ParserError, Errno::ENOENT, Errno::EACCES, TypeError, NoMethodError
+        # Corrupt-but-valid JSON shapes raise TypeError/NoMethodError inside
+        # the entry loop; EACCES covers an unreadable cache file. All mean
+        # "cache unusable" → rebuild from scratch.
+        {}
+      end
+
+      def valid_entry?(file_path, cached_info)
+        return false unless cached_info.is_a?(Hash)
+
+        full_path = File.expand_path(file_path, @root_dir)
+        return false unless File.exist?(full_path) && File.file?(full_path)
+
+        stat = File.stat(full_path)
+        cached_size = (cached_info["size"] || cached_info[:size]).to_i
+        return false unless stat.size == cached_size
+
+        cached_mtime_f = cached_info["mtime_f"]&.to_f
+        cached_mtime = (cached_info["mtime"] || cached_info[:mtime]).to_i
+
+        mtime_matches = if cached_mtime_f
+                          (stat.mtime.to_f - cached_mtime_f).abs < 0.0001
+                        else
+                          stat.mtime.to_i == cached_mtime
+                        end
+
+        return false unless mtime_matches
+
+        cached_digest = cached_info["digest"] || cached_info[:digest]
+        return false unless cached_digest
+
+        current_digest = begin
+          Digest::SHA1.file(full_path).hexdigest
+        rescue Errno::ENOENT, Errno::EACCES
+          nil
+        end
+        return false unless current_digest == cached_digest
+
+        true
+      rescue SystemCallError, NoMethodError, ArgumentError
+        # ArgumentError: a NUL-byte rel_path from a corrupt cache entry is
+        # one invalid entry, not a reason to rebuild the whole cache.
+        false
+      end
+      alias valid_cache_entry? valid_entry?
+
+      def clear!
+        target_path = cache_path
+        FileUtils.rm_f(target_path)
+        true
+      rescue SystemCallError
+        false
+      end
+    end
+  end
+end

data/lib/rubik/ast/extractor.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require_relative "ruby_extractor"
+require_relative "go_extractor"
+require_relative "rust_extractor"
+require_relative "zig_extractor"
+require_relative "c_extractor"
+require_relative "python_extractor"
+require_relative "typescript_extractor"
+require_relative "treesitter_extractor"
+
+module Rubik
+  module AST
+    class Extractor
+      FALLBACK_EXTRACTORS = {
+        ".rb" => RubyExtractor,
+        ".go" => GoExtractor,
+        ".rs" => RustExtractor,
+        ".zig" => ZigExtractor,
+        ".c" => CExtractor,
+        ".h" => CExtractor,
+        ".cpp" => CExtractor,
+        ".hpp" => CExtractor,
+        ".py" => PythonExtractor,
+        ".ts" => TypeScriptExtractor,
+        ".tsx" => TypeScriptExtractor,
+        ".js" => TypeScriptExtractor,
+        ".jsx" => TypeScriptExtractor
+      }.freeze
+
+      EXTENSIONS = FALLBACK_EXTRACTORS
+
+      def self.extract(file_path, content = nil)
+        ext = File.extname(file_path).downcase
+
+        # 1. Ruby files: always use Prism-powered RubyExtractor first
+        return RubyExtractor.extract(file_path, content) if ext == ".rb"
+
+        # 2. Polyglot files: use Tree-Sitter AST extractor if available
+        if TreeSitterExtractor.available? && TreeSitterExtractor.supports?(ext)
+          symbols = TreeSitterExtractor.extract(file_path, content)
+          return symbols unless symbols.nil?
+        end
+
+        # 3. Graceful Fallback: use legacy regex extractor
+        extractor_class = FALLBACK_EXTRACTORS[ext] || RubyExtractor
+        extractor_class.extract(file_path, content)
+      rescue StandardError, ScriptError => e
+        warn "[WARN] Extraction failed for #{file_path}: #{e.class}: #{e.message}" if ENV["RUBIK_DEBUG"]
+        # Graceful fallback to empty symbols on syntax/parsing anomalies
+        []
+      end
+
+      def self.supported_extensions
+        EXTENSIONS.keys
+      end
+    end
+  end
+end

data/lib/rubik/ast/fallback_cleaner.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module Rubik
+  module AST
+    # Quote/comment/heredoc-aware view of Ruby source, used only by
+    # RubyExtractor's Prism-less fallback scanner (Plan 54 D-7). This is
+    # deliberately a "lexer-lite", not a parser: it blanks string bodies,
+    # comments, regex literals, heredoc bodies, =begin/=end blocks,
+    # :keyword symbol literals and everything after __END__ so naive
+    # end-matching cannot be fooled by them. Known, accepted limits:
+    # %-literals (%w[end]), multiple heredocs per line, char classes
+    # containing slashes inside regexes, and `x <<identifier` (no space)
+    # being read as a heredoc.
+    module FallbackCleaner
+      SYMBOL_KEYWORDS = /:(?:class|module|def|do|if|unless|case|begin|while|for|until|end)\b/
+      HEREDOC_START = /\A<<([~-]?)([A-Za-z_]\w*)/
+      REGEX_PRECEDERS = "(=,~!&|{[;:^"
+      REGEX_KEYWORDS = ["and", "begin", "case", "do", "else", "elsif", "if", "not", "or", "return", "then", "unless", "until", "while"].freeze
+
+      module_function
+
+      def code_lines(raw_lines)
+        cleaned = []
+        quote = nil
+        terminator = nil
+        cut = false
+
+        raw_lines.each do |raw|
+          if cut || terminator
+            terminator = nil if heredoc_closed?(raw, terminator)
+            cleaned << blank(raw)
+          elsif raw.match?(/\A\s*=/) # =begin/=end block comments
+            cleaned << blank(raw)
+          else
+            line, quote, terminator = clean_line(raw, quote)
+            cleaned << blank_symbol_keywords(line)
+          end
+
+          cut = true if !cut && terminator.nil? && cleaned.last.strip == "__END__"
+        end
+        cleaned
+      end
+
+      # Cleans one line of code. Returns [cleaned_line, open_quote,
+      # heredoc_terminator]; an open quote carries across lines so
+      # multi-line string bodies are blanked too. Delimiters are kept,
+      # contents are blanked; comments and heredoc openers blank the rest
+      # of the line, regex literals blank their span.
+      def clean_line(raw, quote)
+        out = +""
+        i = 0
+        while i < raw.length
+          ch = raw[i]
+          if quote
+            if ch == "\\" && i + 1 < raw.length
+              out << "  "
+              i += 2
+              next
+            end
+            quote = nil if ch == quote
+            out << (quote ? " " : ch)
+            i += 1
+          elsif ch == "/" && (fin = regex_span_end(raw, i))
+            out << blank(raw[i..fin])
+            i = fin + 1
+          elsif ch == "?" && "\"'`".include?(raw[i + 1].to_s) # ?" ?' ?` char literals
+            out << raw[i, 2]
+            i += 2
+          else
+            case ch
+            when "#"
+              out << blank(raw[i..])
+              return [out, quote, nil]
+            when '"', "'", "`"
+              quote = ch
+              out << ch
+              i += 1
+            when "<"
+              heredoc = heredoc_at(raw, i)
+              return [out << blank(raw[i..]), nil, heredoc] if heredoc
+
+              out << ch
+              i += 1
+            else
+              out << ch
+              i += 1
+            end
+          end
+        end
+        [out, quote, nil]
+      end
+
+      def blank(str)
+        " " * str.length
+      end
+
+      # Returns [terminator_id, indent_allowed] when raw[pos..] opens a
+      # heredoc, else nil.
+      def heredoc_at(raw, pos)
+        m = raw[pos..].match(HEREDOC_START)
+        [m[2], m[1] != ""] if m
+      end
+
+      # A plain <<EOS heredoc only terminates at column zero; <<-/<<~
+      # allow an indented terminator.
+      def heredoc_closed?(raw, terminator)
+        id, indent_allowed = terminator
+        return false unless raw.strip == id
+
+        indent_allowed || raw !~ /\A[ \t]/
+      end
+
+      # Index of the closing "/" of a regex literal opened at raw[start],
+      # nil when the "/" is division rather than an opener.
+      def regex_span_end(raw, start)
+        return nil unless regex_context?(raw, start)
+
+        j = start + 1
+        while j < raw.length
+          j += 1 if raw[j] == "\\"
+          break if raw[j] == "/"
+
+          j += 1
+        end
+        [j, raw.length - 1].min
+      end
+
+      # Crude regex-opener detection: a "/" starts a regex when the previous
+      # significant char is an operator/opening bracket, line start, or one
+      # of the keywords that takes a bare expression.
+      def regex_context?(raw, start)
+        k = start - 1
+        k -= 1 while !k.negative? && " \t\n".include?(raw[k])
+        return true if k.negative?
+        return true if REGEX_PRECEDERS.include?(raw[k])
+
+        REGEX_KEYWORDS.include?(raw[0..k][/([A-Za-z_][A-Za-z0-9_]*)\z/])
+      end
+
+      def blank_symbol_keywords(line)
+        line.gsub(SYMBOL_KEYWORDS) { |m| " " * m.length }
+      end
+    end
+  end
+end

data/lib/rubik/ast/go_extractor.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require_relative "symbol"
+require_relative "body_loader"
+
+module Rubik
+  module AST
+    class GoExtractor
+      include BodyLoader
+
+      def self.extract(file_path, content = nil)
+        new(file_path, content).extract
+      end
+
+      def initialize(file_path, content = nil)
+        @file_path = file_path
+        @content = content || (File.exist?(file_path) ? File.read(file_path) : "")
+        @lines = @content.lines
+      end
+
+      def extract
+        symbols = []
+        return symbols if @content.empty?
+
+        i = 0
+        while i < @lines.size
+          line = @lines[i]
+          line_num = i + 1
+          stripped = line.strip
+
+          if stripped.start_with?("import ")
+            sym, i = parse_import(stripped, line_num, i)
+            symbols << sym
+          elsif (match = stripped.match(/^type\s+([A-Za-z0-9_]+)\s+(struct|interface)\b/))
+            symbols << parse_type(match, line_num, i)
+          elsif (match = stripped.match(/^func\s+(?:\(([A-Za-z0-9_*\s]+)\)\s+)?([A-Za-z0-9_]+)\s*\((.*?)\)/))
+            symbols << parse_func(line, match, line_num, i)
+          end
+
+          i += 1
+        end
+
+        symbols
+      end
+
+      private
+
+      def parse_import(stripped, line_num, index)
+        if stripped.include?("(")
+          end_line = find_closing_paren(index)
+          symbol = Symbol.new(
+            name: "import (...)",
+            kind: :import,
+            file: @file_path,
+            line_start: line_num,
+            line_end: end_line,
+            signature: "import (...)"
+          )
+          [symbol, end_line - 1]
+        else
+          symbol = Symbol.new(
+            name: stripped,
+            kind: :import,
+            file: @file_path,
+            line_start: line_num,
+            line_end: line_num,
+            signature: stripped
+          )
+          [symbol, index]
+        end
+      end
+
+      def parse_type(match, line_num, index)
+        name = match[1]
+        kind = match[2] == "struct" ? :struct : :interface
+        end_line = find_closing_brace(index)
+        Symbol.new(
+          name: name,
+          kind: kind,
+          file: @file_path,
+          line_start: line_num,
+          line_end: end_line,
+          signature: "type #{name} #{match[2]}",
+          body_loader: body_loader_for(line_num, end_line)
+        )
+      end
+
+      def parse_func(line, match, line_num, index)
+        receiver = match[1]&.strip
+        fn_name = match[2].strip
+        full_name = receiver ? "(#{receiver}).#{fn_name}" : fn_name
+        end_line = find_closing_brace(index)
+        body_for_calls = extract_lines(line_num, end_line)
+        calls = extract_calls(body_for_calls, fn_name)
+
+        Symbol.new(
+          name: full_name,
+          kind: :function,
+          file: @file_path,
+          line_start: line_num,
+          line_end: end_line,
+          signature: line.split("{").first.to_s.strip,
+          calls: calls,
+          body_loader: body_loader_for(line_num, end_line)
+        )
+      end
+
+      def extract_calls(body, self_name)
+        calls = body.scan(/\b([A-Za-z0-9_]+)\s*\(/).flatten.uniq
+        calls.reject { |c| c == self_name || ["func", "if", "for", "return", "make", "append", "len", "cap", "panic", "recover"].include?(c) }
+      end
+
+      def find_closing_brace(start_idx)
+        brace_count = 0
+        found_open = false
+
+        (start_idx..(@lines.size - 1)).each do |idx|
+          line = @lines[idx]
+          line.each_char do |char|
+            if char == "{"
+              brace_count += 1
+              found_open = true
+            elsif char == "}"
+              brace_count -= 1
+            end
+          end
+
+          return idx + 1 if found_open && brace_count <= 0
+        end
+
+        start_idx + 1
+      end
+
+      def find_closing_paren(start_idx)
+        (start_idx..(@lines.size - 1)).each do |idx|
+          return idx + 1 if @lines[idx].include?(")")
+        end
+        start_idx + 1
+      end
+
+      def extract_lines(start_line, end_line)
+        s_idx = [start_line - 1, 0].max
+        e_idx = [end_line - 1, @lines.size - 1].min
+        @lines[s_idx..e_idx].join
+      end
+    end
+  end
+end