glancer 1.0.0

data/lib/glancer/indexer/context_indexer.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Glancer
+  module Indexer
+    module ContextIndexer
+      module_function
+
+      def index!
+        Glancer::Utils::Logger.info("Indexer::ContextIndexer", "Starting context indexing...")
+
+        path = Glancer.configuration.context_file_path
+
+        if path.nil? || !File.exist?(path)
+          Glancer::Utils::Logger.error("Indexer::ContextIndexer", "Context file not found at path: #{path.inspect}")
+          raise Glancer::Error, "Context file not found. Expected at: #{path.inspect}"
+        end
+
+        Glancer::Utils::Logger.debug("Indexer::ContextIndexer", "Reading context file from: #{path}")
+        content = File.read(path)
+        Glancer::Utils::Logger.debug("Indexer::ContextIndexer", "Read #{content.bytesize} bytes from context file")
+
+        if content.lines.first&.strip == "--glancer-ignore"
+          Glancer::Utils::Logger.info("Indexer::ContextIndexer",
+                                      "Context file marked with --glancer-ignore, skipping indexing.")
+          return []
+        end
+
+        chunks = split_into_chunks(content)
+        Glancer::Utils::Logger.info("Indexer::ContextIndexer", "Split content into #{chunks.size} chunk(s)")
+
+        chunks.map do |chunk|
+          {
+            content: chunk,
+            source_type: "context",
+            source_path: path
+          }
+        end
+      rescue StandardError => e
+        Glancer::Utils::Logger.error("Indexer::ContextIndexer",
+                                     "Failed to index context: #{e.class} - #{e.message}")
+        Glancer::Utils::Logger.debug("Indexer::ContextIndexer", "Backtrace:\n#{e.backtrace.join("\n")}")
+        raise Glancer::Error, "Context indexing failed: #{e.message}"
+      end
+
+      def split_into_chunks(text)
+        size = Glancer.configuration.chunk_size
+        overlap = Glancer.configuration.chunk_overlap
+        chunks = []
+        start = 0
+        while start < text.length
+          chunks << text[start, size]
+          start += size - overlap
+        end
+        chunks
+      end
+    end
+  end
+end

data/lib/glancer/indexer/model_indexer.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Glancer
+  module Indexer
+    module ModelIndexer
+      module_function
+
+      def index!
+        Glancer::Utils::Logger.info("Indexer::ModelIndexer", "Starting model indexing...")
+
+        model_files = Dir[Rails.root.join("app/models/**/*.rb")]
+
+        if model_files.empty?
+          Glancer::Utils::Logger.warn("Indexer::ModelIndexer", "No model files found for indexing.")
+          return []
+        end
+
+        Glancer::Utils::Logger.info("Indexer::ModelIndexer", "Found #{model_files.size} model file(s)")
+
+        all_chunks = []
+
+        model_files.each do |file|
+          Glancer::Utils::Logger.debug("Indexer::ModelIndexer", "Reading model file: #{file}")
+
+          content = File.read(file)
+          Glancer::Utils::Logger.debug("Indexer::ModelIndexer", "Read #{content.bytesize} bytes from #{file}")
+
+          chunks = split_into_chunks(content)
+          Glancer::Utils::Logger.debug("Indexer::ModelIndexer", "Split into #{chunks.size} chunk(s)")
+
+          all_chunks.concat(
+            chunks.map do |chunk|
+              {
+                content: chunk,
+                source_type: "models",
+                source_path: file
+              }
+            end
+          )
+        end
+
+        Glancer::Utils::Logger.info("Indexer::ModelIndexer", "Completed indexing. Total chunks: #{all_chunks.size}")
+
+        all_chunks
+      rescue StandardError => e
+        Glancer::Utils::Logger.error("Indexer::ModelIndexer", "Model indexing failed: #{e.class} - #{e.message}")
+        Glancer::Utils::Logger.debug("Indexer::ModelIndexer", "Backtrace:\n#{e.backtrace.join("\n")}")
+        raise Glancer::Error, "Model indexing failed: #{e.message}"
+      end
+
+      def split_into_chunks(text)
+        size = Glancer.configuration.chunk_size
+        overlap = Glancer.configuration.chunk_overlap
+        chunks = []
+        start = 0
+        while start < text.length
+          chunks << text[start, size]
+          start += size - overlap
+        end
+        chunks
+      end
+    end
+  end
+end

data/lib/glancer/indexer/schema_indexer.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+module Glancer
+  module Indexer
+    module SchemaIndexer
+      module_function
+
+      def index!
+        Glancer::Utils::Logger.info("Indexer::SchemaIndexer", "Starting schema indexing...")
+
+        schema_file = Rails.root.join("db/schema.rb")
+
+        unless File.exist?(schema_file)
+          Glancer::Utils::Logger.warn("Indexer::SchemaIndexer", "Schema file not found at: #{schema_file}")
+          return []
+        end
+
+        Glancer::Utils::Logger.debug("Indexer::SchemaIndexer", "Reading schema file from: #{schema_file}")
+
+        content = File.read(schema_file)
+        Glancer::Utils::Logger.debug("Indexer::SchemaIndexer", "Read #{content.bytesize} bytes from schema file")
+
+        eager_load_models!
+
+        chunks = split_into_chunks(content)
+        Glancer::Utils::Logger.info("Indexer::SchemaIndexer", "Found #{chunks.size} table definition(s) in schema")
+
+        indexed_chunks = chunks.map do |chunk|
+          table_name = extract_table_name(chunk)
+          if table_name
+            Glancer::Utils::Logger.debug("Indexer::SchemaIndexer", "Indexed table: #{table_name}")
+            enriched = chunk + model_associations_block(table_name)
+            {
+              content: enriched,
+              source_type: "schema",
+              source_path: "#{schema_file}##{table_name}"
+            }
+          else
+            Glancer::Utils::Logger.warn("Indexer::SchemaIndexer", "Could not extract table name from chunk")
+            nil
+          end
+        end.compact
+
+        fk_chunk = extract_foreign_keys(content, schema_file)
+        indexed_chunks << fk_chunk if fk_chunk
+
+        inflections_chunk = extract_inflections
+        indexed_chunks << inflections_chunk if inflections_chunk
+
+        Glancer::Utils::Logger.info("Indexer::SchemaIndexer",
+                                    "Completed schema indexing. Total indexed chunks: #{indexed_chunks.size}")
+
+        indexed_chunks
+      rescue StandardError => e
+        Glancer::Utils::Logger.error("Indexer::SchemaIndexer", "Schema indexing failed: #{e.class} - #{e.message}")
+        Glancer::Utils::Logger.debug("Indexer::SchemaIndexer", "Backtrace:\n#{e.backtrace.join("\n")}")
+        raise Glancer::Error, "Schema indexing failed: #{e.message}"
+      end
+
+      def eager_load_models!
+        Rails.application.eager_load!
+        Glancer::Utils::Logger.debug("Indexer::SchemaIndexer", "Models eager-loaded for association reflection")
+      rescue StandardError => e
+        Glancer::Utils::Logger.warn("Indexer::SchemaIndexer", "Could not eager-load models: #{e.message}")
+      end
+
+      def find_model_for_table(table_name)
+        candidates = ActiveRecord::Base.descendants.select do |model|
+          !model.abstract_class? &&
+            !model.name&.start_with?("Glancer::") &&
+            model.table_name == table_name
+        end
+        return nil if candidates.empty?
+
+        candidates.find { |m| m.superclass.abstract_class? || m.superclass == ActiveRecord::Base } || candidates.first
+      rescue StandardError
+        nil
+      end
+
+      def model_associations_block(table_name)
+        model = find_model_for_table(table_name)
+        return "" unless model
+
+        assocs = model.reflect_on_all_associations
+        return "" if assocs.empty?
+
+        lines = assocs.filter_map do |assoc|
+          format_association(assoc)
+        rescue StandardError
+          nil
+        end
+        return "" if lines.empty?
+
+        "\n\n# ActiveRecord Associations (#{model.name}):\n#{lines.join("\n")}"
+      end
+
+      def format_association(assoc)
+        parts = ["  #{assoc.macro} :#{assoc.name}"]
+        opts = ["class_name: \"#{assoc.class_name}\""]
+
+        fk = assoc.foreign_key.to_s
+        opts << "foreign_key: \"#{fk}\"" if fk.present?
+        opts << "through: :#{assoc.options[:through]}" if assoc.options[:through].present?
+        opts << "polymorphic: true" if assoc.options[:polymorphic]
+        opts << "as: :#{assoc.options[:as]}" if assoc.options[:as].present?
+        opts << "source: :#{assoc.options[:source]}" if assoc.options[:source].present?
+        opts << "dependent: :#{assoc.options[:dependent]}" if assoc.options[:dependent].present?
+
+        "#{parts.join} (#{opts.join(", ")})"
+      end
+
+      def extract_inflections
+        inflections_file = Rails.root.join("config/initializers/inflections.rb")
+        return nil unless File.exist?(inflections_file)
+
+        raw = File.read(inflections_file)
+        return nil unless raw.lines.any? { |l| l.strip.match?(/\binflect\.\w/) }
+
+        Glancer::Utils::Logger.debug("Indexer::SchemaIndexer", "Found custom inflections, adding as schema chunk")
+        {
+          content: "# Custom Rails Inflections\n# These control plural/singular model name mapping.\n\n#{raw.strip}",
+          source_type: "schema",
+          source_path: inflections_file.to_s
+        }
+      rescue StandardError => e
+        Glancer::Utils::Logger.warn("Indexer::SchemaIndexer", "Could not read inflections: #{e.message}")
+        nil
+      end
+
+      def split_into_chunks(schema_text)
+        schema_text.split(/^  create_table /).map do |chunk|
+          next if chunk.strip.empty?
+
+          "create_table #{chunk.strip}"
+        end.compact
+      end
+
+      def extract_table_name(chunk)
+        chunk[/create_table ["']?([a-zA-Z0-9_]+)["']?/, 1]
+      end
+
+      def extract_foreign_keys(schema_text, schema_file)
+        lines = schema_text.lines.select { |l| l.strip.start_with?("add_foreign_key") }
+        return nil if lines.empty?
+
+        relationships = lines.filter_map do |line|
+          # add_foreign_key "orders", "users", column: "user_id"
+          # add_foreign_key "order_items", "orders"
+          m = line.match(/add_foreign_key ["'](\w+)["'],\s*["'](\w+)["'](?:.*column:\s*["'](\w+)["'])?/)
+          next unless m
+
+          child_table = m[1]
+          parent_table = m[2]
+          column = m[3] || "#{parent_table.singularize}_id"
+          "#{child_table}.#{column} → #{parent_table}.id"
+        end
+
+        return nil if relationships.empty?
+
+        content = "# Foreign Key Relationships\n#{relationships.join("\n")}"
+        Glancer::Utils::Logger.debug("Indexer::SchemaIndexer", "Extracted #{relationships.size} foreign key(s)")
+
+        {
+          content: content,
+          source_type: "schema",
+          source_path: "#{schema_file}#foreign_keys"
+        }
+      end
+    end
+  end
+end

data/lib/glancer/indexer.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative "indexer/schema_indexer"
+require_relative "indexer/model_indexer"
+require_relative "indexer/context_indexer"
+
+module Glancer
+  module Indexer
+    module_function
+
+    def rebuild_all!
+      Glancer::Utils::Logger.info("Indexer", "Starting full index rebuild...")
+
+      chunks = []
+
+      if Glancer.configuration.schema_permission
+        Glancer::Utils::Logger.info("Indexer", "Indexing schema (enabled by configuration)...")
+        chunks += SchemaIndexer.index!
+      else
+        Glancer::Utils::Logger.debug("Indexer", "Schema indexing is disabled in configuration.")
+      end
+
+      if Glancer.configuration.models_permission
+        Glancer::Utils::Logger.info("Indexer", "Indexing models (enabled by configuration)...")
+        chunks += ModelIndexer.index!
+      else
+        Glancer::Utils::Logger.debug("Indexer", "Model indexing is disabled in configuration.")
+      end
+
+      if Glancer.configuration.context_file_path
+        Glancer::Utils::Logger.info("Indexer", "Indexing context file (path configured)...")
+        chunks += ContextIndexer.index!
+      else
+        Glancer::Utils::Logger.debug("Indexer", "No context file path configured. Skipping context indexing.")
+      end
+
+      Glancer::Utils::Logger.info("Indexer", "Indexing completed. Total chunks: #{chunks.size}")
+
+      Glancer::Utils::Logger.debug("Indexer", "Storing documents into retriever...")
+      Retriever.store_documents(chunks)
+      Glancer::Utils::Logger.info("Indexer", "Documents stored successfully.")
+
+      chunks
+    rescue StandardError => e
+      Glancer::Utils::Logger.error("Indexer", "Index rebuilding failed: #{e.class} - #{e.message}")
+      Glancer::Utils::Logger.debug("Indexer", "Backtrace:\n#{e.backtrace.join("\n")}")
+      raise Glancer::Error, "Index rebuilding failed: #{e.message}"
+    end
+  end
+end

data/lib/glancer/retriever.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require "ruby_llm"
+
+module Glancer
+  module Retriever
+    module_function
+
+    def store_documents(chunks_with_metadata)
+      Glancer::Utils::Logger.info("Retriever", "Storing #{chunks_with_metadata.size} document chunk(s)...")
+
+      chunks_with_metadata.each_with_index do |data, idx|
+        chunk = data[:content]
+        preview = chunk[0..50].gsub(/\s+/, " ").strip
+
+        Glancer::Utils::Logger.debug("Retriever",
+                                     "Embedding chunk ##{idx + 1} (#{data[:source_type]} - #{data[:source_path]}): '#{preview}...'")
+
+        vector = RubyLLM.embed(
+          chunk,
+          model: Glancer.configuration.resolved_embedding_model,
+          provider: Glancer.configuration.resolved_embedding_provider,
+          assume_model_exists: true
+        ).vectors
+
+        Glancer::Utils::Logger.debug("Retriever",
+                                     "Vector size: #{vector.size}, example values: #{vector.first(5).inspect}")
+
+        Glancer::Embedding.create!(
+          content: chunk,
+          embedding: vector,
+          source_type: data[:source_type],
+          source_path: data[:source_path]
+        )
+
+        Glancer::Utils::Logger.info("Retriever",
+                                    "Stored chunk ##{idx + 1} from #{data[:source_type]}: #{data[:source_path]}")
+      end
+
+      Glancer::Utils::Logger.info("Retriever", "All chunks stored successfully.")
+    rescue StandardError => e
+      Glancer::Utils::Logger.error("Retriever", "Failed to store document chunks: #{e.class} - #{e.message}")
+      Glancer::Utils::Logger.debug("Retriever", "Backtrace:\n#{e.backtrace.join("\n")}")
+      raise Glancer::Error, "Document storage failed: #{e.message}"
+    end
+
+    def search(query)
+      Glancer::Utils::Logger.info("Retriever", "Searching for top #{Glancer.configuration.k} results...")
+
+      query_embedding = RubyLLM.embed(
+        query,
+        model: Glancer.configuration.resolved_embedding_model,
+        provider: Glancer.configuration.resolved_embedding_provider,
+        assume_model_exists: true
+      ).vectors
+
+      # @TODO Postgres with native search?
+      perform_ruby_search(query_embedding)
+    end
+
+    def perform_ruby_search(query_embedding)
+      results = Glancer::Embedding.all.map do |record|
+        # Calculate similarity between query and stored document
+        score = cosine_similarity(query_embedding, record.embedding)
+        weighted_score = score * weight_for(record.source_type)
+
+        { record: record, score: weighted_score }
+      end
+
+      sorted = results.sort_by { |r| -r[:score] }
+
+      # Filter by min_score threshold
+      top_matches = sorted
+                    .select { |r| r[:score] >= Glancer.configuration.min_score }
+                    .first(Glancer.configuration.k)
+
+      # Fallback: if nothing passes the threshold, use best available results so the
+      # LLM always has some schema context rather than generating blind code.
+      if top_matches.empty? && sorted.any?
+        top_matches = sorted.first(Glancer.configuration.k)
+        Glancer::Utils::Logger.warn("Retriever",
+                                    "No results above min_score (#{Glancer.configuration.min_score}); " \
+                                    "using top #{top_matches.size} result(s) as fallback")
+      end
+
+      top_matches = top_matches.map do |r|
+        r[:record].tap do |record|
+          record.define_singleton_method(:score) { r[:score] }
+        end
+      end
+
+      Glancer::Utils::Logger.info("Retriever", "Found #{top_matches.size} relevant document(s)")
+      top_matches
+    end
+
+    def weight_for(source_type)
+      case source_type
+      when "schema"  then Glancer.configuration.schema_documents_weight
+      when "context" then Glancer.configuration.context_documents_weight
+      when "models"  then Glancer.configuration.models_documents_weight
+      else 1.0
+      end
+    end
+
+    def cosine_similarity(vec1, vec2)
+      dot = vec1.zip(vec2).map { |a, b| a * b }.sum
+      mag1 = Math.sqrt(vec1.sum { |x| x**2 })
+      mag2 = Math.sqrt(vec2.sum { |x| x**2 })
+      return 0.0 if mag1.zero? || mag2.zero?
+
+      dot / (mag1 * mag2)
+    end
+  end
+end

data/lib/glancer/utils/logger.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Glancer
+  module Utils
+    class Logger
+      VERBOSITY_LEVELS = {
+        silent: -2,
+        none: -1,
+        info: 1,
+        debug: 2
+      }.freeze
+
+      COLORS = {
+        debug: "\e[36m",
+        info: "\e[32m",
+        warn: "\e[33m",
+        error: "\e[31m",
+        reset: "\e[0m"
+      }.freeze
+
+      # EMOJIS = {
+      #   debug: "🔍",
+      #   info: "✅",
+      #   warn: "⚠️",
+      #   error: "❌"
+      # }.freeze
+
+      class << self
+        def debug(tag, message)
+          write(:debug, tag, message)
+        end
+
+        def info(tag, message)
+          write(:info, tag, message)
+        end
+
+        def warn(tag, message)
+          write(:warn, tag, message)
+        end
+
+        def error(tag, message)
+          write(:error, tag, message)
+        end
+
+        private
+
+        def write(level, tag, message)
+          verbosity = begin
+            Glancer.configuration.log_verbosity.to_sym
+          rescue StandardError
+            :info
+          end
+
+          return if verbosity == :silent
+          return if %i[info debug].include?(level) &&
+                    VERBOSITY_LEVELS[level] > VERBOSITY_LEVELS[verbosity]
+
+          # emoji = EMOJIS[level] || ""
+          color = COLORS[level] || ""
+          reset = COLORS[:reset]
+
+          timestamp = Time.current.strftime("%Y-%m-%d %H:%M:%S")
+          # line = "#{emoji} [#{timestamp}] [Glancer::#{tag}] #{message}"
+          line = "[#{timestamp}] [Glancer::#{tag}] #{message}"
+
+          if Glancer.configuration&.log_output_path
+            File.open(Glancer.configuration.log_output_path, "a") { |f| f.puts(line) }
+          else
+            puts("#{color}#{line}#{reset}")
+          end
+        end
+      end
+
+      def self.with_debug_logs
+        old = Glancer.configuration.log_verbosity
+        Glancer.configuration.log_verbosity = :debug
+        yield
+      ensure
+        Glancer.configuration.log_verbosity = old
+      end
+    end
+  end
+end

data/lib/glancer/utils/markdown_helper.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "commonmarker"
+module Glancer
+  module Utils
+    class MarkdownHelper
+      def self.markdown_to_html(markdown_text, schema_base: nil, valid_tables: nil)
+        content = Commonmarker.to_html(highlight_mentions(markdown_text, schema_base: schema_base, valid_tables: valid_tables),
+                                       options: {
+                                         parse: { smart: true },
+                                         render: { unsafe: true, github_pre_lang: true }
+                                       },
+                                       plugins: { syntax_highlighter: { theme: "InspiredGitHub" } })
+
+        content.gsub!(%r{<table.*?</table>}m) do |table_html|
+          %(<div class="table-scroll-wrapper"><div class="table-scroll-inner">#{table_html}</div></div>)
+        end
+
+        content
+      end
+
+      # Wraps @word tokens with a highlight span. Applied before markdown so that
+      # the renderer preserves the inline HTML. Skips content inside backtick spans
+      # and fenced code blocks so code examples are not affected.
+      def self.highlight_mentions(text, schema_base: nil, valid_tables: nil)
+        valid_set = valid_tables&.to_set
+        # Split on fenced code blocks and inline backtick spans to skip them.
+        parts = text.split(/(```[\s\S]*?```|`[^`]*`)/)
+        parts.map.with_index do |part, idx|
+          if idx.odd?
+            part
+          else
+            # Negative lookbehind prevents matching @ inside emails or identifiers.
+            part.gsub(/(?<![a-zA-Z0-9._])@([a-zA-Z]\w*)/) do
+              table = Regexp.last_match(1)
+              next "@#{table}" if valid_set && !valid_set.include?(table)
+
+              if schema_base
+                href = "#{schema_base}?table=#{table}"
+                attrs = %( href="#{href}" target="_blank" rel="noopener noreferrer" tabindex="0")
+                %(<a class="glancer-mention" data-table="#{table}"#{attrs}>@#{table}</a>)
+              else
+                %(<a class="glancer-mention" data-table="#{table}" href="#" tabindex="0">@#{table}</a>)
+              end
+            end
+          end
+        end.join
+      end
+
+      def self.extract_sql_from_markdown(markdown)
+        match = markdown.match(/```sql\n(.+?)\n```/m)
+        match ? match[1].strip : ""
+      end
+    end
+  end
+end

data/lib/glancer/utils/result_formatter.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Glancer
+  module Utils
+    class ResultFormatter
+      def self.normalize(rows)
+        return rows if rows.empty?
+
+        keys = rows.first.keys
+
+        if rows.all? { |r| r.keys.sort == keys.sort }
+          normalized = {}
+
+          keys.each do |key|
+            normalized[key] = rows.map { |row| row[key] }
+          end
+
+          normalized
+        else
+          rows
+        end
+      end
+    end
+  end
+end

data/lib/glancer/utils/table_stats.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Glancer
+  module Utils
+    module TableStats
+      module_function
+
+      def count_for(table_name)
+        return -1 unless Glancer::Configuration.valid_table_name?(table_name)
+
+        ActiveRecord::Base.connection.select_value("SELECT COUNT(*) FROM #{table_name}").to_i
+      rescue StandardError => e
+        Glancer::Utils::Logger.warn("TableStats", "Could not count rows in #{table_name}: #{e.message}")
+        -1
+      end
+    end
+  end
+end