woods 1.0.0

data/lib/woods/retrieval/context_assembler.rb

@@ -0,0 +1,261 @@
+# frozen_string_literal: true
+
+module Woods
+  module Retrieval
+    # Transforms ranked search candidates into a token-budgeted context string
+    # for LLM consumption.
+    #
+    # Allocates a fixed token budget across four sections:
+    # - Structural (10%): Always-included codebase overview
+    # - Primary (50%): Direct query results
+    # - Supporting (25%): Dependencies and related context
+    # - Framework (15%): Rails/gem source when query has framework context
+    #
+    # When framework context is not needed, primary and supporting sections
+    # receive the framework allocation proportionally.
+    #
+    # @example
+    #   assembler = ContextAssembler.new(metadata_store: store)
+    #   result = assembler.assemble(candidates: ranked, classification: cls)
+    #   result.context     # => "## User (model)\n..."
+    #   result.tokens_used # => 4200
+    #   result.sections    # => [:structural, :primary, :supporting]
+    #
+    class ContextAssembler
+      DEFAULT_BUDGET = 8000 # tokens
+
+      BUDGET_ALLOCATION = {
+        structural: 0.10,
+        primary: 0.50,
+        supporting: 0.25,
+        framework: 0.15
+      }.freeze
+
+      # Minimum token count for a section to be worth including.
+      MIN_USEFUL_TOKENS = 200
+
+      # @param metadata_store [#find] Store that resolves identifiers to unit data
+      # @param budget [Integer] Total token budget
+      def initialize(metadata_store:, budget: DEFAULT_BUDGET)
+        @metadata_store = metadata_store
+        @budget = budget
+      end
+
+      # Assemble context from ranked candidates within token budget.
+      #
+      # @param candidates [Array<Candidate>] Ranked search candidates
+      # @param classification [QueryClassifier::Classification] Query classification
+      # @param structural_context [String, nil] Optional codebase overview text
+      # @param budget [Integer, nil] Override token budget; falls back to @budget
+      # @return [AssembledContext] Token-budgeted context with source attribution
+      def assemble(candidates:, classification:, structural_context: nil, budget: nil)
+        effective_budget = budget || @budget
+        sections = []
+        sources = []
+        tokens_used = 0
+
+        # Pre-fetch all candidate metadata in one batch query
+        @unit_cache = @metadata_store.find_batch(candidates.map(&:identifier))
+
+        # 1. Structural context (always first if provided)
+        tokens_used = add_structural_section(sections, structural_context, tokens_used, effective_budget)
+
+        # 2. Compute per-section budgets from remaining tokens
+        budgets = compute_section_budgets(effective_budget - tokens_used, classification)
+
+        # 3. Primary, supporting, and framework sections
+        add_candidate_section(sections, sources, :primary,
+                              candidates.reject { |c| c.source == :graph_expansion }, budgets[:primary])
+        add_candidate_section(sections, sources, :supporting,
+                              candidates.select { |c| c.source == :graph_expansion }, budgets[:supporting])
+        if budgets[:framework].positive?
+          add_candidate_section(sections, sources, :framework,
+                                candidates.select { |c| framework_candidate?(c) }, budgets[:framework])
+        end
+
+        build_result(sections, sources, effective_budget)
+      end
+
+      private
+
+      # Add structural context section if provided.
+      #
+      # @return [Integer] Updated tokens_used count
+      def add_structural_section(sections, structural_context, tokens_used, effective_budget)
+        return tokens_used unless structural_context
+
+        budget = (effective_budget * BUDGET_ALLOCATION[:structural]).to_i
+        text = truncate_to_budget(structural_context, budget)
+        sections << { section: :structural, content: text }
+        tokens_used + estimate_tokens(text)
+      end
+
+      # Add a candidate-based section if candidates produce content.
+      #
+      # @return [void]
+      def add_candidate_section(sections, sources, section_name, candidates, budget)
+        return if candidates.empty?
+
+        content, section_sources = assemble_section(candidates, budget)
+        return if content.empty?
+
+        sections << { section: section_name, content: content }
+        sources.concat(section_sources)
+      end
+
+      # Compute token budgets for primary/supporting/framework sections.
+      #
+      # @param remaining [Integer] Tokens available after structural
+      # @param classification [QueryClassifier::Classification]
+      # @return [Hash<Symbol, Integer>]
+      def compute_section_budgets(remaining, classification)
+        if classification.framework_context
+          {
+            primary: (remaining * 0.55).to_i,
+            supporting: (remaining * 0.25).to_i,
+            framework: (remaining * 0.20).to_i
+          }
+        else
+          {
+            primary: (remaining * 0.65).to_i,
+            supporting: (remaining * 0.35).to_i,
+            framework: 0
+          }
+        end
+      end
+
+      # Assemble content for a single section within a token budget.
+      #
+      # @param candidates [Array<Candidate>] Candidates for this section
+      # @param budget [Integer] Token budget for this section
+      # @return [Array(String, Array<Hash>)] Content string and source attributions
+      def assemble_section(candidates, budget)
+        content_parts = []
+        sources = []
+        tokens_used = 0
+
+        candidates.sort_by { |c| -c.score }.each do |candidate|
+          tokens_used = append_candidate(content_parts, sources, candidate, budget, tokens_used)
+          break if tokens_used.nil?
+        end
+
+        [content_parts.join("\n\n"), sources]
+      end
+
+      # Append a single candidate to the section. Returns updated tokens_used, or nil to stop.
+      def append_candidate(parts, sources, candidate, budget, tokens_used)
+        unit = @unit_cache[candidate.identifier]
+        return tokens_used unless unit
+
+        text = format_unit(unit, candidate)
+        tokens = estimate_tokens(text)
+        remaining = budget - tokens_used
+
+        if tokens <= remaining
+          parts << text
+          sources << build_source_attribution(candidate, unit)
+          tokens_used + tokens
+        elsif remaining > MIN_USEFUL_TOKENS
+          parts << truncate_to_budget(text, remaining)
+          sources << build_source_attribution(candidate, unit, truncated: true)
+          nil
+        end
+      end
+
+      # Format a unit for inclusion in context.
+      #
+      # @param unit [Hash] Unit data from metadata store
+      # @param candidate [Candidate] The search candidate
+      # @return [String]
+      def format_unit(unit, _candidate)
+        identifier = unit_field(unit, :identifier)
+        type = unit_field(unit, :type)
+        file_path = unit_field(unit, :file_path)
+        source = unit_field(unit, :source_code) || ''
+
+        <<~UNIT.strip
+          ## #{identifier} (#{type})
+          File: #{file_path}
+
+          #{source}
+        UNIT
+      end
+
+      # Build source attribution hash for a candidate.
+      #
+      # @return [Hash]
+      def build_source_attribution(candidate, unit, truncated: false)
+        attribution = {
+          identifier: candidate.identifier,
+          type: unit_field(unit, :type),
+          score: candidate.score,
+          file_path: unit_field(unit, :file_path)
+        }
+        attribution[:truncated] = true if truncated
+        attribution
+      end
+
+      # Read a field from a unit hash, accepting either symbol or string keys.
+      #
+      # @param unit [Hash]
+      # @param key [Symbol]
+      # @return [Object, nil]
+      def unit_field(unit, key)
+        unit[key] || unit[key.to_s]
+      end
+
+      # Check if a candidate is framework source.
+      #
+      # @param candidate [Candidate]
+      # @return [Boolean]
+      def framework_candidate?(candidate)
+        metadata = candidate.metadata
+        return false unless metadata
+
+        type = metadata[:type] || metadata['type']
+        %w[rails_source gem_source].include?(type.to_s)
+      end
+
+      # Truncate text to fit within a token budget.
+      #
+      # @param text [String]
+      # @param token_budget [Integer]
+      # @return [String]
+      def truncate_to_budget(text, token_budget)
+        return text if estimate_tokens(text) <= token_budget
+
+        # Estimate target character count with 10% safety margin
+        target_chars = (token_budget * 4.0 * 0.9).to_i
+        "#{text[0...target_chars]}\n... [truncated]"
+      end
+
+      # Estimate token count using the project convention.
+      #
+      # @param text [String]
+      # @return [Integer]
+      def estimate_tokens(text)
+        (text.length / 4.0).ceil
+      end
+
+      # Build the final AssembledContext result.
+      #
+      # @param sections [Array<Hash>] Assembled sections
+      # @param sources [Array<Hash>] Source attributions
+      # @param effective_budget [Integer] The budget actually used for assembly
+      # @return [AssembledContext]
+      def build_result(sections, sources, effective_budget)
+        context = sections.map { |s| s[:content] }.join("\n\n---\n\n")
+        AssembledContext.new(
+          context: context,
+          tokens_used: estimate_tokens(context),
+          budget: effective_budget,
+          sources: sources.uniq,
+          sections: sections.map { |s| s[:section] }
+        )
+      end
+    end
+
+    # Result of context assembly.
+    AssembledContext = Struct.new(:context, :tokens_used, :budget, :sources, :sections, keyword_init: true)
+  end
+end

data/lib/woods/retrieval/query_classifier.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module Woods
+  module Retrieval
+    # Classifies natural language queries to determine retrieval strategy.
+    #
+    # Uses heuristic pattern matching to determine:
+    # - Intent: what the user wants to do
+    # - Scope: how broad the search should be
+    # - Target type: what kind of code unit to look for
+    # - Framework context: whether this is about Rails/gems vs app code
+    #
+    class QueryClassifier
+      # Classification result
+      Classification = Struct.new(:intent, :scope, :target_type, :framework_context, :keywords, keyword_init: true)
+
+      INTENTS = %i[understand locate trace debug implement reference compare framework].freeze
+      SCOPES = %i[pinpoint focused exploratory comprehensive].freeze
+
+      STOP_WORDS = Set.new(%w[the a an is are was were be been being have has had do does did will would could
+                              should may might can shall in on at to for of and or but not with by from as
+                              this that these those it its how what when where why who which]).freeze
+
+      # Intent patterns — order matters (first match wins)
+      INTENT_PATTERNS = {
+        locate: /\b(where|find|which file|locate|look for|search for)\b/i,
+        trace: /\b(trace|follow|track|call(s|ed by)|depends on|used by|who calls|what calls)\b/i,
+        debug: /\b(bug|error|fix|broken|failing|wrong|issue|problem|crash|exception)\b/i,
+        implement: /\b(implement|add|create|build|write|make|generate)\b/i,
+        compare: /\b(compare|difference|vs|versus|between|contrast)\b/i,
+        # rubocop:disable Layout/LineLength
+        framework: /\b(how does rails|what does rails|rails .+ work|work.+\brails\b|in rails\b|activerecord|actioncontroller|activejob)\b/i,
+        # rubocop:enable Layout/LineLength
+        reference: /\b(show me|what is|what are|list|options for|api|interface|signature)\b/i,
+        understand: /\b(how|why|explain|understand|what happens|describe|overview)\b/i
+      }.freeze
+
+      # Scope patterns
+      SCOPE_PATTERNS = {
+        pinpoint: /\b(exactly|specific|this one|just the|only the)\b/i,
+        comprehensive: /\b(all|every|entire|whole|complete|everything)\b/i,
+        exploratory: /\b(related|around|near|similar|like|associated)\b/i
+      }.freeze
+
+      # Target type patterns
+      TARGET_PATTERNS = {
+        model: /\b(model|activerecord|association|schema|table|column|scope|validation)\b/i,
+        controller: /\b(controller|action|route|endpoint|api|request|response|filter|callback)\b/i,
+        service: /\b(service|interactor|operation|command|use.?case|business.?logic)\b/i,
+        job: /\b(job|worker|background|async|sidekiq|queue|perform)\b/i,
+        mailer: /\b(mailer|email|notification|send.?mail)\b/i,
+        graphql: /\b(graphql|mutation|query|type|resolver|field|argument|schema)\b/i,
+        concern: /\b(concern|mixin|module|included|extend)\b/i,
+        route: /\b(route|path|url|endpoint|uri|http|get|post|put|patch|delete)\b/i,
+        middleware: /\b(middleware|rack|request.?pipeline|before.?action)\b/i,
+        i18n: /\b(i18n|translation|locale|internationalization|t\(|translate)\b/i,
+        pundit_policy: /\b(pundit|authorize|policy|allowed|permitted)\b/i,
+        configuration: /\b(config|initializer|environment|setting|configure)\b/i,
+        engine: /\b(engine|mountable|mount|railtie|plugin|isolated.?namespace)\b/i,
+        view_template: /\b(view|template|partial|render|erb|layout|html)\b/i,
+        # rubocop:disable Layout/LineLength
+        migration: /\b(migration|migrate|schema.?change|add.?column|remove.?column|create.?table|drop.?table|db.?migrate)\b/i,
+        action_cable_channel: /\b(action.?cable|websocket|broadcast|cable.?channel|subscription.?channel|realtime|real.?time)\b/i,
+        scheduled_job: /\b(schedule[dr]?|recurring|cron|periodic|every\s+\d|daily|hourly|weekly|solid.?queue.*recur|sidekiq.?cron|whenever)\b/i,
+        rake_task: /\b(rake|rake.?task|lib.?tasks?|maintenance.?script|batch.?script)\b/i
+        # rubocop:enable Layout/LineLength
+      }.freeze
+
+      # Classify a query string
+      #
+      # @param query [String] Natural language query
+      # @return [Classification] Classified query
+      def classify(query)
+        Classification.new(
+          intent: detect_intent(query),
+          scope: detect_scope(query),
+          target_type: detect_target_type(query),
+          framework_context: framework_query?(query),
+          keywords: extract_keywords(query)
+        )
+      end
+
+      private
+
+      # @param query [String]
+      # @return [Symbol]
+      def detect_intent(query)
+        match_first(INTENT_PATTERNS, query, default: :understand)
+      end
+
+      # @param query [String]
+      # @return [Symbol]
+      def detect_scope(query)
+        match_first(SCOPE_PATTERNS, query, default: :focused)
+      end
+
+      # @param query [String]
+      # @return [Symbol, nil]
+      def detect_target_type(query)
+        match_first(TARGET_PATTERNS, query, default: nil)
+      end
+
+      # Match query against a hash of {key => pattern}, returning the first matching key.
+      #
+      # @param patterns [Hash{Symbol => Regexp}]
+      # @param query [String]
+      # @param default [Object] value if no pattern matches
+      # @return [Object]
+      def match_first(patterns, query, default:)
+        patterns.each { |key, pattern| return key if query.match?(pattern) }
+        default
+      end
+
+      # @param query [String]
+      # @return [Boolean]
+      def framework_query?(query)
+        query.match?(/\b(rails|activerecord|actioncontroller|activejob|actionmailer|activesupport|rack|middleware)\b/i)
+      end
+
+      # @param query [String]
+      # @return [Array<String>]
+      def extract_keywords(query)
+        query.downcase
+             .gsub(/[^\w\s]/, ' ')
+             .split
+             .reject { |w| STOP_WORDS.include?(w) || w.length < 2 }
+             .uniq
+      end
+    end
+  end
+end

data/lib/woods/retrieval/ranker.rb

@@ -0,0 +1,277 @@
+# frozen_string_literal: true
+
+module Woods
+  module Retrieval
+    # Ranks search candidates using weighted signal scoring and diversity adjustment.
+    #
+    # Combines multiple ranking signals into a final score:
+    # - Semantic similarity from vector search
+    # - Keyword match quality
+    # - Recency (git change frequency)
+    # - Importance (PageRank / structural importance)
+    # - Type match (bonus when result type matches query target_type)
+    # - Diversity (penalty for too many results of same type/namespace)
+    #
+    # After initial scoring, applies Reciprocal Rank Fusion (RRF) when
+    # candidates come from multiple retrieval sources.
+    #
+    # @example
+    #   ranker = Ranker.new(metadata_store: store)
+    #   ranked = ranker.rank(candidates, classification: classification)
+    #
+    class Ranker
+      # Signal weights for ranking — sum to 1.0.
+      WEIGHTS = {
+        semantic: 0.40,
+        keyword: 0.20,
+        recency: 0.15,
+        importance: 0.10,
+        type_match: 0.10,
+        diversity: 0.05
+      }.freeze
+
+      # RRF constant — balances rank position vs. absolute score.
+      # Standard value from the original RRF paper (Cormack et al., 2009).
+      RRF_K = 60
+
+      # @param metadata_store [#find] Store that resolves identifiers to unit metadata
+      def initialize(metadata_store:)
+        @metadata_store = metadata_store
+      end
+
+      # Rank candidates by weighted signal scoring with diversity adjustment.
+      #
+      # @param candidates [Array<Candidate>] Search candidates from executor
+      # @param classification [QueryClassifier::Classification] Query classification
+      # @return [Array<Candidate>] Re-ranked candidates (best first)
+      def rank(candidates, classification:)
+        return [] if candidates.empty?
+
+        # Apply RRF if candidates come from multiple sources
+        candidates = apply_rrf(candidates) if multi_source?(candidates)
+
+        scored = score_candidates(candidates, classification)
+        sorted = sorted_by_weighted_score(scored)
+        apply_diversity_penalty(sorted)
+
+        sorted.map { |item| item[:candidate] }
+      end
+
+      private
+
+      # Check if candidates come from multiple retrieval sources.
+      #
+      # @param candidates [Array<Candidate>]
+      # @return [Boolean]
+      def multi_source?(candidates)
+        candidates.map(&:source).uniq.size > 1
+      end
+
+      # Apply Reciprocal Rank Fusion across sources.
+      #
+      # RRF formula: score(d) = sum(1/(k + rank_i(d)))
+      # Each source's candidates are ranked independently, then RRF
+      # merges ranks into a single score.
+      #
+      # @param candidates [Array<Candidate>]
+      # @return [Array<Candidate>] Merged candidates with RRF scores
+      def apply_rrf(candidates)
+        rrf_scores, metadata_map = compute_rrf_scores(candidates)
+        rebuild_rrf_candidates(candidates, rrf_scores, metadata_map)
+      end
+
+      # Compute RRF scores across all sources.
+      #
+      # @return [Array(Hash, Hash)] [rrf_scores, metadata_map]
+      def compute_rrf_scores(candidates)
+        rrf_scores = Hash.new(0.0)
+        metadata_map = {}
+
+        candidates.group_by(&:source).each_value do |source_candidates|
+          ranked = source_candidates.sort_by { |c| -c.score }
+          ranked.each_with_index do |candidate, rank|
+            rrf_scores[candidate.identifier] += 1.0 / (RRF_K + rank)
+            metadata_map[candidate.identifier] ||= candidate.metadata
+          end
+        end
+
+        [rrf_scores, metadata_map]
+      end
+
+      # Rebuild candidates with merged RRF scores.
+      #
+      # @return [Array<Candidate>]
+      def rebuild_rrf_candidates(candidates, rrf_scores, metadata_map)
+        original_by_id = candidates.index_by(&:identifier)
+        rrf_scores.sort_by { |_id, score| -score }.map do |identifier, score|
+          original = original_by_id[identifier]
+          build_candidate(
+            identifier: identifier,
+            score: score,
+            source: original&.source || :rrf,
+            metadata: metadata_map[identifier]
+          )
+        end
+      end
+
+      # Score each candidate across all signals.
+      #
+      # @param candidates [Array<Candidate>]
+      # @param classification [QueryClassifier::Classification]
+      # @return [Array<Hash>]
+      def score_candidates(candidates, classification)
+        # Batch-fetch all metadata in one query instead of per-candidate lookups
+        unit_map = @metadata_store.find_batch(candidates.map(&:identifier))
+
+        candidates.map do |candidate|
+          unit = unit_map[candidate.identifier]
+
+          {
+            candidate: candidate,
+            unit: unit, # cached to avoid double lookup in apply_diversity_penalty
+            scores: {
+              semantic: candidate.score.to_f,
+              keyword: keyword_score(candidate),
+              recency: recency_score(unit),
+              importance: importance_score(unit),
+              type_match: type_match_score(unit, classification),
+              diversity: 1.0 # Adjusted after initial sort
+            }
+          }
+        end
+      end
+
+      # Calculate weighted score for each item.
+      #
+      # @param scored [Array<Hash>]
+      # @return [Array<Hash>] Sorted by weighted_score descending
+      def sorted_by_weighted_score(scored)
+        scored.each do |item|
+          item[:weighted_score] = WEIGHTS.sum do |signal, weight|
+            item[:scores][signal] * weight
+          end
+        end
+
+        scored.sort_by { |item| -item[:weighted_score] }
+      end
+
+      # Keyword match score based on matched field count.
+      #
+      # @param candidate [Candidate]
+      # @return [Float] 0.0 to 1.0
+      def keyword_score(candidate)
+        return 0.0 unless candidate.respond_to?(:matched_fields) && candidate.matched_fields
+
+        [candidate.matched_fields.size * 0.25, 1.0].min
+      end
+
+      # Recency score based on git change frequency metadata.
+      #
+      # @param unit [Hash, nil] Unit metadata from store
+      # @return [Float] 0.0 to 1.0
+      def recency_score(unit)
+        return 0.5 unless unit
+
+        frequency = dig_metadata(unit, :git, :change_frequency)
+        case frequency&.to_sym
+        when :hot then 1.0
+        when :active then 0.8
+        when :dormant then 0.3
+        when :new then 0.7
+        else 0.5 # stable or unknown
+        end
+      end
+
+      # Importance score based on PageRank / structural importance.
+      #
+      # @param unit [Hash, nil] Unit metadata from store
+      # @return [Float] 0.0 to 1.0
+      def importance_score(unit)
+        return 0.5 unless unit
+
+        importance = dig_metadata(unit, :importance)
+        case importance&.to_s
+        when 'high' then 1.0
+        when 'medium' then 0.6
+        when 'low' then 0.3
+        else 0.5
+        end
+      end
+
+      # Type match score — bonus when result type matches query target_type.
+      #
+      # @param unit [Hash, nil] Unit metadata from store
+      # @param classification [QueryClassifier::Classification]
+      # @return [Float] 0.0 to 1.0
+      def type_match_score(unit, classification)
+        return 0.5 unless unit
+        return 0.5 unless classification.target_type
+
+        unit_type = dig_metadata(unit, :type) || unit[:type]
+        unit_type&.to_sym == classification.target_type ? 1.0 : 0.3
+      end
+
+      # Apply diversity penalty to avoid clustering by type/namespace.
+      #
+      # @param sorted [Array<Hash>] Scored items sorted by weighted_score
+      # @return [void] Mutates items in place
+      def apply_diversity_penalty(sorted)
+        seen_namespaces = Hash.new(0)
+        seen_types = Hash.new(0)
+
+        sorted.each do |item|
+          penalty = diversity_penalty_for(item, seen_namespaces, seen_types)
+          next unless penalty
+
+          item[:scores][:diversity] = 1.0 - penalty
+          item[:weighted_score] -= penalty * WEIGHTS[:diversity]
+        end
+
+        sorted.sort_by! { |item| -item[:weighted_score] }
+      end
+
+      # Compute diversity penalty for a single item and update seen counts.
+      #
+      # Uses the unit cached in item[:unit] to avoid a redundant metadata store lookup.
+      #
+      # @return [Float, nil] Penalty amount, or nil if unit not found
+      def diversity_penalty_for(item, seen_namespaces, seen_types)
+        unit = item[:unit]
+        return nil unless unit
+
+        namespace = dig_metadata(unit, :namespace) || 'root'
+        type = (dig_metadata(unit, :type) || 'unknown').to_s
+
+        penalty = [(seen_namespaces[namespace] + seen_types[type]) * 0.1, 0.5].min
+        seen_namespaces[namespace] += 1
+        seen_types[type] += 1
+        penalty
+      end
+
+      # Dig into unit metadata, handling both hash and object access.
+      #
+      # @param unit [Hash, Object] Unit data
+      # @param keys [Array<Symbol>] Key path
+      # @return [Object, nil]
+      def dig_metadata(unit, *keys)
+        if keys.size == 1
+          unit.is_a?(Hash) ? (unit.dig(:metadata, keys[0]) || unit[keys[0]]) : nil
+        else
+          unit.is_a?(Hash) ? unit.dig(:metadata, *keys) : nil
+        end
+      end
+
+      # Build a Candidate struct compatible with SearchExecutor::Candidate.
+      #
+      # @return [Candidate-like Struct]
+      def build_candidate(identifier:, score:, source:, metadata:)
+        SearchExecutor::Candidate.new(
+          identifier: identifier,
+          score: score,
+          source: source,
+          metadata: metadata
+        )
+      end
+    end
+  end
+end