woods 1.2.0 → 1.3.0

data/lib/woods/embedding/text_preparer.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative '../token_utils'
+
 module Woods
   module Embedding
     # Prepares ExtractedUnit data for embedding by building context-prefixed text.
@@ -19,12 +21,25 @@ module Woods
     #   chunks = preparer.prepare_chunks(unit)
     class TextPreparer
       DEFAULT_MAX_TOKENS = 8192
+      # Aliased to the single source of truth in {Woods::TokenUtils} so the
+      # OpenAI 4.0 / Ollama 1.5 ratios stay consistent across TextPreparer,
+      # ContextAssembler, Builder, and cost_model/. See
+      # docs/TOKEN_BENCHMARK.md and lib/woods/token_utils.rb.
+      DEFAULT_CHARS_PER_TOKEN = TokenUtils::DEFAULT_CHARS_PER_TOKEN
 
       # @param max_tokens [Integer] maximum token budget for prepared text
-      def initialize(max_tokens: DEFAULT_MAX_TOKENS)
+      # @param chars_per_token [Float] tokenizer-calibrated char/token ratio
+      def initialize(max_tokens: DEFAULT_MAX_TOKENS, chars_per_token: DEFAULT_CHARS_PER_TOKEN)
         @max_tokens = max_tokens
+        @chars_per_token = chars_per_token
       end
 
+      # @return [Float] configured chars-per-token ratio
+      attr_reader :chars_per_token
+
+      # @return [Integer] configured token budget
+      attr_reader :max_tokens
+
       # Prepare text for embedding from an ExtractedUnit.
       #
       # Builds a context prefix and appends the unit's source code (or first
@@ -98,13 +113,18 @@ module Woods
 
       # Truncate text to fit within the token budget.
       #
+      # Uses the configured `chars_per_token` ratio to estimate both the
+      # token count and the safe character cap. Truncation is a last
+      # resort — by the time text reaches here the chunker should have
+      # already split oversize units into pieces that fit.
+      #
       # @param text [String] the text to truncate
       # @return [String] text within token limits
       def enforce_token_limit(text)
-        estimated = (text.length / 4.0).ceil
+        estimated = (text.length / @chars_per_token).ceil
         return text if estimated <= @max_tokens
 
-        max_chars = (@max_tokens * 4.0).floor
+        max_chars = (@max_tokens * @chars_per_token).floor
         text[0...max_chars]
       end
     end

data/lib/woods/embedding/token_counter.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module Woods
+  module Embedding
+    # Exact or estimated token counts for embedding inputs.
+    #
+    # When the optional `tokenizers` gem (ankane) is installed, loads the
+    # `bert-base-uncased` WordPiece tokenizer that nomic-embed-text is
+    # built on and returns exact token counts. Otherwise falls back to a
+    # conservative chars/token ratio and warns once.
+    #
+    # Exact counting is strictly preferred for the Ollama path — Ollama
+    # v0.13.5+ stopped honouring the `truncate: true` flag on
+    # `/api/embed` (see ollama/ollama#14186), so chunks that exceed
+    # `num_ctx` return a 400 instead of being truncated. Client-side
+    # sizing is the only reliable option until the regression is fixed
+    # upstream, and chars/token ratios vary too widely across Rails
+    # internals to cover every case with a fixed number.
+    #
+    # @example
+    #   counter = Woods::Embedding::TokenCounter.new
+    #   counter.count("ActionController::Metal::ConditionalGet")  # => 13
+    class TokenCounter
+      # HuggingFace tokenizer id shared by every nomic-embed-text variant.
+      BERT_MODEL = 'bert-base-uncased'
+
+      # Conservative floor for when the tokenizer gem isn't installed.
+      # Lower than any ratio we've observed failing in the testbed
+      # against dense Rails source. Still approximate — install
+      # `tokenizers` for exact counts.
+      CONSERVATIVE_CHARS_PER_TOKEN = 1.2
+
+      # @param chars_per_token [Float] fallback ratio when the tokenizer
+      #   is unavailable
+      # @param tokenizer_id [String] HuggingFace model id passed to
+      #   `Tokenizers.from_pretrained`
+      def initialize(chars_per_token: CONSERVATIVE_CHARS_PER_TOKEN, tokenizer_id: BERT_MODEL)
+        @chars_per_token = chars_per_token
+        @tokenizer_id = tokenizer_id
+        @load_attempted = false
+        @load_mutex = Mutex.new
+      end
+
+      # @return [Float] fallback chars-per-token ratio
+      attr_reader :chars_per_token
+
+      # Exact token count when the tokenizer is loaded, chars/token
+      # estimate otherwise.
+      #
+      # @param text [String, nil]
+      # @return [Integer]
+      def count(text)
+        return 0 if text.nil? || text.empty?
+
+        tok = tokenizer
+        tok ? tok.encode(text).ids.length : estimate(text)
+      end
+
+      # True when the real tokenizer is loaded and in use.
+      #
+      # @return [Boolean]
+      def exact?
+        !tokenizer.nil?
+      end
+
+      private
+
+      def estimate(text)
+        (text.length / @chars_per_token).ceil
+      end
+
+      # Lazy-load the tokenizer under a mutex so concurrent first-calls
+      # don't each trigger a separate download. After the first attempt
+      # (successful or not) we memoize the result and skip the load path.
+      def tokenizer
+        @load_mutex.synchronize do
+          return @tokenizer if @load_attempted
+
+          @load_attempted = true
+          @tokenizer = try_load
+        end
+      end
+
+      def try_load
+        require 'tokenizers'
+        Tokenizers.from_pretrained(@tokenizer_id)
+      rescue LoadError
+        warn_once(
+          'Exact token counting disabled: `tokenizers` gem not installed. ' \
+          "Falling back to #{@chars_per_token} chars/token estimation. " \
+          "Add `gem 'tokenizers', '~> 0.5'` to your Gemfile for exact sizing on Ollama."
+        )
+        nil
+      rescue StandardError => e
+        warn_once(
+          "Could not load tokenizer #{@tokenizer_id.inspect} " \
+          "(#{e.class}: #{e.message}). Falling back to chars/token estimate."
+        )
+        nil
+      end
+
+      # Per-process dedup so multiple TokenCounter instances (one per
+      # retriever build, plus one per chunker, plus tests) don't each
+      # spam the same fallback warning. The mutex keeps the dedup set
+      # consistent under the same concurrent-first-call pattern that
+      # the per-instance load mutex protects against.
+      @warned_messages = Set.new
+      @warned_mutex = Mutex.new
+
+      class << self
+        attr_reader :warned_messages, :warned_mutex
+
+        # Reset the per-process warning dedup. For tests only — production
+        # callers should never need to clear it.
+        def reset_warned!
+          @warned_mutex.synchronize { @warned_messages.clear }
+        end
+      end
+
+      def warn_once(message)
+        full = "[woods] #{message}"
+        self.class.warned_mutex.synchronize do
+          return if self.class.warned_messages.include?(full)
+
+          self.class.warned_messages << full
+        end
+        Kernel.warn(full)
+      end
+    end
+  end
+end

data/lib/woods/evaluation/baseline_runner.rb

@@ -19,8 +19,19 @@ module Woods
       VALID_STRATEGIES = %i[grep random file_level].freeze
 
       # @param metadata_store [Object] Store that responds to #all_identifiers and #find_by_type
-      def initialize(metadata_store:)
+      # @param seed [Integer, nil] Optional RNG seed for the `:random`
+      #   baseline. Seeding makes evaluation runs reproducible — essential
+      #   for comparing the real retriever against the baseline on the same
+      #   query set across two invocations. `nil` (default) keeps the
+      #   historical behavior of drawing from system entropy.
+      def initialize(metadata_store:, seed: nil)
         @metadata_store = metadata_store
+        @random = seed.nil? ? Random.new : Random.new(seed)
+        # Ruby's `Random` instance isn't documented thread-safe; multiple
+        # evaluator threads sharing one runner would otherwise interleave
+        # `sample` calls and drift from the seeded sequence. A plain Mutex
+        # around the read path is enough — `sample` is the only caller.
+        @random_mutex = Mutex.new
       end
 
       # Run a baseline strategy for a query.
@@ -64,11 +75,18 @@ module Woods
 
       # Random strategy: random selection from all available units.
       #
+      # Uses the instance's injected {Random} generator so results are
+      # reproducible when {#initialize} was called with a seed. Guarded by
+      # a mutex so concurrent evaluator threads don't interleave calls on
+      # the shared `Random` and drift from the seeded sequence.
+      #
       # @param _query [String] Query string (unused)
       # @param limit [Integer] Max results
       # @return [Array<String>]
       def run_random(_query, limit)
-        @metadata_store.all_identifiers.sample(limit)
+        @random_mutex.synchronize do
+          @metadata_store.all_identifiers.sample(limit, random: @random)
+        end
       end
 
       # File-level strategy: matches identifiers that look like file paths

data/lib/woods/evaluation/metrics.rb

@@ -22,7 +22,10 @@ module Woods
         top_k = retrieved.first(cutoff)
         relevant_set = relevant.to_set
         hits = top_k.count { |id| relevant_set.include?(id) }
-        hits.to_f / cutoff
+        # Divide by actual slice size, not the cutoff — when fewer than
+        # `cutoff` items are retrieved, dividing by `cutoff` understates
+        # precision (returns 0.2 for 1-of-1 at cutoff=5 instead of 1.0).
+        hits.to_f / top_k.size
       end
 
       # Fraction of relevant items that were retrieved.

data/lib/woods/extracted_unit.rb

@@ -2,6 +2,7 @@
 
 require 'digest'
 require 'json'
+require 'time' # Time#iso8601
 
 module Woods
   # ExtractedUnit represents a single meaningful unit of code from the codebase.

data/lib/woods/extractor.rb

@@ -403,9 +403,15 @@ module Woods
     def extract_all_concurrent
       # Pre-compute ModelNameCache to avoid race on lazy memoization.
       # Multiple threads calling model_names concurrently could trigger
-      # duplicate compute_model_names calls without this warm-up.
+      # duplicate compute_model_names calls without this warm-up. All four
+      # derived caches must be warmed — `short_name_map` and
+      # `short_names_regex` were added for the three-pass dependency
+      # scanner and are reached from every extractor that calls
+      # `scan_model_dependencies`.
       ModelNameCache.model_names
       ModelNameCache.model_names_regex
+      ModelNameCache.short_name_map if ModelNameCache.respond_to?(:short_name_map)
+      ModelNameCache.short_names_regex if ModelNameCache.respond_to?(:short_names_regex)
 
       results_mutex = Mutex.new
       threads = EXTRACTORS.map do |type, extractor_class|

data/lib/woods/extractors/controller_extractor.rb

@@ -4,6 +4,7 @@ require 'digest'
 require_relative 'ast_source_extraction'
 require_relative 'shared_utility_methods'
 require_relative 'shared_dependency_scanner'
+require_relative 'route_helper_resolver'
 
 module Woods
   module Extractors
@@ -25,9 +26,11 @@ module Woods
       include AstSourceExtraction
       include SharedUtilityMethods
       include SharedDependencyScanner
+      include RouteHelperResolver
 
       def initialize
         @routes_map = build_routes_map
+        build_route_helper_map
       end
 
       # Extract all controllers in the application
@@ -312,6 +315,9 @@ module Woods
           source.scan(%r{render\s+["'](\w+/\w+)["']}).flatten.uniq.each do |template|
             deps << { type: :view, target: template, via: :render }
           end
+
+          # redirect_to with named route helpers
+          deps.concat(scan_navigation_dependencies(source, via_type: :redirect_to))
         end
 
         deps.uniq { |d| [d[:type], d[:target]] }

data/lib/woods/extractors/mailer_extractor.rb

@@ -4,6 +4,7 @@ require 'digest'
 require_relative 'ast_source_extraction'
 require_relative 'shared_utility_methods'
 require_relative 'shared_dependency_scanner'
+require_relative 'route_helper_resolver'
 
 module Woods
   module Extractors
@@ -23,9 +24,11 @@ module Woods
       include AstSourceExtraction
       include SharedUtilityMethods
       include SharedDependencyScanner
+      include RouteHelperResolver
 
       def initialize
         @mailer_base = defined?(ApplicationMailer) ? ApplicationMailer : ActionMailer::Base
+        build_route_helper_map
       end
 
       # Extract all mailers in the application
@@ -223,8 +226,19 @@ module Woods
         deps = []
         deps.concat(scan_model_dependencies(source))
         deps.concat(scan_service_dependencies(source))
-
-        # URL helpers (indicates what resources emails link to)
+        # Navigation edges — resolve `_path`/`_url` helpers to real
+        # controllers via RouteHelperResolver (wired through the include +
+        # build_route_helper_map call in #initialize). This adds resolved
+        # {type: :controller, via: :link_to} edges on top of the raw
+        # helper scan below.
+        deps.concat(scan_navigation_dependencies(source))
+        deps.concat(scan_form_dependencies(source))
+
+        # Raw-helper fallback — emits {type: :route, target: 'confirmation'}
+        # for every `_path`/`_url` helper referenced, regardless of whether
+        # RouteHelperResolver could resolve it. Kept so mailers that link to
+        # engine-mounted routes (not in the main routes table) still produce
+        # a dependency edge.
         source.scan(/(\w+)_(?:url|path)/).flatten.uniq.each do |route|
           deps << { type: :route, target: route, via: :url_helper }
         end

data/lib/woods/extractors/model_extractor.rb

@@ -530,7 +530,12 @@ module Woods
               conditions: format_callback_conditions(cb)
             }
           end
-        rescue NoMethodError
+        rescue StandardError
+          # Widen beyond NoMethodError per CLAUDE.md — callback-chain
+          # introspection can raise a variety of errors across Rails
+          # versions (NameError, TypeError, LoadError for missing
+          # concerns), and silently swallowing only NoMethodError left
+          # the rest to crash extraction.
           []
         end.compact
       end

data/lib/woods/extractors/phlex_extractor.rb

@@ -2,6 +2,7 @@
 
 require_relative 'shared_utility_methods'
 require_relative 'shared_dependency_scanner'
+require_relative 'route_helper_resolver'
 
 module Woods
   module Extractors
@@ -23,6 +24,7 @@ module Woods
     class PhlexExtractor
       include SharedUtilityMethods
       include SharedDependencyScanner
+      include RouteHelperResolver
 
       # Common Phlex base classes to look for
       PHLEX_BASES = %w[
@@ -33,6 +35,10 @@ module Woods
 
       def initialize
         @component_base = find_component_base
+        # Precompute the _path/_url → controller#action map once per
+        # extraction run so navigation edges resolve to real targets
+        # instead of the unresolved helper literal.
+        build_route_helper_map
       end
 
       # Extract all Phlex/ViewComponent components
@@ -240,10 +246,13 @@ module Woods
           deps << { type: :stimulus_controller, target: controller, via: :html_attribute }
         end
 
-        # URL helpers
-        source.scan(/(\w+)_(?:path|url)/).flatten.uniq.each do |route|
-          deps << { type: :route, target: route, via: :url_helper }
-        end
+        # Navigation edges — resolve _path / _url helpers to real controllers
+        # via RouteHelperResolver (wired through the include + build_route_helper_map
+        # call in #initialize). Replaces an earlier manual regex that emitted
+        # unresolved {type: :route, target: 'users'} edges; the graph now
+        # gets {type: :controller, target: 'UsersController', via: :link_to}.
+        deps.concat(scan_navigation_dependencies(source))
+        deps.concat(scan_form_dependencies(source))
 
         deps.uniq { |d| [d[:type], d[:target]] }
       end

data/lib/woods/extractors/rails_source_extractor.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative '../extracted_unit'
+
 module Woods
   module Extractors
     # RailsSourceExtractor indexes selected parts of the Rails framework

data/lib/woods/extractors/route_helper_resolver.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module Woods
+  module Extractors
+    # Shared module for resolving named route helpers to controller#action targets.
+    #
+    # Builds an inverse lookup from `Rails.application.routes.named_routes`,
+    # mapping route helper names (e.g., "new_post") to their controller and action.
+    # Include this module and call {#build_route_helper_map} in your initializer.
+    #
+    # @example
+    #   class MyExtractor
+    #     include RouteHelperResolver
+    #
+    #     def initialize
+    #       build_route_helper_map
+    #     end
+    #
+    #     def find_target(source)
+    #       resolve_route_helper("posts_path")
+    #       #=> { controller: "PostsController", action: "index", path: "/posts", verb: "GET" }
+    #     end
+    #   end
+    #
+    module RouteHelperResolver
+      # Route helper prefixes that produce non-navigation dependencies.
+      # These generate asset URLs or are common false positives from
+      # non-route uses of _path/_url suffixes in Ruby code.
+      #
+      # NOTE: `root` is intentionally excluded — root_path is the most common
+      # Rails route helper, but it appears so frequently in non-navigation contexts
+      # (path construction, config, tests) that it generates excessive noise.
+      # The tradeoff: "what links to the home page?" won't appear in graph queries.
+      # Add new prefixes here when false positives are discovered in host apps.
+      IGNORED_HELPER_PREFIXES = %w[
+        asset
+        image
+        stylesheet
+        javascript
+        font
+        audio
+        video
+        turbo_stream
+        file
+        tmp
+        base
+        root
+        log
+        socket
+        download
+      ].freeze
+
+      # Build the route helper lookup map from Rails named routes.
+      # Call this once in your extractor's initialize method.
+      #
+      # Resilient to partial test doubles: any exception raised while
+      # traversing Rails routes (unstubbed `application` on a double,
+      # missing `named_routes`, etc.) is swallowed and leaves the map
+      # empty — extractors fall back to returning the helper name
+      # literal as the dependency target.
+      def build_route_helper_map
+        @route_helper_map = {}
+        return unless defined?(Rails)
+
+        routes = safe_rails_application_routes
+        return unless routes
+
+        routes.named_routes.each do |name, route|
+          controller = route.defaults[:controller]
+          action = route.defaults[:action]
+          next unless controller && action
+
+          @route_helper_map[name.to_s] = {
+            controller: "#{controller.camelize}Controller",
+            action: action,
+            path: route.path.spec.to_s.gsub('(.:format)', ''),
+            verb: extract_route_verb(route)
+          }
+        end
+      rescue StandardError
+        # Leave @route_helper_map empty — navigation-edge extractors will
+        # fall back to the helper-name literal.
+        @route_helper_map = {}
+      end
+
+      # True when Rails.application.routes is reachable. Probing via
+      # `respond_to?` first so partial RSpec doubles that haven't
+      # stubbed `.application` don't raise MockExpectationError (which
+      # descends from Exception, not StandardError — `rescue StandardError`
+      # would not catch it).
+      def safe_rails_application_routes
+        return nil unless Rails.respond_to?(:application)
+
+        app = Rails.application
+        return nil unless app.respond_to?(:routes)
+
+        app.routes
+      rescue StandardError
+        nil
+      end
+
+      # Resolve a _path/_url helper to its controller#action target.
+      #
+      # @param helper_name [String] e.g., "new_post_path", "users_url"
+      # @return [Hash, nil] { controller:, action:, path:, verb: } or nil if unresolvable
+      def resolve_route_helper(helper_name)
+        base = helper_name.sub(/_(path|url)\z/, '')
+        return nil if IGNORED_HELPER_PREFIXES.any? { |prefix| base.start_with?("#{prefix}_") || base == prefix }
+
+        @route_helper_map&.[](base)
+      end
+
+      private
+
+      # Extract the HTTP verb from a route.
+      #
+      # @param route [ActionDispatch::Journey::Route] The route object
+      # @return [String] HTTP verb (GET, POST, etc.)
+      def extract_route_verb(route)
+        if route.respond_to?(:verb) && route.verb.is_a?(String)
+          route.verb
+        elsif route.respond_to?(:verb)
+          route.verb.to_s.gsub(/[^A-Z|]/, '')
+        else
+          'GET'
+        end
+      end
+    end
+  end
+end