woods 1.6.1 → 2.0.0.beta2

data/lib/woods/evaluation/evaluator.rb

@@ -20,31 +20,57 @@ module Woods
       QueryResult = Struct.new(:query, :expected_units, :retrieved_units, :scores, :tokens_used,
                                keyword_init: true)
 
-      # Aggregate report across all queries.
-      EvaluationReport = Struct.new(:results, :aggregates, keyword_init: true)
+      # Aggregate report across all queries. +threshold_report+ is nil unless
+      # thresholds were given — absent thresholds stay report-only.
+      EvaluationReport = Struct.new(:results, :aggregates, :threshold_report, keyword_init: true)
+
+      # Structured pass/fail against a thresholds hash. +metrics+ maps each
+      # thresholded aggregate key to { threshold:, actual:, delta:, passed: }.
+      ThresholdReport = Struct.new(:thresholds, :metrics, :passed, keyword_init: true)
 
       METRIC_KEYS = %i[precision_at5 precision_at10 recall mrr context_completeness token_efficiency].freeze
 
       # @param retriever [Woods::Retriever] Configured retriever instance
       # @param query_set [QuerySet] Set of evaluation queries with ground truth
       # @param budget [Integer] Token budget per query
-      def initialize(retriever:, query_set:, budget: 8000)
+      # @param thresholds [Hash{Symbol=>Numeric}, nil] minimum value per
+      #   aggregate key (e.g. `mean_recall: 0.7`). nil/empty means report-only:
+      #   metrics are still computed but nothing is scored pass/fail.
+      def initialize(retriever:, query_set:, budget: 8000, thresholds: nil)
         @retriever = retriever
         @query_set = query_set
         @budget = budget
+        @thresholds = thresholds
       end
 
       # Run all queries and produce an evaluation report.
       #
-      # @return [EvaluationReport] Per-query results and aggregate metrics
+      # @return [EvaluationReport] Per-query results, aggregate metrics, and
+      #   (when thresholds were given) a structured pass/fail
       def evaluate
         results = @query_set.queries.map { |q| evaluate_query(q) }
         aggregates = compute_aggregates(results)
-        EvaluationReport.new(results: results, aggregates: aggregates)
+        EvaluationReport.new(results: results, aggregates: aggregates,
+                             threshold_report: evaluate_thresholds(aggregates))
       end
 
       private
 
+      # @param aggregates [Hash] computed aggregate metrics
+      # @return [ThresholdReport, nil] nil when no thresholds were configured
+      def evaluate_thresholds(aggregates)
+        return nil if @thresholds.nil? || @thresholds.empty?
+
+        metrics = @thresholds.to_h do |key, minimum|
+          actual = aggregates[key]
+          passed = !actual.nil? && actual >= minimum
+          delta = actual.nil? ? nil : (actual - minimum).round(6)
+          [key, { threshold: minimum, actual: actual, delta: delta, passed: passed }]
+        end
+
+        ThresholdReport.new(thresholds: @thresholds, metrics: metrics, passed: metrics.values.all? { |m| m[:passed] })
+      end
+
       # Evaluate a single query against the retriever.
       #
       # @param query [QuerySet::Query] Evaluation query
@@ -53,7 +79,8 @@ module Woods
         retrieval_result = @retriever.retrieve(query.query, budget: @budget)
         retrieved_ids = extract_identifiers(retrieval_result)
 
-        scores = compute_scores(retrieved_ids, query.expected_units, retrieval_result)
+        scores = compute_scores(retrieved_ids, query.expected_units, retrieval_result,
+                                required: query.completeness_units)
 
         QueryResult.new(
           query: query.query,
@@ -76,17 +103,23 @@ module Woods
 
       # Compute all metrics for a query result.
       #
+      # +required+ is what separates `context_completeness` from `recall`
+      # (EXP-10): passing `expected` for both made the two metrics one number
+      # under two names. A query with no `required_units` annotation still
+      # passes its expected set, keeping the historical value.
+      #
       # @param retrieved [Array<String>] Retrieved identifiers
       # @param expected [Array<String>] Expected identifiers
       # @param result [Retriever::RetrievalResult] Retrieval result
+      # @param required [Array<String>] Identifiers the retrieval must surface
       # @return [Hash] Metric scores
-      def compute_scores(retrieved, expected, result)
+      def compute_scores(retrieved, expected, result, required: expected)
         {
           precision_at5: Metrics.precision_at_k(retrieved, expected, cutoff: 5),
           precision_at10: Metrics.precision_at_k(retrieved, expected, cutoff: 10),
           recall: Metrics.recall(retrieved, expected),
           mrr: Metrics.mrr(retrieved, expected),
-          context_completeness: Metrics.context_completeness(retrieved, expected),
+          context_completeness: Metrics.context_completeness(retrieved, required),
           token_efficiency: compute_token_efficiency(retrieved, expected, result)
         }
       end

data/lib/woods/evaluation/query_set.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require 'json'
+require_relative '../retrieval/query_classifier'
+
 module Woods
   module Evaluation
     # Manages a set of evaluation queries with expected results.
@@ -21,15 +24,50 @@ module Woods
       # @!attribute [r] expected_units
       #   @return [Array<String>] Expected unit identifiers (ground truth)
       # @!attribute [r] intent
-      #   @return [Symbol] Query intent (:lookup, :trace, :explain, :compare)
+      #   @return [Symbol] Query intent — one of {VALID_INTENTS}, the same
+      #     vocabulary {Woods::Retrieval::QueryClassifier} produces
       # @!attribute [r] scope
-      #   @return [Symbol] Query scope (:specific, :bounded, :broad)
+      #   @return [Symbol] Query scope — one of {VALID_SCOPES}, the same
+      #     vocabulary {Woods::Retrieval::QueryClassifier} produces
       # @!attribute [r] tags
       #   @return [Array<String>] Tags for filtering queries
-      Query = Struct.new(:query, :expected_units, :intent, :scope, :tags, keyword_init: true)
+      # @!attribute [r] required_units
+      #   @return [Array<String>, nil] The subset of {#expected_units} a
+      #     retrieval *must* surface for the answer to be usable. Optional;
+      #     absent (or empty) means "every expected unit is required", which
+      #     is the semantics the harness had before the field existed.
+      Query = Struct.new(:query, :expected_units, :intent, :scope, :tags, :required_units, keyword_init: true) do
+        # The set +context_completeness+ is scored against.
+        #
+        # +context_completeness+ used to be handed {#expected_units}, making it
+        # recall under a second name (EXP-10). A query that annotates
+        # +required_units+ separates the two; one that doesn't keeps the old
+        # meaning.
+        #
+        # @return [Array<String>]
+        def completeness_units
+          required = required_units
+          required.nil? || required.empty? ? (expected_units || []) : required
+        end
+      end
 
-      VALID_INTENTS = %i[lookup trace explain compare].freeze
-      VALID_SCOPES = %i[specific bounded broad].freeze
+      # The annotation vocabulary is the *classifier's* vocabulary (#218 /
+      # B-105). These lists used to be a parallel invention — intents
+      # `lookup/trace/explain/compare` against the classifier's
+      # `understand/locate/trace/debug/implement/reference/compare/framework`,
+      # and scopes `specific/bounded/broad` against
+      # `pinpoint/focused/exploratory/comprehensive`, which share nothing at
+      # all. A ground-truth annotation could therefore never be compared with
+      # what the pipeline actually classified a query as, so the harness could
+      # measure retrieval hits but never classification accuracy.
+      VALID_INTENTS = Woods::Retrieval::QueryClassifier::INTENTS
+      VALID_SCOPES = Woods::Retrieval::QueryClassifier::SCOPES
+
+      # Old annotation values, accepted on load and translated. No shipped
+      # query set uses them (the harness had no working entry point until
+      # #212), but a hand-written file might.
+      LEGACY_INTENTS = { lookup: :locate, explain: :understand }.freeze
+      LEGACY_SCOPES = { specific: :pinpoint, bounded: :focused, broad: :exploratory }.freeze
 
       # @return [Array<Query>] The queries in this set
       attr_reader :queries
@@ -47,7 +85,12 @@ module Woods
       # @return [QuerySet] Loaded query set
       # @raise [Woods::Error] if the file cannot be read or parsed
       def self.load(path)
-        data = JSON.parse(File.read(path))
+        # Explicit UTF-8: a query set is UTF-8 on disk, and a bare read tags
+        # it with the process default external encoding — under a POSIX locale
+        # the first accented query raised a raw Encoding error out of
+        # JSON.parse, past both typed branches below (EXP-7). Same pattern as
+        # {Woods::Evaluation::Baseline.load}.
+        data = JSON.parse(File.read(path, encoding: 'UTF-8'))
         queries = data.fetch('queries', []).map { |q| parse_query(q) }
         new(queries: queries)
       rescue JSON::ParserError => e
@@ -64,7 +107,9 @@ module Woods
         data = {
           'queries' => queries.map { |q| serialize_query(q) }
         }
-        File.write(path, JSON.pretty_generate(data))
+        # Pair the explicit UTF-8 with {.load} so a saved set reads back
+        # byte-for-byte regardless of the writer's locale (EXP-7).
+        File.write(path, JSON.pretty_generate(data), encoding: 'UTF-8')
       end
 
       # Filter queries by intent, scope, or tags.
@@ -108,9 +153,10 @@ module Woods
         Query.new(
           query: hash.fetch('query'),
           expected_units: hash.fetch('expected_units', []),
-          intent: hash.fetch('intent', 'lookup').to_sym,
-          scope: hash.fetch('scope', 'specific').to_sym,
-          tags: hash.fetch('tags', [])
+          intent: LEGACY_INTENTS.fetch(hash.fetch('intent', 'understand').to_sym) { |v| v },
+          scope: LEGACY_SCOPES.fetch(hash.fetch('scope', 'focused').to_sym) { |v| v },
+          tags: hash.fetch('tags', []),
+          required_units: hash['required_units']
         )
       end
 
@@ -118,16 +164,21 @@ module Woods
 
       # Serialize a Query to a hash for JSON output.
       #
+      # +required_units+ is omitted when the query does not annotate one, so a
+      # set that never used the field round-trips byte-identically.
+      #
       # @param query [Query] Query to serialize
       # @return [Hash]
       def serialize_query(query)
-        {
+        data = {
           'query' => query.query,
           'expected_units' => query.expected_units,
           'intent' => query.intent.to_s,
           'scope' => query.scope.to_s,
           'tags' => query.tags
         }
+        data['required_units'] = query.required_units if query.required_units
+        data
       end
 
       # Validate intent and scope values.
@@ -139,9 +190,24 @@ module Woods
           raise ArgumentError, "Invalid intent: #{query.intent}. Must be one of #{VALID_INTENTS.join(', ')}"
         end
 
-        return if VALID_SCOPES.include?(query.scope)
+        unless VALID_SCOPES.include?(query.scope)
+          raise ArgumentError, "Invalid scope: #{query.scope}. Must be one of #{VALID_SCOPES.join(', ')}"
+        end
+
+        validate_required_units!(query)
+      end
+
+      # +required_units+ is documented as a subset of +expected_units+; a
+      # required unit outside the ground truth could never be scored as
+      # retrieved-and-relevant, so it is a typo, not a stricter bar.
+      #
+      # @param query [Query]
+      # @raise [ArgumentError] when a required unit is not expected
+      def validate_required_units!(query)
+        stray = Array(query.required_units) - Array(query.expected_units)
+        return if stray.empty?
 
-        raise ArgumentError, "Invalid scope: #{query.scope}. Must be one of #{VALID_SCOPES.join(', ')}"
+        raise ArgumentError, "required_units must be a subset of expected_units; unknown: #{stray.join(', ')}"
       end
     end
   end

data/lib/woods/evaluation/report_generator.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require 'json'
+require 'time'      # Time#iso8601 — the file is loadable standalone (EXP-8)
+require 'fileutils' # FileUtils.mkdir_p in #save
 
 module Woods
   module Evaluation
@@ -44,11 +46,14 @@ module Woods
       # @param metadata [Hash] Additional metadata
       # @return [Hash]
       def build_report_hash(report, metadata)
-        {
+        hash = {
           'metadata' => build_metadata(metadata),
           'aggregates' => serialize_aggregates(report.aggregates),
           'results' => report.results.map { |r| serialize_result(r) }
         }
+        threshold_report = report.threshold_report
+        hash['threshold_report'] = serialize_threshold_report(threshold_report) if threshold_report
+        hash
       end
 
       # Build the metadata section.
@@ -72,6 +77,20 @@ module Woods
         end
       end
 
+      # Serialize a threshold report (nil-safe values: a missing aggregate has
+      # a nil actual/delta, which JSON renders as null rather than raising).
+      #
+      # @param threshold_report [Evaluator::ThresholdReport]
+      # @return [Hash] String-keyed hash
+      def serialize_threshold_report(threshold_report)
+        {
+          'passed' => threshold_report.passed,
+          'metrics' => threshold_report.metrics.transform_keys(&:to_s).transform_values do |m|
+            m.transform_keys(&:to_s)
+          end
+        }
+      end
+
       # Serialize a single query result.
       #
       # @param result [Evaluator::QueryResult] Query result

data/lib/woods/export/unit_facts.rb

@@ -59,17 +59,6 @@ module Woods
         @meta['callbacks'].map { |cb| { type: cb['type'], filter: cb['filter'] } }
       end
 
-      # Blast-radius bucket for a dependent count.
-      #
-      # @param dependent_count [Integer]
-      # @return [Symbol] :high (>50), :moderate (>20), or :none
-      def self.blast_radius(dependent_count)
-        return :high if dependent_count > 50
-        return :moderate if dependent_count > 20
-
-        :none
-      end
-
       private
 
       def associations

data/lib/woods/extracted_unit.rb

@@ -69,10 +69,31 @@ module Woods
     # @return [Integer] Estimated token count
     def estimated_tokens
       source_tokens = source_code ? (source_code.length / 4.0).ceil : 0
-      metadata_tokens = metadata.any? ? (metadata.to_json.length / 4.0).ceil : 0
+      metadata_tokens = metadata.any? ? (serialized_metadata.length / 4.0).ceil : 0
       source_tokens + metadata_tokens
     end
 
+    # Metadata serialized the way the index writer serializes it.
+    #
+    # `Hash#to_json` and `JSON.generate` disagree about values JSON has no
+    # native representation for. With ActiveSupport loaded, `to_json` routes
+    # through `as_json`, which renders a Class as its (empty) instance values
+    # — `{}`. `JSON.generate`, which `Extractor#json_serialize` uses to write
+    # the unit file, falls back to `to_s` — `"ActionDispatch::Session::CookieStore"`.
+    #
+    # Estimating against `to_json` therefore described a document that was
+    # never written: on Rails < 7.1, `BehavioralProfile` carries
+    # `config.session_store` as a Class, and the estimate came out ~9 tokens
+    # short of the file on disk. Since an incremental run recomputes the
+    # estimate by reading that file back, the two paths disagreed (#164).
+    #
+    # @return [String]
+    def serialized_metadata
+      JSON.generate(metadata)
+    rescue StandardError
+      metadata.to_json
+    end
+
     # Check if unit needs chunking based on size
     #
     # @param threshold [Integer] Token threshold for chunking (default: 1500)
@@ -80,67 +101,5 @@ module Woods
     def needs_chunking?(threshold: 1500)
       estimated_tokens > threshold
     end
-
-    # Build semantic chunks for large units
-    # Preserves context by including unit header in each chunk
-    #
-    # @param max_tokens [Integer] Maximum tokens per chunk
-    # @return [Array<Hash>] List of chunk hashes
-    def build_default_chunks(max_tokens: 1500)
-      return [] unless needs_chunking?
-
-      chunks = []
-      current_chunk = []
-      current_tokens = 0
-
-      # Always include a header with unit context
-      header = build_chunk_header
-      header_tokens = (header.length / 4.0).ceil
-
-      source_code.lines.each do |line|
-        line_tokens = (line.length / 4.0).ceil
-
-        if current_tokens + line_tokens > max_tokens && current_chunk.any?
-          content = header + current_chunk.join
-          chunks << {
-            chunk_index: chunks.size,
-            identifier: "#{identifier}#chunk_#{chunks.size}",
-            content: content,
-            content_hash: Digest::SHA256.hexdigest(content),
-            estimated_tokens: current_tokens + header_tokens
-          }
-          current_chunk = []
-          current_tokens = 0
-        end
-
-        current_chunk << line
-        current_tokens += line_tokens
-      end
-
-      # Final chunk
-      if current_chunk.any?
-        content = header + current_chunk.join
-        chunks << {
-          chunk_index: chunks.size,
-          identifier: "#{identifier}#chunk_#{chunks.size}",
-          content: content,
-          content_hash: Digest::SHA256.hexdigest(content),
-          estimated_tokens: current_tokens + header_tokens
-        }
-      end
-
-      chunks
-    end
-
-    private
-
-    def build_chunk_header
-      <<~HEADER
-        # Unit: #{identifier} (#{type})
-        # File: #{file_path}
-        # Namespace: #{namespace || '(root)'}
-        # ---
-      HEADER
-    end
   end
 end