woods 1.4.1 → 1.6.0

data/lib/woods/extractor.rb

@@ -8,8 +8,10 @@ require 'pathname'
 require 'set'
 
 require_relative 'filename_utils'
+require_relative 'token_utils'
 require_relative 'extracted_unit'
 require_relative 'dependency_graph'
+require_relative 'git_provenance'
 require_relative 'extractors/model_extractor'
 require_relative 'extractors/controller_extractor'
 require_relative 'extractors/phlex_extractor'
@@ -248,12 +250,6 @@ module Woods
       Rails.logger.info '[Woods] Analyzing dependency graph...'
       @graph_analysis = GraphAnalyzer.new(@dependency_graph).analyze
 
-      # Phase 3.5: Precompute request flows (opt-in)
-      if Woods.configuration.precompute_flows
-        Rails.logger.info '[Woods] Precomputing request flows...'
-        precompute_flows
-      end
-
       # Phase 4: Enrich with git data
       Rails.logger.info '[Woods] Enriching with git data...'
       enrich_with_git_data
@@ -265,6 +261,17 @@ module Woods
       # Phase 5: Write output
       Rails.logger.info '[Woods] Writing output...'
       write_results
+
+      # Phase 5.5: Precompute request flows (opt-in). Must run AFTER
+      # write_results — FlowAssembler loads unit JSON from disk, so running
+      # earlier assembled every flow from absent (fresh output dir) or
+      # stale (previous run's) data. precompute_flows re-writes the
+      # controller units it annotates with metadata[:flow_paths].
+      if Woods.configuration.precompute_flows
+        Rails.logger.info '[Woods] Precomputing request flows...'
+        precompute_flows
+      end
+
       write_dependency_graph
       write_graph_analysis
       write_manifest
@@ -315,11 +322,17 @@ module Woods
         regenerate_type_index(type_key)
       end
 
-      # Update graph, manifest, and summary
+      # Update graph, manifest, and summary. No capture_snapshot here:
+      # snapshots must hash the FULL unit set, and incremental runs only
+      # re-extract affected units (@results stays empty) — capturing would
+      # record a snapshot whose diff reports every unit as deleted.
+      # Snapshots are captured on full extraction only.
       write_dependency_graph
-      write_manifest
+      write_manifest(incremental: true)
       write_structural_summary
-      capture_snapshot
+      if Woods.configuration.enable_snapshots
+        Rails.logger.info '[Woods] Skipping snapshot capture — snapshots are captured on full extraction only'
+      end
 
       affected_ids
     end
@@ -505,11 +518,42 @@ module Woods
       all_units = @results.values.flatten(1)
       precomputer = FlowPrecomputer.new(units: all_units, graph: @dependency_graph, output_dir: @output_dir.to_s)
       flow_map = precomputer.precompute
+      rewrite_flow_annotated_units
       Rails.logger.info "[Woods] Precomputed #{flow_map.size} request flows"
     rescue StandardError => e
       Rails.logger.error "[Woods] Flow precomputation failed: #{e.message}"
     end
 
+    # Precompute runs after write_results (FlowAssembler reads unit JSON
+    # from disk), so units annotated in memory with metadata[:flow_paths]
+    # must be re-written and their type index refreshed to pick up the
+    # annotation.
+    #
+    # The index is rebuilt from the in-memory `units` (the authoritative
+    # full-extraction `@results`), NOT from a disk glob: this is a full
+    # extraction, the output dir is never wiped, and globbing would
+    # resurrect stale unit files for app classes deleted since the last run.
+    # (The incremental path, which only holds changed units in memory, still
+    # rebuilds from disk via {#regenerate_type_index}.)
+    def rewrite_flow_annotated_units
+      @results.each do |type, units|
+        annotated = units.select { |u| u.metadata[:flow_paths] }
+        next if annotated.empty?
+
+        type_dir = @output_dir.join(type.to_s)
+        annotated.each do |unit|
+          File.write(
+            type_dir.join(collision_safe_filename(unit.identifier)),
+            json_serialize(unit.to_h)
+          )
+        end
+        File.write(
+          type_dir.join('_index.json'),
+          json_serialize(type_index_entries(units))
+        )
+      end
+    end
+
     # ──────────────────────────────────────────────────────────────────────
     # Git Enrichment
     # ──────────────────────────────────────────────────────────────────────
@@ -701,23 +745,32 @@ module Woods
         end
 
         # Also write a type index for fast lookups
-        index = units.map do |u|
-          {
-            identifier: u.identifier,
-            file_path: u.file_path,
-            namespace: u.namespace,
-            estimated_tokens: u.estimated_tokens,
-            chunk_count: u.chunks.size
-          }
-        end
-
         File.write(
           type_dir.join('_index.json'),
-          json_serialize(index)
+          json_serialize(type_index_entries(units))
         )
       end
     end
 
+    # Build the `_index.json` entry list for a set of in-memory units.
+    # Shared by {#write_results} and {#rewrite_flow_annotated_units} so both
+    # emit the index from the authoritative in-memory `@results` rather than
+    # re-deriving it from disk.
+    #
+    # @param units [Array<ExtractedUnit>]
+    # @return [Array<Hash>]
+    def type_index_entries(units)
+      units.map do |u|
+        {
+          identifier: u.identifier,
+          file_path: u.file_path,
+          namespace: u.namespace,
+          estimated_tokens: u.estimated_tokens,
+          chunk_count: u.chunks.size
+        }
+      end
+    end
+
     def write_dependency_graph
       graph_data = @dependency_graph.to_h
       graph_data[:pagerank] = @dependency_graph.pagerank
@@ -744,23 +797,39 @@ module Woods
       )
     end
 
-    def write_manifest
+    def write_manifest(incremental: false)
+      # Worktree-aware git provenance. In a linked worktree +.git+ is a file
+      # pointing at the real git dir; when that dir is unreachable (e.g. an
+      # unmounted host path inside a container) this resolves to "unknown"
+      # rather than a stale GIT_BRANCH/GIT_SHA build arg. See GitProvenance (#137).
+      provenance = GitProvenance.new(root: Rails.root).to_h
+
+      # Incremental runs never populate @results — deriving counts from
+      # memory would clobber a good manifest with zeros. Recompute from the
+      # persisted per-type _index.json files instead.
+      counts, total_chunks =
+        if incremental
+          persisted_counts
+        else
+          [@results.transform_values(&:size),
+           @results.sum { |_, units| units.sum { |u| u.chunks.size } }]
+        end
+
       manifest = {
         extracted_at: Time.current.iso8601,
         rails_version: Rails.version,
         ruby_version: RUBY_VERSION,
 
         # Counts by type
-        counts: @results.transform_values(&:size),
+        counts: counts,
 
         # Total stats
-        total_units: @results.values.sum(&:size),
-        total_chunks: @results.sum { |_, units| units.sum { |u| u.chunks.size } },
+        total_units: counts.values.sum,
+        total_chunks: total_chunks,
 
-        # Git info — fall back to env vars for Docker/worktree environments
-        # where the git repo may not be directly accessible
-        git_sha: run_git('rev-parse', 'HEAD').presence || ENV['GIT_SHA'].presence,
-        git_branch: run_git('rev-parse', '--abbrev-ref', 'HEAD').presence || ENV['GIT_BRANCH'].presence,
+        # Git provenance (branch/sha), or "unknown" when unresolvable
+        git_sha: provenance[:git_sha],
+        git_branch: provenance[:git_branch],
 
         # For change detection
         gemfile_lock_sha: gemfile_lock_sha,
@@ -773,6 +842,30 @@ module Woods
       )
     end
 
+    # Unit and chunk counts derived from the per-type _index.json files on
+    # disk — the source of truth after an incremental run, where only the
+    # affected units were re-extracted.
+    #
+    # @return [Array(Hash{Symbol => Integer}, Integer)] counts by type, total chunk count
+    def persisted_counts
+      counts = {}
+      chunks = 0
+
+      Dir[@output_dir.join('*/_index.json').to_s].each do |index_path|
+        entries = JSON.parse(File.read(index_path))
+        counts[File.basename(File.dirname(index_path)).to_sym] = entries.size
+        chunks += entries.sum { |e| e['chunk_count'].to_i }
+      rescue JSON::ParserError => e
+        # An unreadable index silently drops that whole type from the manifest
+        # counts — warn rather than undercount without a trace.
+        type = File.basename(File.dirname(index_path))
+        Rails.logger.warn("[Woods] Skipping unreadable #{type}/_index.json in manifest counts: #{e.message}")
+        next
+      end
+
+      [counts, chunks]
+    end
+
     # Capture a temporal snapshot after extraction completes.
     #
     # Reads the manifest and computes per-unit content hashes, then delegates
@@ -787,7 +880,10 @@ module Woods
       return unless manifest_path.exist?
 
       manifest = JSON.parse(File.read(manifest_path))
-      return unless manifest['git_sha']
+      # Snapshots are keyed on the commit SHA — an unresolvable provenance
+      # ("unknown", see GitProvenance/#137) must not key or collide a snapshot.
+      git_sha = manifest['git_sha']
+      return if git_sha.nil? || git_sha == Woods::GitProvenance::UNKNOWN
 
       store = build_snapshot_store
       return unless store
@@ -906,7 +1002,10 @@ module Woods
           identifier: data['identifier'],
           file_path: data['file_path'],
           namespace: data['namespace'],
-          estimated_tokens: data['estimated_tokens'],
+          # Unit JSON has no estimated_tokens field (ExtractedUnit#to_h
+          # doesn't emit one) — recompute it, or every unit of a type
+          # touched by an incremental run would index as null.
+          estimated_tokens: estimated_tokens_from(data),
           chunk_count: (data['chunks'] || []).size
         }
       end
@@ -917,6 +1016,20 @@ module Woods
       )
     end
 
+    # Token estimate for a unit parsed back from JSON, mirroring
+    # ExtractedUnit#estimated_tokens (see docs/TOKEN_BENCHMARK.md).
+    #
+    # @param data [Hash] Parsed unit JSON (string keys)
+    # @return [Integer]
+    def estimated_tokens_from(data)
+      source = data['source_code']
+      metadata = data['metadata'] || {}
+
+      source_tokens = source ? TokenUtils.estimate_tokens(source) : 0
+      metadata_tokens = metadata.any? ? TokenUtils.estimate_tokens(metadata.to_json) : 0
+      source_tokens + metadata_tokens
+    end
+
     # ──────────────────────────────────────────────────────────────────────
     # Helpers
     # ──────────────────────────────────────────────────────────────────────
@@ -1014,12 +1127,20 @@ module Woods
 
       return unless unit
 
-      # Update dependency graph
+      # Update dependency graph. Register BEFORE normalizing the path —
+      # the graph's file_map stores absolute paths (affected_by matches
+      # changed files against them), exactly as full extraction registers
+      # in Phase 1 and only normalizes in Phase 4.5.
       @dependency_graph.register(unit)
 
       # Track which type was affected
       affected_types&.add(extractor_key)
 
+      # Unit JSON carries Rails.root-relative paths (full extraction's
+      # Phase 4.5); writing the raw absolute source_location here would
+      # leak container-absolute paths into the index after incremental runs.
+      unit.file_path = normalize_file_path(unit.file_path)
+
       # Write updated unit
       type_dir = @output_dir.join(extractor_key.to_s)
 

data/lib/woods/extractors/concern_extractor.rb

@@ -48,10 +48,8 @@ module Woods
       #
       # @return [Array<ExtractedUnit>] List of concern units
       def extract_all
-        @directories.flat_map do |dir|
-          Dir[dir.join('**/*.rb')].filter_map do |file|
-            extract_concern_file(file)
-          end
+        find_files_in_directories(@directories).filter_map do |file|
+          extract_concern_file(file)
         end
       end
 
@@ -285,7 +283,7 @@ module Woods
         deps.concat(scan_service_dependencies(source))
         deps.concat(scan_job_dependencies(source))
 
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
     end
   end

data/lib/woods/extractors/configuration_extractor.rb

@@ -36,10 +36,8 @@ module Woods
       #
       # @return [Array<ExtractedUnit>] List of configuration units
       def extract_all
-        units = @directories.flat_map do |dir|
-          Dir[dir.join('**/*.rb')].filter_map do |file|
-            extract_configuration_file(file)
-          end
+        units = find_files_in_directories(@directories).filter_map do |file|
+          extract_configuration_file(file)
         end
 
         profile = BehavioralProfile.new.extract
@@ -212,7 +210,7 @@ module Woods
 
         deps.concat(scan_service_dependencies(source))
 
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
     end
   end

data/lib/woods/extractors/controller_extractor.rb

@@ -320,7 +320,7 @@ module Woods
           deps.concat(scan_navigation_dependencies(source, via_type: :redirect_to))
         end
 
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
 
       # ──────────────────────────────────────────────────────────────────────

data/lib/woods/extractors/database_view_extractor.rb

@@ -271,7 +271,7 @@ module Woods
           deps << { type: :model, target: model_name, via: :table_name }
         end
 
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
     end
   end

data/lib/woods/extractors/decorator_extractor.rb

@@ -49,10 +49,8 @@ module Woods
       #
       # @return [Array<ExtractedUnit>] List of decorator units
       def extract_all
-        @directories.flat_map do |dir|
-          Dir[dir.join('**/*.rb')].filter_map do |file|
-            extract_decorator_file(file)
-          end
+        find_files_in_directories(@directories).filter_map do |file|
+          extract_decorator_file(file)
         end
       end
 
@@ -246,7 +244,7 @@ module Woods
 
         deps.concat(scan_common_dependencies(source))
 
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
     end
   end

data/lib/woods/extractors/event_extractor.rb

@@ -42,7 +42,7 @@ module Woods
       def extract_all
         event_map = {}
 
-        @directories.flat_map { |dir| Dir[dir.join('**/*.rb')] }.each do |file_path|
+        find_files_in_directories(@directories).each do |file_path|
           scan_file(file_path, event_map)
         end
 
@@ -204,7 +204,7 @@ module Woods
       # @return [Array<Hash>]
       def build_dependencies(combined_source)
         deps = scan_common_dependencies(combined_source)
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
     end
   end

data/lib/woods/extractors/factory_extractor.rb

@@ -35,9 +35,7 @@ module Woods
       #
       # @return [Array<ExtractedUnit>] List of factory units
       def extract_all
-        @directories.flat_map do |dir|
-          Dir[dir.join('**/*.rb')].flat_map { |file| extract_factory_file(file) }
-        end
+        find_files_in_directories(@directories).flat_map { |file| extract_factory_file(file) }
       end
 
       # Extract factory definitions from a single factory file.
@@ -282,7 +280,7 @@ module Woods
           deps << { type: :factory, target: assoc, via: :factory_association }
         end
 
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
     end
   end

data/lib/woods/extractors/graphql_extractor.rb

@@ -770,7 +770,7 @@ module Woods
           deps << { type: :graphql_resolver, target: resolver, via: :field_resolver }
         end
 
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
 
       # ──────────────────────────────────────────────────────────────────────

data/lib/woods/extractors/i18n_extractor.rb

@@ -2,6 +2,8 @@
 
 require 'yaml'
 
+require_relative 'shared_utility_methods'
+
 module Woods
   module Extractors
     # I18nExtractor handles internationalization locale file extraction.
@@ -16,6 +18,8 @@ module Woods
     #   en = units.find { |u| u.identifier == "en.yml" }
     #
     class I18nExtractor
+      include SharedUtilityMethods
+
       # Directories to scan for locale files
       I18N_DIRECTORIES = %w[
         config/locales
@@ -30,10 +34,8 @@ module Woods
       #
       # @return [Array<ExtractedUnit>] List of i18n units
       def extract_all
-        @directories.flat_map do |dir|
-          Dir[dir.join('**/*.yml')].filter_map do |file|
-            extract_i18n_file(file)
-          end
+        find_files_in_directories(@directories, '**/*.yml').filter_map do |file|
+          extract_i18n_file(file)
         end
       end
 

data/lib/woods/extractors/job_extractor.rb

@@ -46,11 +46,9 @@ module Woods
         units = []
 
         # File-based discovery (catches everything)
-        @directories.each do |dir|
-          Dir[dir.join('**/*.rb')].each do |file|
-            unit = extract_job_file(file)
-            units << unit if unit
-          end
+        find_files_in_directories(@directories).each do |file|
+          unit = extract_job_file(file)
+          units << unit if unit
         end
 
         # Also try class-based discovery for ActiveJob
@@ -356,7 +354,7 @@ module Woods
 
         deps << { type: :infrastructure, target: :redis, via: :code_reference } if source.match?(/Redis\.current|REDIS/)
 
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
 
       # Scan source for job class enqueue calls and return the list of enqueued job names.

data/lib/woods/extractors/lib_extractor.rb

@@ -211,7 +211,7 @@ module Woods
       # @return [Array<Hash>] Dependency hashes with :type, :target, :via
       def extract_dependencies(source)
         deps = scan_common_dependencies(source)
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
     end
   end

data/lib/woods/extractors/mailer_extractor.rb

@@ -243,7 +243,7 @@ module Woods
           deps << { type: :route, target: route, via: :url_helper }
         end
 
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
 
       # ──────────────────────────────────────────────────────────────────────

data/lib/woods/extractors/manager_extractor.rb

@@ -40,10 +40,8 @@ module Woods
       #
       # @return [Array<ExtractedUnit>] List of manager units
       def extract_all
-        @directories.flat_map do |dir|
-          Dir[dir.join('**/*.rb')].filter_map do |file|
-            extract_manager_file(file)
-          end
+        find_files_in_directories(@directories).filter_map do |file|
+          extract_manager_file(file)
         end
       end
 
@@ -181,7 +179,7 @@ module Woods
 
         deps.concat(scan_common_dependencies(source))
 
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
     end
   end

data/lib/woods/extractors/migration_extractor.rb

@@ -462,7 +462,7 @@ module Woods
         # Scan data migration code for common dependencies
         deps.concat(scan_common_dependencies(source))
 
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
     end
   end

data/lib/woods/extractors/model_extractor.rb

@@ -501,7 +501,7 @@ module Woods
               options: v.options.except(:if, :unless, :on),
               conditions: format_validation_conditions(v)
             }
-            entry[:implicit_belongs_to] = true if implicit_belongs_to_validator?(v)
+            entry[:implicit_belongs_to] = true if implicit_belongs_to_validator?(model, v)
             entry
           end
         end
@@ -654,7 +654,7 @@ module Woods
           end
         end
 
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
 
       # Enrich callback metadata with side-effect analysis.
@@ -922,17 +922,30 @@ module Woods
         conditions
       end
 
-      # Detect Rails-generated implicit belongs_to presence validators
+      # Detect Rails-generated implicit belongs_to presence validators.
       #
+      # `belongs_to` (with belongs_to_required_by_default, Rails 5+)
+      # registers an ActiveRecord PresenceValidator on the association
+      # attribute. Reflection offers no declaration provenance — the
+      # validator class and its #validate source_location are identical for
+      # an explicit `validates :assoc, presence: true` — so this flags any
+      # AR presence validator whose attributes are all belongs_to
+      # association names. (The previous source_location heuristic matched
+      # EVERY AR presence validator, since PresenceValidator#validate is
+      # always defined in the gem, never under Rails.root.)
+      #
+      # @param model [Class] The ActiveRecord model class
       # @param validator [ActiveModel::Validator]
       # @return [Boolean]
-      def implicit_belongs_to_validator?(validator)
-        if defined?(ActiveRecord::Validations::PresenceValidator) && !validator.is_a?(ActiveRecord::Validations::PresenceValidator)
-          return false
-        end
+      def implicit_belongs_to_validator?(model, validator)
+        return false unless defined?(ActiveRecord::Validations::PresenceValidator)
+        return false unless validator.is_a?(ActiveRecord::Validations::PresenceValidator)
+
+        belongs_to_names = model.reflect_on_all_associations(:belongs_to).map { |r| r.name.to_sym }
+        return false if belongs_to_names.empty?
 
-        loc = validator.class.instance_method(:validate).source_location&.first
-        loc && !loc.start_with?(Rails.root.to_s)
+        attributes = validator.respond_to?(:attributes) ? Array(validator.attributes) : []
+        attributes.any? && attributes.all? { |a| belongs_to_names.include?(a.to_sym) }
       rescue StandardError
         false
       end
@@ -975,7 +988,10 @@ module Woods
 
       def callback_count(model)
         %i[validation save create update destroy commit rollback].sum do |type|
-          model.send("_#{type}_callbacks").size
+          # CallbackChain includes Enumerable but defines no #size on any
+          # supported Rails version — #size raises and the rescue would
+          # zero the count. #count is the only correct API here.
+          model.send("_#{type}_callbacks").count
         rescue StandardError
           0
         end

data/lib/woods/extractors/phlex_extractor.rb

@@ -254,7 +254,7 @@ module Woods
         deps.concat(scan_navigation_dependencies(source))
         deps.concat(scan_form_dependencies(source))
 
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
     end
   end

data/lib/woods/extractors/policy_extractor.rb

@@ -43,10 +43,8 @@ module Woods
       #
       # @return [Array<ExtractedUnit>] List of policy units
       def extract_all
-        @directories.flat_map do |dir|
-          Dir[dir.join('**/*.rb')].filter_map do |file|
-            extract_policy_file(file)
-          end
+        find_files_in_directories(@directories).filter_map do |file|
+          extract_policy_file(file)
         end
       end
 
@@ -184,7 +182,7 @@ module Woods
         deps.concat(scan_service_dependencies(source))
         deps.concat(scan_job_dependencies(source))
 
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
     end
   end

data/lib/woods/extractors/poro_extractor.rb

@@ -222,7 +222,7 @@ module Woods
       # @return [Array<Hash>] Dependency hashes with :type, :target, :via
       def extract_dependencies(source)
         deps = scan_common_dependencies(source)
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
     end
   end

data/lib/woods/extractors/pundit_extractor.rb

@@ -38,10 +38,8 @@ module Woods
       #
       # @return [Array<ExtractedUnit>] List of Pundit policy units
       def extract_all
-        @directories.flat_map do |dir|
-          Dir[dir.join('**/*.rb')].filter_map do |file|
-            extract_pundit_file(file)
-          end
+        find_files_in_directories(@directories).filter_map do |file|
+          extract_pundit_file(file)
         end
       end
 
@@ -216,7 +214,7 @@ module Woods
         deps.concat(scan_model_dependencies(source))
         deps.concat(scan_service_dependencies(source))
 
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
     end
   end