woods 2.0.0.beta2 → 2.0.0.beta4

data/lib/woods/extractor.rb

@@ -8,12 +8,14 @@ require 'pathname'
 require 'set'
 
 require_relative 'atomic_file'
+require_relative 'version'
 require_relative 'filename_utils'
 require_relative 'token_utils'
 require_relative 'extracted_unit'
 require_relative 'dependency_graph'
 require_relative 'payload_store'
 require_relative 'git_provenance'
+require_relative 'git_history'
 require_relative 'extractors/model_extractor'
 require_relative 'extractors/controller_extractor'
 require_relative 'extractors/phlex_extractor'
@@ -55,6 +57,7 @@ require_relative 'flow_precomputer'
 require_relative 'change_set'
 require_relative 'generation'
 require_relative 'path_dispatcher'
+require_relative 'source_inputs/session'
 
 module Woods
   # Extractor is the main orchestrator for codebase extraction.
@@ -208,7 +211,6 @@ module Woods
       configuration: :extract_configuration_file,
       view_template: :extract_view_template_file,
       migration: :extract_migration_file,
-      rake_task: :extract_rake_file,
       decorator: :extract_decorator_file,
       database_view: :extract_view_file,
       caching: :extract_caching_file,
@@ -290,7 +292,7 @@ module Woods
                  method: :extract_from_runtime_type, reconcile_removals: false }
     }.freeze
 
-    # Extractors with no per-file entry point: they scan the whole app (or
+    # Extractors requiring a complete source set: they scan the whole app (or
     # introspect the whole runtime) in one pass, so an incremental run
     # replaces their output wholesale rather than per unit. Before #164
     # these types were simply skipped by incremental runs while
@@ -299,6 +301,7 @@ module Woods
     #
     # @return [Hash{Symbol => Symbol}] extractor key => unit type
     WHOLE_APP_EXTRACTORS = {
+      rake_tasks: :rake_task,
       routes: :route,
       middleware: :middleware,
       engines: :engine,
@@ -355,7 +358,7 @@ module Woods
     # flat index — the output root also holds `generation.json`, `dumps/`,
     # `tasks/`, `woods.sqlite3` and `payloads/` itself, none of which belong
     # to a generation's payload.
-    PAYLOAD_FILES = %w[manifest.json dependency_graph.json graph_analysis.json SUMMARY.md].freeze
+    PAYLOAD_FILES = %w[manifest.json source_inputs.json dependency_graph.json graph_analysis.json SUMMARY.md].freeze
 
     # Payload directories that are not per-type unit directories.
     PAYLOAD_DIRS = %w[flows].freeze
@@ -394,7 +397,9 @@ module Woods
     #
     # @return [Hash] Results keyed by extractor type
     def extract_all
+      profile_started = Process.clock_gettime(Process::CLOCK_MONOTONIC) if profiling?
       setup_output_directory
+      profile_phase('source capture') { begin_source_inputs('full') }
       ModelNameCache.reset!
       # @package_resolver alone is not enough: #package_resolver builds
       # through #extractor_for, which memoizes into @incremental_extractors.
@@ -422,29 +427,33 @@ module Woods
 
       # Phase 1.5: Deduplicate results
       Rails.logger.info '[Woods] Deduplicating results...'
-      deduplicate_results
+      profile_phase('deduplication') { deduplicate_results }
 
       # Phase 1.6: Package membership. Runs before the graph is rebuilt so
       # registration copies metadata[:package] onto the node (#280).
-      annotate_packages
+      profile_phase('package annotation') { annotate_packages }
 
       # Rebuild the graph from deduped results. #164 gave DependencyGraph
       # `#remove`/`#unregister`, so surgical removal is now possible — but a
       # full extraction has just registered every unit including duplicates,
       # and rebuilding from the deduped set is both cheaper and less
       # error-prone than unwinding registrations one at a time.
-      @dependency_graph = DependencyGraph.new
-      @results.each_value { |units| units.each { |u| @dependency_graph.register(u) } }
+      profile_phase('graph rebuild') do
+        @dependency_graph = DependencyGraph.new
+        @results.each_value { |units| units.each { |u| @dependency_graph.register(u) } }
+      end
 
       # Phase 2: Resolve dependents (reverse dependencies)
       Rails.logger.info '[Woods] Resolving dependents...'
-      resolve_dependents
+      profile_phase('dependents') { resolve_dependents }
 
       # Phase 3: Enrich with git data. Runs BEFORE analysis now: the
       # volatile_dependencies report reads commit counts off graph nodes.
       Rails.logger.info '[Woods] Enriching with git data...'
-      enrich_with_git_data
-      annotate_graph_with_git_data
+      profile_phase('git enrichment') do
+        enrich_with_git_data
+        annotate_graph_with_git_data
+      end
 
       # Phase 4: Graph analysis (PageRank, structural metrics)
       Rails.logger.info '[Woods] Analyzing dependency graph...'
@@ -452,7 +461,7 @@ module Woods
 
       # Phase 4.5: Normalize file_path to relative paths
       Rails.logger.info '[Woods] Normalizing file paths...'
-      normalize_file_paths
+      profile_phase('path normalization') { normalize_file_paths }
 
       # Phase 5: Write output
       Rails.logger.info '[Woods] Writing output...'
@@ -462,7 +471,7 @@ module Woods
       # run after write_results — the just-written set is what defines
       # "legitimate" — and belongs to the full path only; the incremental path
       # deletes through the graph instead. See {#sweep_orphaned_unit_files}.
-      sweep_orphaned_unit_files
+      profile_phase('orphan sweep') { sweep_orphaned_unit_files }
 
       # Phase 5.5: Precompute request flows (opt-in). Must run AFTER
       # write_results — FlowAssembler loads unit JSON from disk, so running
@@ -485,18 +494,23 @@ module Woods
         profile_phase('flows') { precompute_flows }
       end
 
-      write_dependency_graph
-      write_graph_analysis
+      profile_phase('graph write') do
+        write_dependency_graph
+        write_graph_analysis
+      end
       profile_phase('manifest and summary') do
         write_manifest
         write_structural_summary
       end
-      capture_snapshot
-      profile_phase('publish') { publish_generation('full') }
+      profile_phase('snapshot') { capture_snapshot }
+      @source_inputs.full_units(@results, consumers: @extractors)
+      publish_generation('full')
 
       log_summary
 
       @results
+    ensure
+      log_profile_total('full', profile_started)
     end
 
     # ══════════════════════════════════════════════════════════════════════
@@ -528,7 +542,8 @@ module Woods
     # @param changed_files [Array<String>] List of changed file paths
     # @return [Array<String>] Identifiers of units re-extracted, added, or removed
     def extract_changed(changed_files)
-      prepare_incremental_run
+      profile_started = Process.clock_gettime(Process::CLOCK_MONOTONIC) if profiling?
+      prepare_incremental_run(operation: 'incremental')
 
       change_set = ChangeSet.new(paths: changed_files, root: Rails.root)
       affected_types = Set.new
@@ -551,37 +566,41 @@ module Woods
         acc
       end
 
-      touched.merge(reconcile_class_based_types(affected_types))
-      touched.merge(rerun_whole_app_extractors(change_set, affected_types))
-      touched.merge(reannotate_packages(change_set, affected_types))
-      pruned = prune_vanished_units(change_set, affected_types)
-      touched.merge(pruned)
-
-      # Reconcile once more, because pruning can un-know a class the first pass
-      # skipped. A class-based file moved between autoload directories with its
-      # constant unchanged is still registered under the old path when
-      # reconciliation runs, so it looks known and is not re-extracted; the
-      # prune that follows then removes it for its vanished path. This pass
-      # re-adds it in the same run (M1) instead of leaving the unit missing
-      # until some later run happens to notice. Idempotent when nothing was
-      # pruned: the discovery set is compared against the graph, so an
-      # already-registered class is skipped.
-      #
-      # But not everything pruning removed may come back. `except:` keeps the
-      # *deletion* shape pruned: without a reload, a constant outlives the file
-      # that defined it — so deleting `app/models/user.rb` prunes `User`, and
-      # this pass finds `User` still in `ActiveRecord::Base.descendants`.
-      # Re-registering it would pin the unit to a path that no longer exists,
-      # and nothing could ever remove it: the sweep excludes class-based units
-      # and no future change set names that path again. A resident daemon
-      # processing a batch before its reload hits this every time. What
-      # separates the two shapes is the filesystem — only pruned identifiers
-      # that a still-existing file in the change set actually declares are
-      # re-addable. See {#readdable_pruned_classes}.
-      touched.merge(reconcile_class_based_types(
-                      affected_types, except: pruned - readdable_pruned_classes(pruned, change_set)
-                    ))
-
+      profile_phase('reconciliation') do
+        touched.merge(reconcile_class_based_types(affected_types))
+        touched.merge(reconcile_model_mixins(affected_types))
+        touched.merge(rerun_whole_app_extractors(change_set, affected_types))
+        touched.merge(reannotate_packages(change_set, affected_types))
+        pruned = prune_vanished_units(change_set, affected_types)
+        touched.merge(pruned)
+
+        # Reconcile once more, because pruning can un-know a class the first pass
+        # skipped. A class-based file moved between autoload directories with its
+        # constant unchanged is still registered under the old path when
+        # reconciliation runs, so it looks known and is not re-extracted; the
+        # prune that follows then removes it for its vanished path. This pass
+        # re-adds it in the same run (M1) instead of leaving the unit missing
+        # until some later run happens to notice. Idempotent when nothing was
+        # pruned: the discovery set is compared against the graph, so an
+        # already-registered class is skipped.
+        #
+        # But not everything pruning removed may come back. `except:` keeps the
+        # *deletion* shape pruned: without a reload, a constant outlives the file
+        # that defined it — so deleting `app/models/user.rb` prunes `User`, and
+        # this pass finds `User` still in `ActiveRecord::Base.descendants`.
+        # Re-registering it would pin the unit to a path that no longer exists,
+        # and nothing could ever remove it: the sweep excludes class-based units
+        # and no future change set names that path again. A resident daemon
+        # processing a batch before its reload hits this every time. What
+        # separates the two shapes is the filesystem — only pruned identifiers
+        # that a still-existing file in the change set actually declares are
+        # re-addable. See {#readdable_pruned_classes}.
+        touched.merge(reconcile_class_based_types(
+                        affected_types, except: pruned - readdable_pruned_classes(pruned, change_set)
+                      ))
+      end
+
+      raise_on_handled_extraction_failure!
       finalize_incremental_unit_json(affected_types)
 
       # Regenerate type indexes for affected types
@@ -594,6 +613,8 @@ module Woods
       finalize_incremental_run(touched)
 
       touched.to_a
+    ensure
+      log_profile_total('incremental', profile_started)
     end
 
     # ══════════════════════════════════════════════════════════════════════
@@ -628,6 +649,7 @@ module Woods
     #   extractor
     # @raise [ArgumentError] when no recognized key is given
     def refresh(*keys)
+      profile_started = Process.clock_gettime(Process::CLOCK_MONOTONIC) if profiling?
       keys = Array(keys).flatten.map(&:to_sym).uniq
       known, unknown = keys.partition { |key| EXTRACTORS.key?(key) }
       raise ArgumentError, "No known extractor in #{keys.inspect}" if known.empty?
@@ -635,17 +657,20 @@ module Woods
       known += ROUTE_CONSUMER_EXTRACTORS if known.include?(:routes)
       known.uniq!
 
-      prepare_incremental_run
+      prepare_incremental_run(operation: 'refresh')
       affected_types = Set.new
       touched = known.each_with_object(Set.new) do |key, acc|
         acc.merge(replace_type_wholesale(key, affected_types))
       end
 
+      raise_on_handled_extraction_failure!
       finalize_incremental_unit_json(affected_types)
-      affected_types.each { |type_key| regenerate_type_index(type_key) }
+      profile_phase('type index') { affected_types.each { |type_key| regenerate_type_index(type_key) } }
       finalize_incremental_run(touched, reason: "refresh:#{known.sort.join(',')}")
 
       { types: known, touched: touched.to_a, unknown: unknown }
+    ensure
+      log_profile_total('refresh', profile_started)
     end
 
     # Raise when the most recent extraction run wrote a payload but could not
@@ -667,6 +692,15 @@ module Woods
 
     private
 
+    # Whole-run wall time, including unprofiled setup and failed runs. This
+    # separate log family must never be added to the individual phase times.
+    def log_profile_total(name, started)
+      return unless started
+
+      elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
+      Rails.logger.info "[Woods] [profile total] #{name} in #{elapsed.round(2)}s"
+    end
+
     # Time one phase of a run and log how long it took, when WOODS_PROFILE=1.
     #
     # The per-extractor lines (see {#extract_all_sequential}) already report
@@ -745,7 +779,8 @@ module Woods
     #
     # @return [void]
     # @raise [Woods::ExtractionError] see {#begin_payload!}
-    def prepare_incremental_run
+    def prepare_incremental_run(operation: 'incremental')
+      profile_phase('source capture') { begin_source_inputs(operation) }
       profile_phase('payload seed') { begin_payload!(strict: true) }
       graph_path = payload_dir.join('dependency_graph.json')
       ensure_incremental_baseline!(graph_path)
@@ -795,7 +830,7 @@ module Woods
     #   reading "incremental" after a `woods:refresh[routes]` is being misled
     # @return [void]
     def finalize_incremental_run(touched, reason: 'incremental')
-      write_dependency_graph
+      profile_phase('graph write') { write_dependency_graph }
 
       if touched.empty?
         Rails.logger.info '[Woods] Incremental run changed nothing — leaving manifest timestamp untouched'
@@ -808,7 +843,7 @@ module Woods
         write_manifest(incremental: true)
         write_structural_summary
       end
-      profile_phase('publish') { publish_generation(reason) }
+      publish_generation(reason)
 
       return unless Woods.configuration.enable_snapshots
 
@@ -831,9 +866,10 @@ module Woods
       # Resolve (and if necessary rename) the payload first, so the flush
       # below covers the directory under the name the pointer will carry.
       payload = publishable_payload_name(generation)
+      profile_phase('source verification') { write_source_inputs } if payload
       profile_phase('payload sync') { sync_payload }
-      marker = generation.bump!(reason: reason, payload: payload)
-      prune_payloads(marker.number)
+      marker = profile_phase('publish') { generation.bump!(reason: reason, payload: payload) }
+      profile_phase('payload prune') { prune_payloads(marker.number) }
       marker
     rescue StandardError => e
       # A failed bump must not fail the extraction that produced a perfectly
@@ -852,6 +888,61 @@ module Woods
       nil
     end
 
+    # Capture before eager loading or extraction; only an explicit fresh-launch
+    # handoff can additionally establish the pre-Bundler/Rails boot boundary.
+    def begin_source_inputs(operation)
+      @failed_consumers = Set.new
+      @source_inputs = SourceInputs::Session.new(root: Rails.root, output_dir: @output_dir,
+                                                 baseline_path: source_input_baseline_path,
+                                                 operation: operation)
+    end
+
+    def source_input_baseline_path
+      generation = Generation.new(output_dir: @output_dir)
+      marker = generation.current
+      directory = generation.payload_dir(marker)
+      return nil unless marker.payload && directory != generation.root
+
+      directory.join(SourceInputs::Manifest::FILE_NAME)
+    rescue TypeError, NoMethodError
+      nil
+    end
+
+    def source_consumer_failed?(key, consumer = extractor_for(key))
+      failed = consumer.nil? || SourceInputs::ConsumerErrors.failed?(consumer)
+      @source_inputs&.unverified("extractor:#{key}") if failed
+      failed
+    end
+
+    # A rescued consumer error is not a successful empty result. Reset the
+    # per-call flag because one extractor instance serves several paths, but
+    # retain the failed scope for the run so a later success cannot authorize
+    # publication. Watch then carries the complete batch forward for retry.
+    def checked_extraction(key, consumer)
+      (@failed_consumers ||= Set.new).add(key) if SourceInputs::ConsumerErrors.failed?(consumer)
+      SourceInputs::ConsumerErrors.reset(consumer)
+      result = yield
+      return result unless source_consumer_failed?(key, consumer)
+
+      (@failed_consumers ||= Set.new).add(key)
+      nil
+    end
+
+    def raise_on_handled_extraction_failure!
+      return if @failed_consumers.nil? || @failed_consumers.empty?
+
+      raise Woods::ExtractionError,
+            "Extraction failed for #{@failed_consumers.to_a.sort.join(', ')}; " \
+            'the previous generation remains active. Fix the logged source errors and retry the complete batch.'
+    end
+
+    def write_source_inputs
+      return unless @source_inputs
+
+      manifest = @source_inputs.finish(generation: @payload_generation, eager_load_complete: @eager_load_complete)
+      AtomicFile.write(payload_dir.join(SourceInputs::Manifest::FILE_NAME), JSON.pretty_generate(manifest.data))
+    end
+
     # Open the payload directory this run publishes into, seeded from the
     # generation currently on disk.
     #
@@ -1268,9 +1359,6 @@ module Woods
 
     def setup_output_directory
       FileUtils.mkdir_p(@output_dir)
-      EXTRACTORS.each_key do |type|
-        FileUtils.mkdir_p(payload_dir.join(type.to_s))
-      end
     end
 
     # ──────────────────────────────────────────────────────────────────────
@@ -1370,8 +1458,10 @@ module Woods
     def same_type_collision_message(type, unit, prior_path)
       "same-type identifier collision: #{type.to_s.singularize} '#{unit.identifier}' derived from " \
         "two different sources ('#{prior_path || 'no file'}' and '#{unit.file_path || 'no file'}'); " \
-        'only one unit could ever be indexed, so extraction aborted — either merge the ' \
-        'declarations into one file or split them into distinct constants'
+        'only one unit could ever be indexed, so extraction aborted. ' \
+        'Wrapper-nested class naming requires Zeitwerk mode with Zeitwerk >= 2.6.9; on older loaders or ' \
+        'classic-mode hosts, check that support before changing valid namespace wrappers. ' \
+        'For a genuine duplicate, merge the declarations into one file or split them into distinct constants'
     end
 
     # ──────────────────────────────────────────────────────────────────────
@@ -1753,6 +1843,7 @@ module Woods
       GraphAnalyzer.new(
         @dependency_graph,
         volatile_ratio: ratio,
+        volatile_limit_per_target: config&.volatile_dependency_limit_per_target,
         cycle_limit: config ? config.graph_cycle_limit : GraphAnalyzer::DEFAULT_CYCLE_LIMIT,
         cycle_max_length: config ? config.graph_cycle_max_length : GraphAnalyzer::DEFAULT_CYCLE_MAX_LENGTH
       )
@@ -1869,11 +1960,9 @@ module Woods
     # Is this a path worth asking git about?
     #
     # A gem-owned unit (an engine model) carries its real path. Outside
-    # Rails.root, git refuses the whole `log` invocation when any pathspec is
-    # outside the repository — one gem path would erase the git metadata of
-    # the other 499 units in its 500-path batch. Inside Rails.root, a bundle
+    # Rails.root, it has no app repository history. Inside Rails.root, a bundle
     # vendored at `vendor/bundle` puts the same gem files under the root
-    # prefix, gitignored, so sending them is wasted pathspec work every run.
+    # prefix, gitignored, so requesting their history serves no app-owned unit.
     # Same exclusions as {Extractors::SharedUtilityMethods#app_source?}.
     #
     # @param path [String, nil] absolute file path
@@ -1927,13 +2016,13 @@ module Woods
     # exactly this value shape — no fractional seconds, `Z` or a `±hh:mm`
     # offset — and `spec/extracted_unit_spec.rb` pins that, so a change to the
     # stamp's shape fails a spec instead of quietly un-matching this mask.
-    # The value constraint is what keeps the mask honest against user code: a
-    # bare `"extracted_at":` cannot occur inside any JSON *string* value
-    # (interior quotes serialize as `\"`), so only a real JSON key can match,
-    # and only when it holds a timestamp — which no extractor emits below the
-    # top level.
+    # Match only the final top-level stamp, followed by the source_hash field
+    # and the document's closing brace. Nested metadata may use the same key
+    # and timestamp shape; changing it must still rewrite the unit. Escaped
+    # quotes inside string values cannot match these JSON field boundaries.
     EXTRACTED_AT_SCALAR =
-      /("extracted_at":\s*")\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:Z|[+-]\d{2}:\d{2})(?=")/
+      /("extracted_at":\s*")\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:Z|[+-]\d{2}:\d{2})
+       (?=",\s*"source_hash":\s*"[0-9a-f]{64}"\s*}\s*\z)/x
     # An implementation detail of the byte comparison, not part of the
     # extractor's surface (`private` does not scope constants).
     private_constant :EXTRACTED_AT_SCALAR
@@ -1967,7 +2056,7 @@ module Woods
     #   original encoding
     # @return [String] the bytes with the stamp's value removed
     def mask_extracted_at(bytes)
-      bytes.gsub(EXTRACTED_AT_SCALAR, '\1')
+      bytes.sub(EXTRACTED_AT_SCALAR, '\1')
     end
 
     def normalize_file_paths
@@ -1999,7 +2088,7 @@ module Woods
     # to say. Enrichment then wrote `commit_count: 0` and
     # `change_frequency: new` onto every unit, which reads exactly like a file
     # that was never committed, where an absent git directory correctly omits
-    # the keys (B-186). HEAD has to resolve.
+    # the keys (B-186). HEAD has to resolve, with complete ancestry (B-189).
     #
     # Memoized, so the warning below is emitted at most once per run.
     #
@@ -2008,13 +2097,31 @@ module Woods
       return @git_available if defined?(@git_available)
 
       _output, error, status = Open3.capture3(*git_argv('rev-parse', 'HEAD'))
-      @git_available = status.success?
-      warn_unresolvable_git(error) unless @git_available
-      @git_available
+      unless status.success?
+        warn_unresolvable_git(error)
+        return @git_available = false
+      end
+
+      @git_available = complete_git_history?
     rescue StandardError
       @git_available = false
     end
 
+    # A shallow HEAD resolves but represents an incomplete ancestry. Do not
+    # turn that boundary into apparent one-commit/new-file churn facts.
+    def complete_git_history?
+      output, _error, status = Open3.capture3(*git_argv('rev-parse', '--is-shallow-repository'))
+      return true if status.success? && output.strip == 'false'
+
+      shallow = status.success? && output.strip == 'true'
+      reason = shallow ? 'shallow repository' : 'repository depth could not be verified'
+      Rails.logger.warn(
+        "[Woods] Git enrichment omitted: #{reason}. Fetch full history with git fetch --unshallow " \
+        '(or actions/checkout fetch-depth: 0), then run full extraction to refresh git metadata.'
+      )
+      false
+    end
+
     # Say once why no unit will carry git metadata, but only when there is a
     # working tree to explain. No `.git` at the root is the ordinary source
     # tarball or `COPY`-without-`.git` case, and it is not a fault.
@@ -2058,63 +2165,19 @@ module Woods
       ''
     end
 
-    # Batch-fetch git data for all file paths in two git commands.
-    #
-    # Duplicate paths are collapsed before slicing (audit P9d): many units
-    # share one file_path, and duplicates only repeat a pathspec another
-    # batch also sent. The result is keyed by relative path, so the output
-    # is identical.
-    #
-    # @param file_paths [Array<String>] Absolute file paths
-    # @return [Hash{String => Hash}] Keyed by relative path
+    # One HEAD history walk, independent of requested-path grouping. See
+    # GitHistory for explicit merge semantics and binary record framing.
+    # @param file_paths [Array<String>] absolute file paths
+    # @return [Hash{String => Hash}] keyed by Rails.root-relative path
     def batch_git_data(file_paths)
       return {} if file_paths.empty?
 
-      root = "#{Rails.root}/"
-      relative_paths = file_paths.map { |f| f.sub(root, '') }.uniq
-      result = {}
-      relative_paths.each { |rp| result[rp] = {} }
-
-      path_set = relative_paths.to_set
-      relative_paths.each_slice(500) do |batch|
-        log_output = run_git(
-          'log', '--all', '--name-only',
-          '--format=__COMMIT__%H|||%an|||%cI|||%s',
-          '--since=365 days ago',
-          '--', *batch
-        )
-        parse_git_log_output(log_output, path_set, result)
-      end
-
-      ninety_days_ago = (Time.current - 90.days).iso8601
-      result.each do |relative_path, data|
-        result[relative_path] = build_file_metadata(data, ninety_days_ago)
-      end
-
-      result
-    end
-
-    # Parse git log output line-by-line, populating result with per-file commit data.
-    def parse_git_log_output(log_output, path_set, result)
-      current_commit = nil
-
-      log_output.each_line do |line|
-        line = line.strip
-        next if line.empty?
+      relative_paths = file_paths.map { |path| normalize_file_path(path) }.uniq
+      recent_after = Time.current - 90.days
+      raw = GitHistory.new(root: Rails.root, logger: Rails.logger).read(relative_paths, recent_after: recent_after)
+      return {} unless raw
 
-        if line.start_with?('__COMMIT__')
-          parts = line.sub('__COMMIT__', '').split('|||', 4)
-          current_commit = { sha: parts[0], author: parts[1], date: parts[2], message: parts[3] }
-        elsif current_commit && path_set.include?(line)
-          entry = result[line] ||= {}
-          unless entry[:last_modified]
-            entry[:last_modified] = current_commit[:date]
-            entry[:last_author] = current_commit[:author]
-          end
-          (entry[:commits] ||= []) << current_commit
-          (entry[:contributors] ||= Hash.new(0))[current_commit[:author]] += 1
-        end
-      end
+      raw.transform_values { |data| build_file_metadata(data, recent_after.iso8601) }
     end
 
     # Classify how frequently a file changes based on commit counts.
@@ -2136,12 +2199,13 @@ module Woods
     def build_file_metadata(data, ninety_days_ago)
       all_commits = data[:commits] || []
       contributor_counts = data[:contributors] || {}
-      recent_count = all_commits.count { |c| c[:date] && c[:date] > ninety_days_ago }
+      recent_count = data.fetch(:recent_count) { all_commits.count { |c| c[:date] && c[:date] > ninety_days_ago } }
+      total_count = data.fetch(:commit_count, all_commits.size)
 
       {
         last_modified: data[:last_modified],
         last_author: data[:last_author],
-        commit_count: all_commits.size,
+        commit_count: total_count,
         contributors: contributor_counts
                       .sort_by { |_, count| -count }
                       .first(5)
@@ -2149,7 +2213,7 @@ module Woods
         recent_commits: all_commits.first(5).map do |c|
           { sha: c[:sha]&.first(8), message: c[:message], date: c[:date], author: c[:author] }
         end,
-        change_frequency: classify_change_frequency(all_commits.size, recent_count)
+        change_frequency: classify_change_frequency(total_count, recent_count)
       }
     end
 
@@ -2324,6 +2388,7 @@ module Woods
 
       manifest = {
         extracted_at: Time.current.iso8601,
+        woods_version: Woods::VERSION,
         rails_version: Rails.version,
         ruby_version: RUBY_VERSION,
 
@@ -2697,6 +2762,7 @@ module Woods
 
       @incremental_extractors[key] = EXTRACTORS[key]&.new
     rescue StandardError => e
+      @source_inputs&.unverified("extractor:#{key}")
       Rails.logger.warn "[Woods] Could not build #{key} extractor: #{e.message}"
       @incremental_extractors[key] = nil
     end
@@ -2719,10 +2785,11 @@ module Woods
     #
     # This is the fix for #164 gap 1 (a path the index has never seen routed
     # nowhere, so new files were silently ignored) and the per-path half of
-    # gap 3 (a file that defines several units — a `.rake` file with multiple
-    # tasks, an i18n YAML — could only ever resolve to one identifier).
-    # Reconciling the whole path at once means a task deleted from a
-    # multi-task file is removed rather than left behind.
+    # gap 3 (a file defining several units could only ever resolve to one
+    # identifier). Rake tasks need the wholesale path because definitions
+    # of one task can span several files.
+    # Reconciling the whole path removes definitions deleted from a surviving
+    # file rather than leaving them behind.
     #
     # Removal is scoped to the unit types the matching rules could have
     # produced, so a class-based unit sharing the path (the `User` model unit
@@ -2758,6 +2825,10 @@ module Woods
           # one method over — CORE-1).
           produced.merge(units.map { |unit| [unit.identifier, unit.type] })
           touched.merge(register_and_write(rule.extractor_key, units, affected_types))
+          unless source_consumer_failed?(rule.extractor_key)
+            @source_inputs&.consume_file(rule.extractor_key,
+                                         absolute_path)
+          end
         end
 
         next if raised
@@ -2786,10 +2857,13 @@ module Woods
       # on every changed path of that type, with the generation bumped over
       # the loss. Construction failure tells us nothing about the path; only
       # a genuinely constructed extractor that lacks the method earns the [].
-      return nil if extractor.nil?
+      if extractor.nil?
+        source_consumer_failed?(rule.extractor_key, extractor)
+        return nil
+      end
       return [] unless extractor.respond_to?(rule.method_name)
 
-      result =
+      result = checked_extraction(rule.extractor_key, extractor) do
         if rule.extractor_key == :poros
           # PoroExtractor needs the AR name set to reject persisted models;
           # its default is an empty set, which would misfile every model
@@ -2798,9 +2872,12 @@ module Woods
         else
           extractor.public_send(rule.method_name, absolute_path)
         end
+      end
+      return nil if result.nil? && SourceInputs::ConsumerErrors.failed?(extractor)
 
       Array(result).compact
     rescue StandardError => e
+      @source_inputs&.unverified("extractor:#{rule.extractor_key}")
       Rails.logger.warn "[Woods] #{rule.extractor_key} re-extraction of #{absolute_path} failed: #{e.message}"
       # `nil`, not `[]`. The caller treats an empty result as "this path defines
       # nothing any more" and prunes the units previously registered to it — so
@@ -2871,6 +2948,34 @@ module Woods
       touched
     end
 
+    # Runtime-only model mixins can enter or leave discovery when their
+    # includer changes, even if the mixin file itself is untouched.
+    # @param affected_types [Set<Symbol>]
+    # @return [Set<String>] Added or removed concern identifiers
+    def reconcile_model_mixins(affected_types)
+      extractor = extractor_for(:concerns)
+      return Set.new unless extractor.respond_to?(:runtime_model_mixins)
+
+      live = extractor.runtime_model_mixins
+      known = @dependency_graph.units_of_type(:concern).to_set
+      added = live.flat_map do |path, modules|
+        next [] if modules.all? { |mod| known.include?(mod.name) }
+
+        Array(extractor.extract_model_mixin_file(path)).reject { |unit| known.include?(unit.identifier) }
+      end
+      touched = register_and_write(:concerns, added, affected_types)
+      return touched unless @eager_load_complete
+
+      live_names = live.values.flatten.to_set(&:name)
+      known.each do |identifier|
+        path = @dependency_graph.node(identifier, type: :concern)[:file_path]
+        next if extractor.conventional_concern_path?(path) || live_names.include?(identifier)
+
+        touched.add(identifier) if remove_unit(identifier, affected_types, type: :concern)
+      end
+      touched
+    end
+
     # Pruned class-based identifiers the tree still governs, and that the
     # second reconciliation pass may therefore re-add.
     #
@@ -2948,10 +3053,12 @@ module Woods
       units = new_classes.filter_map do |klass|
         extractor_for(key).public_send(spec[:method], klass)
       rescue StandardError => e
+        @source_inputs&.unverified("extractor:#{key}")
         Rails.logger.warn "[Woods] #{key} extraction of #{klass} failed: #{e.message}"
         nil
       end
 
+      source_consumer_failed?(key)
       register_and_write(key, units, affected_types)
     end
 
@@ -3086,14 +3193,23 @@ module Woods
     #   mutating durable state
     def replace_type_wholesale(key, affected_types)
       extractor = extractor_for(key)
+      if extractor.nil?
+        source_consumer_failed?(key, extractor)
+        return Set.new
+      end
       return Set.new unless extractor.respond_to?(:extract_all)
 
       @wholesale_mutations = 0
-      units = Array(extractor.extract_all).compact.uniq(&:identifier)
+      result = checked_extraction(key, extractor) { extractor.extract_all }
+      return Set.new if SourceInputs::ConsumerErrors.failed?(extractor)
+
+      units = Array(result).compact.uniq(&:identifier)
       Rails.logger.info "[Woods] Re-ran #{key} wholesale: #{units.size} units"
 
       touched = register_and_write(key, units, affected_types)
       touched.merge(remove_replaced_units(key, units, affected_types))
+      @source_inputs&.consume_extractor(key, units) unless source_consumer_failed?(key, extractor)
+      touched
     rescue StandardError => e
       if @wholesale_mutations.to_i.positive?
         raise Woods::ExtractionError, <<~MSG.tr("\n", ' ').strip
@@ -3106,6 +3222,7 @@ module Woods
         MSG
       end
 
+      @source_inputs&.unverified("extractor:#{key}")
       Rails.logger.error "[Woods] Wholesale re-run of #{key} failed: #{e.message}"
       Set.new
     end
@@ -3267,6 +3384,7 @@ module Woods
 
           removed.add(identifier) if remove_unit(identifier, affected_types, type: type)
         end
+        @source_inputs&.consume_deleted(path)
       end
     end
 
@@ -3370,6 +3488,7 @@ module Woods
         (@incremental_written ||= {})[unit.identifier] = unit.file_path
 
         write_unit_file(type_dir.join(collision_safe_filename(unit.identifier)), unit)
+        @source_inputs&.consume_unit(extractor_key, unit.file_path) unless source_consumer_failed?(extractor_key)
         written.add(unit.identifier)
       end
     end
@@ -3446,12 +3565,14 @@ module Woods
     def finalize_incremental_unit_json(affected_types)
       dependents_dirty = @dependents_dirty || Set.new
       git_dirty = @incremental_written || {}
-      git_data = incremental_git_data(git_dirty.keys)
+      git_data = profile_phase('git enrichment') { incremental_git_data(git_dirty.keys) }
 
-      (dependents_dirty | git_dirty.keys).each do |identifier|
-        rewrite_unit_json(identifier, affected_types,
-                          refresh_dependents: dependents_dirty.include?(identifier),
-                          git_data: git_dirty.key?(identifier) ? git_data : nil)
+      profile_phase('unit finalization') do
+        (dependents_dirty | git_dirty.keys).each do |identifier|
+          rewrite_unit_json(identifier, affected_types,
+                            refresh_dependents: dependents_dirty.include?(identifier),
+                            git_data: git_dirty.key?(identifier) ? git_data : nil)
+        end
       end
     end
 
@@ -3560,7 +3681,7 @@ module Woods
     end
 
     # Batch-fetch git metadata for the units written by this run, in a single
-    # git invocation, keyed by Rails.root-relative path the way
+    # history walk, keyed by Rails.root-relative path the way
     # {#batch_git_data} returns it.
     #
     # @param identifiers [Array<String>]
@@ -3568,11 +3689,12 @@ module Woods
     def incremental_git_data(identifiers)
       return {} if identifiers.empty? || !git_available?
 
+      root = "#{Rails.root}/"
       paths = identifiers.flat_map do |identifier|
         @dependency_graph.nodes_for(identifier).filter_map do |node|
           next if %i[rails_source gem_source].include?(node[:type])
 
-          node[:file_path] if node[:file_path] && File.exist?(node[:file_path])
+          node[:file_path] if git_enrichable_path?(node[:file_path], root)
         end
       end
 
@@ -3627,11 +3749,17 @@ module Woods
       return nil unless extractor_key
 
       extractor = extractor_for(extractor_key)
-      return nil unless extractor
+      if extractor.nil?
+        source_consumer_failed?(extractor_key, extractor)
+        return nil
+      end
 
-      # File-based extractors can return several units from one file (a .rake
-      # file defining multiple tasks, etc.); class-based extractors return one.
-      units = Array(re_extracted_units(extractor, type, unit_id, file_path, extractor_key)).compact
+      # File-based extractors can return several units from one file;
+      # class-based extractors return one.
+      result = checked_extraction(extractor_key, extractor) do
+        re_extracted_units(extractor, type, unit_id, file_path, extractor_key)
+      end
+      units = Array(result).compact
       return nil if units.empty?
 
       register_and_write(extractor_key, units, affected_types)