woods 1.5.0 → 1.6.0

data/lib/woods/coordination/pipeline_lock.rb

@@ -2,6 +2,7 @@
 
 require 'fileutils'
 require 'json'
+require 'securerandom'
 
 module Woods
   module Coordination
@@ -38,12 +39,13 @@ module Woods
       def acquire
         FileUtils.mkdir_p(@lock_dir)
 
-        # Check for stale lock first (separate from atomic creation)
         if File.exist?(@lock_path)
           return false unless stale?
-
-          # Remove stale lock
-          FileUtils.rm_f(@lock_path)
+          # Retire the stale lock atomically. A bare rm_f + create here is
+          # a TOCTOU race: two processes passing the stale check together
+          # could each delete-and-create, the second deleting the first's
+          # FRESH lock — both would then "hold" it.
+          return false unless retire_stale_lock
         end
 
         # Atomic lock creation: File::EXCL ensures this fails if file already exists
@@ -58,10 +60,37 @@ module Woods
 
       # Release the lock.
       #
+      # Deletes the lock file only if it still carries this instance's
+      # token — a run that outlived stale_timeout may have been
+      # legitimately taken over, and deleting unconditionally would drop
+      # the new holder's lock.
+      #
       # @return [void]
       def release
-        FileUtils.rm_f(@lock_path) if @held
+        return unless @held
+
+        # Clear @held up front so no later failure can leave this instance
+        # believing it still holds the lock.
         @held = false
+
+        # Rename first, then inspect: a plain read-then-unlink is a TOCTOU —
+        # after we read our own token a takeover could replace the file, and
+        # our unlink would then delete the NEW holder's lock. Renaming
+        # atomically captures whatever is at the path.
+        graveyard = "#{@lock_path}.release.#{Process.pid}.#{SecureRandom.hex(4)}"
+        begin
+          File.rename(@lock_path, graveyard)
+        rescue Errno::ENOENT
+          return # already gone
+        end
+
+        if own_lock?(graveyard)
+          FileUtils.rm_f(graveyard)
+        else
+          # We were legitimately taken over — put the successor's lock back
+          # without clobbering a still-newer holder (see {#restore_lock}).
+          restore_lock(graveyard)
+        end
       end
 
       # Execute a block while holding the lock.
@@ -100,9 +129,85 @@ module Woods
         true
       end
 
-      # @return [String] Lock file content (JSON with PID and timestamp)
+      # Atomically retire a stale lock file via rename. Rename is atomic on
+      # POSIX: of any processes racing to take over the same stale lock,
+      # exactly one rename succeeds; the losers get ENOENT and back off.
+      # Winning the rename does NOT guarantee winning the lock — another
+      # process may O_EXCL-create between our rename and our create, which
+      # the caller's EEXIST rescue handles.
+      #
+      # Rename alone is not enough, though: a competitor that already passed
+      # `stale?` on the SAME original file may have retired it and created a
+      # FRESH lock before we run. Our rename would then move that fresh lock
+      # aside — and both processes would "hold" the lock. So after winning
+      # the rename we re-check the retired file's age; if it turns out to be
+      # fresh (someone beat us to the takeover), we put it back and lose the
+      # race instead of clobbering a live holder.
+      #
+      # @return [Boolean] true if this process retired a genuinely stale lock
+      def retire_stale_lock
+        graveyard = "#{@lock_path}.stale.#{Process.pid}.#{SecureRandom.hex(4)}"
+        File.rename(@lock_path, graveyard)
+
+        unless stale_file?(graveyard)
+          # We grabbed a lock that is no longer stale — a competitor already
+          # took over. Restore it (without clobbering a still-newer holder)
+          # and back off.
+          restore_lock(graveyard)
+          return false
+        end
+
+        FileUtils.rm_f(graveyard)
+        true
+      rescue Errno::ENOENT
+        false
+      end
+
+      # Whether the lock file at +path+ carries this instance's token.
+      #
+      # @param path [String]
+      # @return [Boolean] true when the token matches, or the file is corrupt
+      #   (an unparseable lock we already renamed aside is treated as ours to
+      #   discard rather than restore).
+      def own_lock?(path)
+        JSON.parse(File.read(path))['token'] == @token
+      rescue JSON::ParserError
+        true
+      end
+
+      # Put a lock file we renamed aside back at @lock_path WITHOUT clobbering
+      # a lock another process may have O_EXCL-created in the meantime.
+      # `File.link` is atomic and fails with EEXIST if @lock_path already
+      # exists, so a newer holder always wins; the aside copy is discarded
+      # either way. A plain `File.rename` back would overwrite that newer
+      # holder's lock — reintroducing a double-hold.
+      #
+      # @param graveyard [String] path of the renamed-aside lock file
+      # @return [void]
+      def restore_lock(graveyard)
+        File.link(graveyard, @lock_path)
+      rescue Errno::EEXIST
+        # A newer holder already claimed the path — our copy is obsolete.
+        nil
+      ensure
+        FileUtils.rm_f(graveyard)
+      end
+
+      # Whether the file at +path+ is older than the stale timeout.
+      #
+      # @param path [String]
+      # @return [Boolean]
+      def stale_file?(path)
+        Time.now - File.mtime(path) > @stale_timeout
+      rescue Errno::ENOENT
+        false
+      end
+
+      # @return [String] Lock file content (JSON with PID, timestamp, and
+      #   an ownership token release verifies before deleting)
       def lock_content
-        JSON.generate(pid: Process.pid, locked_at: Time.now.iso8601, name: @name)
+        @token = SecureRandom.hex(8)
+        JSON.generate(pid: Process.pid, locked_at: Time.now.iso8601, name: @name, token: @token)
       end
     end
   end

data/lib/woods/dependency_graph.rb

@@ -31,12 +31,20 @@ module Woods
       @to_h = nil
     end
 
-    # Register a unit in the graph
+    # Register a unit in the graph.
+    #
+    # Re-registering an identifier (incremental extraction registers into a
+    # graph loaded from disk) first removes the previous registration's
+    # reverse edges, file-map entry, and type-index entry — otherwise stale
+    # dependents accumulate across incremental runs and get persisted back
+    # to dependency_graph.json.
     #
     # @param unit [ExtractedUnit] The unit to register
     def register(unit)
       @to_h = nil
 
+      unregister(unit.identifier) if @nodes.key?(unit.identifier)
+
       @nodes[unit.identifier] = {
         type: unit.type,
         file_path: unit.file_path,
@@ -56,6 +64,35 @@ module Woods
       end
     end
 
+    # Remove an identifier's registration side effects: its contribution to
+    # the reverse indexes (derived from its recorded forward edges), its
+    # file-map entry, and its type-index entry. Forward node/edge data is
+    # overwritten by the caller (register), so it is not cleared here.
+    #
+    # @param identifier [String] Previously-registered unit identifier
+    # @return [void]
+    def unregister(identifier)
+      (@edges[identifier] || []).each do |edge|
+        if (set = @reverse[edge[:target]])
+          set.delete(identifier)
+          @reverse.delete(edge[:target]) if set.empty?
+        end
+
+        via_key = [edge[:target], edge[:via]]
+        next unless (set = @reverse_via[via_key])
+
+        set.delete(identifier)
+        @reverse_via.delete(via_key) if set.empty?
+      end
+
+      old_node = @nodes[identifier]
+      return unless old_node
+
+      old_path = old_node[:file_path]
+      @file_map.delete(old_path) if old_path && @file_map[old_path] == identifier
+      @type_index[old_node[:type]]&.delete(identifier)
+    end
+
     # Find all units affected by changes to given files
     # Uses BFS to find transitive dependents
     #

data/lib/woods/embedding/text_preparer.rb

@@ -95,7 +95,12 @@ module Woods
       def append_dependency_line(lines, dependencies)
         return unless dependencies&.any?
 
-        dep_names = dependencies.map { |d| d[:target] }.compact.first(10)
+        # Dependency hashes arrive symbol-keyed from the extractor's
+        # in-memory units but string-keyed from the indexer (Indexer#build_unit
+        # reads JSON and does not symbolize dependency keys, unlike chunks).
+        # Read both forms or the whole "dependencies:" prefix silently
+        # vanishes from every embedded document on the indexing path.
+        dep_names = dependencies.filter_map { |d| d[:target] || d['target'] }.first(10)
         lines << "dependencies: #{dep_names.join(', ')}" if dep_names.any?
       end
 

data/lib/woods/export/unit_facts.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Woods
+  # Shared building blocks for the export formatters (Unblocked DocumentBuilder,
+  # Obsidian NoteBuilder).
+  module Export
+    # Format-agnostic extraction of facts from a unit's metadata.
+    #
+    # Both the Unblocked and Obsidian exporters need the same *facts* about a
+    # unit — its associations grouped by macro, its schema highlights — but
+    # render them differently (plain text + GitHub URIs vs. wikilinks +
+    # frontmatter). This value object owns the one definition of "what a unit's
+    # metadata says"; each exporter is a thin formatter over it. The returned
+    # structures are deliberately raw (no truncation, no ordering imposed beyond
+    # the source array): formatting decisions — sort order, top-N caps — belong
+    # to each formatter so neither exporter's output is constrained by the other.
+    class UnitFacts
+      # @param unit [Hash] parsed unit JSON
+      def initialize(unit)
+        @meta = (unit && unit['metadata']) || {}
+      end
+
+      # Associations grouped by macro, each as a structured entry. Group keys are
+      # whatever macros are present (insertion order from the source array); the
+      # formatter picks the display order.
+      #
+      # @return [Hash{String=>Array<Hash>}] macro => [{ target:, dependent: }]
+      def associations_by_type
+        associations.group_by { |assoc| assoc['type'] }.transform_values do |items|
+          items.map { |a| { target: a['target'] || a['name'], dependent: a.dig('options', 'dependent') } }
+        end
+      end
+
+      # @return [Integer] total association count across all macros
+      def association_count
+        associations.size
+      end
+
+      # @return [Hash] { enums:, scopes:, concerns:, callbacks: } — raw, untruncated
+      def schema_highlights
+        { enums: enums, scopes: scopes, concerns: concerns, callbacks: callbacks }
+      end
+
+      def enums
+        @meta['enums'].is_a?(Hash) ? @meta['enums'] : {}
+      end
+
+      def scopes
+        @meta['scopes'].is_a?(Array) ? @meta['scopes'].filter_map { |s| s['name'] } : []
+      end
+
+      def concerns
+        @meta['inlined_concerns'].is_a?(Array) ? @meta['inlined_concerns'] : []
+      end
+
+      def callbacks
+        return [] unless @meta['callbacks'].is_a?(Array)
+
+        @meta['callbacks'].map { |cb| { type: cb['type'], filter: cb['filter'] } }
+      end
+
+      # Blast-radius bucket for a dependent count.
+      #
+      # @param dependent_count [Integer]
+      # @return [Symbol] :high (>50), :moderate (>20), or :none
+      def self.blast_radius(dependent_count)
+        return :high if dependent_count > 50
+        return :moderate if dependent_count > 20
+
+        :none
+      end
+
+      private
+
+      def associations
+        @meta['associations'].is_a?(Array) ? @meta['associations'] : []
+      end
+    end
+  end
+end

data/lib/woods/extractor.rb

@@ -8,6 +8,7 @@ require 'pathname'
 require 'set'
 
 require_relative 'filename_utils'
+require_relative 'token_utils'
 require_relative 'extracted_unit'
 require_relative 'dependency_graph'
 require_relative 'git_provenance'
@@ -249,12 +250,6 @@ module Woods
       Rails.logger.info '[Woods] Analyzing dependency graph...'
       @graph_analysis = GraphAnalyzer.new(@dependency_graph).analyze
 
-      # Phase 3.5: Precompute request flows (opt-in)
-      if Woods.configuration.precompute_flows
-        Rails.logger.info '[Woods] Precomputing request flows...'
-        precompute_flows
-      end
-
       # Phase 4: Enrich with git data
       Rails.logger.info '[Woods] Enriching with git data...'
       enrich_with_git_data
@@ -266,6 +261,17 @@ module Woods
       # Phase 5: Write output
       Rails.logger.info '[Woods] Writing output...'
       write_results
+
+      # Phase 5.5: Precompute request flows (opt-in). Must run AFTER
+      # write_results — FlowAssembler loads unit JSON from disk, so running
+      # earlier assembled every flow from absent (fresh output dir) or
+      # stale (previous run's) data. precompute_flows re-writes the
+      # controller units it annotates with metadata[:flow_paths].
+      if Woods.configuration.precompute_flows
+        Rails.logger.info '[Woods] Precomputing request flows...'
+        precompute_flows
+      end
+
       write_dependency_graph
       write_graph_analysis
       write_manifest
@@ -316,11 +322,17 @@ module Woods
         regenerate_type_index(type_key)
       end
 
-      # Update graph, manifest, and summary
+      # Update graph, manifest, and summary. No capture_snapshot here:
+      # snapshots must hash the FULL unit set, and incremental runs only
+      # re-extract affected units (@results stays empty) — capturing would
+      # record a snapshot whose diff reports every unit as deleted.
+      # Snapshots are captured on full extraction only.
       write_dependency_graph
-      write_manifest
+      write_manifest(incremental: true)
       write_structural_summary
-      capture_snapshot
+      if Woods.configuration.enable_snapshots
+        Rails.logger.info '[Woods] Skipping snapshot capture — snapshots are captured on full extraction only'
+      end
 
       affected_ids
     end
@@ -506,11 +518,42 @@ module Woods
       all_units = @results.values.flatten(1)
       precomputer = FlowPrecomputer.new(units: all_units, graph: @dependency_graph, output_dir: @output_dir.to_s)
       flow_map = precomputer.precompute
+      rewrite_flow_annotated_units
       Rails.logger.info "[Woods] Precomputed #{flow_map.size} request flows"
     rescue StandardError => e
       Rails.logger.error "[Woods] Flow precomputation failed: #{e.message}"
     end
 
+    # Precompute runs after write_results (FlowAssembler reads unit JSON
+    # from disk), so units annotated in memory with metadata[:flow_paths]
+    # must be re-written and their type index refreshed to pick up the
+    # annotation.
+    #
+    # The index is rebuilt from the in-memory `units` (the authoritative
+    # full-extraction `@results`), NOT from a disk glob: this is a full
+    # extraction, the output dir is never wiped, and globbing would
+    # resurrect stale unit files for app classes deleted since the last run.
+    # (The incremental path, which only holds changed units in memory, still
+    # rebuilds from disk via {#regenerate_type_index}.)
+    def rewrite_flow_annotated_units
+      @results.each do |type, units|
+        annotated = units.select { |u| u.metadata[:flow_paths] }
+        next if annotated.empty?
+
+        type_dir = @output_dir.join(type.to_s)
+        annotated.each do |unit|
+          File.write(
+            type_dir.join(collision_safe_filename(unit.identifier)),
+            json_serialize(unit.to_h)
+          )
+        end
+        File.write(
+          type_dir.join('_index.json'),
+          json_serialize(type_index_entries(units))
+        )
+      end
+    end
+
     # ──────────────────────────────────────────────────────────────────────
     # Git Enrichment
     # ──────────────────────────────────────────────────────────────────────
@@ -702,23 +745,32 @@ module Woods
         end
 
         # Also write a type index for fast lookups
-        index = units.map do |u|
-          {
-            identifier: u.identifier,
-            file_path: u.file_path,
-            namespace: u.namespace,
-            estimated_tokens: u.estimated_tokens,
-            chunk_count: u.chunks.size
-          }
-        end
-
         File.write(
           type_dir.join('_index.json'),
-          json_serialize(index)
+          json_serialize(type_index_entries(units))
         )
       end
     end
 
+    # Build the `_index.json` entry list for a set of in-memory units.
+    # Shared by {#write_results} and {#rewrite_flow_annotated_units} so both
+    # emit the index from the authoritative in-memory `@results` rather than
+    # re-deriving it from disk.
+    #
+    # @param units [Array<ExtractedUnit>]
+    # @return [Array<Hash>]
+    def type_index_entries(units)
+      units.map do |u|
+        {
+          identifier: u.identifier,
+          file_path: u.file_path,
+          namespace: u.namespace,
+          estimated_tokens: u.estimated_tokens,
+          chunk_count: u.chunks.size
+        }
+      end
+    end
+
     def write_dependency_graph
       graph_data = @dependency_graph.to_h
       graph_data[:pagerank] = @dependency_graph.pagerank
@@ -745,24 +797,35 @@ module Woods
       )
     end
 
-    def write_manifest
+    def write_manifest(incremental: false)
       # Worktree-aware git provenance. In a linked worktree +.git+ is a file
       # pointing at the real git dir; when that dir is unreachable (e.g. an
       # unmounted host path inside a container) this resolves to "unknown"
       # rather than a stale GIT_BRANCH/GIT_SHA build arg. See GitProvenance (#137).
       provenance = GitProvenance.new(root: Rails.root).to_h
 
+      # Incremental runs never populate @results — deriving counts from
+      # memory would clobber a good manifest with zeros. Recompute from the
+      # persisted per-type _index.json files instead.
+      counts, total_chunks =
+        if incremental
+          persisted_counts
+        else
+          [@results.transform_values(&:size),
+           @results.sum { |_, units| units.sum { |u| u.chunks.size } }]
+        end
+
       manifest = {
         extracted_at: Time.current.iso8601,
         rails_version: Rails.version,
         ruby_version: RUBY_VERSION,
 
         # Counts by type
-        counts: @results.transform_values(&:size),
+        counts: counts,
 
         # Total stats
-        total_units: @results.values.sum(&:size),
-        total_chunks: @results.sum { |_, units| units.sum { |u| u.chunks.size } },
+        total_units: counts.values.sum,
+        total_chunks: total_chunks,
 
         # Git provenance (branch/sha), or "unknown" when unresolvable
         git_sha: provenance[:git_sha],
@@ -779,6 +842,30 @@ module Woods
       )
     end
 
+    # Unit and chunk counts derived from the per-type _index.json files on
+    # disk — the source of truth after an incremental run, where only the
+    # affected units were re-extracted.
+    #
+    # @return [Array(Hash{Symbol => Integer}, Integer)] counts by type, total chunk count
+    def persisted_counts
+      counts = {}
+      chunks = 0
+
+      Dir[@output_dir.join('*/_index.json').to_s].each do |index_path|
+        entries = JSON.parse(File.read(index_path))
+        counts[File.basename(File.dirname(index_path)).to_sym] = entries.size
+        chunks += entries.sum { |e| e['chunk_count'].to_i }
+      rescue JSON::ParserError => e
+        # An unreadable index silently drops that whole type from the manifest
+        # counts — warn rather than undercount without a trace.
+        type = File.basename(File.dirname(index_path))
+        Rails.logger.warn("[Woods] Skipping unreadable #{type}/_index.json in manifest counts: #{e.message}")
+        next
+      end
+
+      [counts, chunks]
+    end
+
     # Capture a temporal snapshot after extraction completes.
     #
     # Reads the manifest and computes per-unit content hashes, then delegates
@@ -915,7 +1002,10 @@ module Woods
           identifier: data['identifier'],
           file_path: data['file_path'],
           namespace: data['namespace'],
-          estimated_tokens: data['estimated_tokens'],
+          # Unit JSON has no estimated_tokens field (ExtractedUnit#to_h
+          # doesn't emit one) — recompute it, or every unit of a type
+          # touched by an incremental run would index as null.
+          estimated_tokens: estimated_tokens_from(data),
           chunk_count: (data['chunks'] || []).size
         }
       end
@@ -926,6 +1016,20 @@ module Woods
       )
     end
 
+    # Token estimate for a unit parsed back from JSON, mirroring
+    # ExtractedUnit#estimated_tokens (see docs/TOKEN_BENCHMARK.md).
+    #
+    # @param data [Hash] Parsed unit JSON (string keys)
+    # @return [Integer]
+    def estimated_tokens_from(data)
+      source = data['source_code']
+      metadata = data['metadata'] || {}
+
+      source_tokens = source ? TokenUtils.estimate_tokens(source) : 0
+      metadata_tokens = metadata.any? ? TokenUtils.estimate_tokens(metadata.to_json) : 0
+      source_tokens + metadata_tokens
+    end
+
     # ──────────────────────────────────────────────────────────────────────
     # Helpers
     # ──────────────────────────────────────────────────────────────────────
@@ -1023,12 +1127,20 @@ module Woods
 
       return unless unit
 
-      # Update dependency graph
+      # Update dependency graph. Register BEFORE normalizing the path —
+      # the graph's file_map stores absolute paths (affected_by matches
+      # changed files against them), exactly as full extraction registers
+      # in Phase 1 and only normalizes in Phase 4.5.
       @dependency_graph.register(unit)
 
       # Track which type was affected
       affected_types&.add(extractor_key)
 
+      # Unit JSON carries Rails.root-relative paths (full extraction's
+      # Phase 4.5); writing the raw absolute source_location here would
+      # leak container-absolute paths into the index after incremental runs.
+      unit.file_path = normalize_file_path(unit.file_path)
+
       # Write updated unit
       type_dir = @output_dir.join(extractor_key.to_s)
 

data/lib/woods/extractors/concern_extractor.rb

@@ -48,10 +48,8 @@ module Woods
       #
       # @return [Array<ExtractedUnit>] List of concern units
       def extract_all
-        @directories.flat_map do |dir|
-          Dir[dir.join('**/*.rb')].filter_map do |file|
-            extract_concern_file(file)
-          end
+        find_files_in_directories(@directories).filter_map do |file|
+          extract_concern_file(file)
         end
       end
 
@@ -285,7 +283,7 @@ module Woods
         deps.concat(scan_service_dependencies(source))
         deps.concat(scan_job_dependencies(source))
 
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
     end
   end

data/lib/woods/extractors/configuration_extractor.rb

@@ -36,10 +36,8 @@ module Woods
       #
       # @return [Array<ExtractedUnit>] List of configuration units
       def extract_all
-        units = @directories.flat_map do |dir|
-          Dir[dir.join('**/*.rb')].filter_map do |file|
-            extract_configuration_file(file)
-          end
+        units = find_files_in_directories(@directories).filter_map do |file|
+          extract_configuration_file(file)
         end
 
         profile = BehavioralProfile.new.extract
@@ -212,7 +210,7 @@ module Woods
 
         deps.concat(scan_service_dependencies(source))
 
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
     end
   end

data/lib/woods/extractors/controller_extractor.rb

@@ -320,7 +320,7 @@ module Woods
           deps.concat(scan_navigation_dependencies(source, via_type: :redirect_to))
         end
 
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
 
       # ──────────────────────────────────────────────────────────────────────

data/lib/woods/extractors/database_view_extractor.rb

@@ -271,7 +271,7 @@ module Woods
           deps << { type: :model, target: model_name, via: :table_name }
         end
 
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
     end
   end

data/lib/woods/extractors/decorator_extractor.rb

@@ -49,10 +49,8 @@ module Woods
       #
       # @return [Array<ExtractedUnit>] List of decorator units
       def extract_all
-        @directories.flat_map do |dir|
-          Dir[dir.join('**/*.rb')].filter_map do |file|
-            extract_decorator_file(file)
-          end
+        find_files_in_directories(@directories).filter_map do |file|
+          extract_decorator_file(file)
         end
       end
 
@@ -246,7 +244,7 @@ module Woods
 
         deps.concat(scan_common_dependencies(source))
 
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
     end
   end

data/lib/woods/extractors/event_extractor.rb

@@ -42,7 +42,7 @@ module Woods
       def extract_all
         event_map = {}
 
-        @directories.flat_map { |dir| Dir[dir.join('**/*.rb')] }.each do |file_path|
+        find_files_in_directories(@directories).each do |file_path|
           scan_file(file_path, event_map)
         end
 
@@ -204,7 +204,7 @@ module Woods
       # @return [Array<Hash>]
       def build_dependencies(combined_source)
         deps = scan_common_dependencies(combined_source)
-        deps.uniq { |d| [d[:type], d[:target]] }
+        consolidate_dependencies(deps)
       end
     end
   end