woods 2.0.0.beta1 → 2.0.0.beta2

data/lib/woods/flow_assembler.rb

@@ -4,7 +4,6 @@ require 'digest'
 require 'json'
 require 'set'
 require_relative 'ast/parser'
-require_relative 'ast/method_extractor'
 require_relative 'flow_analysis/operation_extractor'
 require_relative 'flow_document'
 
@@ -25,15 +24,25 @@ module Woods
   #   puts flow.to_markdown
   #
   class FlowAssembler
+    # How many entries each per-instance memo holds before the oldest is
+    # dropped. One assembler serves a whole precompute run (every controller,
+    # every action), so an unbounded memo would hold the source and the parsed
+    # AST of every unit the run ever reached. A thousand covers the shared
+    # services a large app's flows keep landing on without pinning the whole
+    # index in memory.
+    MEMO_LIMIT = 1_000
+
     # @param graph [DependencyGraph] The dependency graph for resolving targets
     # @param extracted_dir [String] Directory containing extracted unit JSON files
     def initialize(graph:, extracted_dir:)
       @graph = graph
       @extracted_dir = extracted_dir
       @parser = Ast::Parser.new
-      @method_extractor = Ast::MethodExtractor.new(parser: @parser)
       @operation_extractor = FlowAnalysis::OperationExtractor.new
       @resolved_targets = {}
+      @unit_cache = {}
+      @ast_cache = {}
+      @method_node_cache = {}
     end
 
     # Assemble an execution flow from the given entry point.
@@ -114,7 +123,7 @@ module Woods
         file_path = unit_data[:file_path]
 
         # Extract operations from the relevant method
-        operations = extract_operations(source_code, method_name, metadata, unit_type)
+        operations = extract_operations(unit_id, source_code, method_name, metadata, unit_type)
 
         step = {
           unit: identifier,
@@ -146,19 +155,75 @@ module Woods
     # one the callee actually defines locally (inherited, dynamically
     # defined, or metaprogrammed) — losing the trace entirely would be worse
     # than over-including it.
-    def extract_operations(source_code, method_name, metadata, unit_type)
+    def extract_operations(unit_id, source_code, method_name, metadata, unit_type)
       operations = []
 
       # For controllers, prepend before_action callbacks
       prepend_callbacks(operations, metadata, method_name) if unit_type == 'controller'
 
-      scope_node = method_name && @method_extractor.extract_method(source_code, method_name)
-      scope_node ||= @parser.parse(source_code)
+      scope_node = method_node(unit_id, source_code, method_name)
+      scope_node ||= parsed_source(unit_id, source_code)
       operations.concat(@operation_extractor.extract(scope_node))
 
       operations
     end
 
+    # The unit's whole parsed source, parsed once per assembler instance.
+    #
+    # A unit reached from several controllers used to be re-parsed once per
+    # action of every controller that reached it, and Prism dominates the
+    # flow run's cost on a large app.
+    #
+    # @param unit_id [String] cache key; a unit's source is fixed for a run
+    # @param source_code [String] the unit's source
+    # @return [Ast::Node] root node
+    def parsed_source(unit_id, source_code)
+      memoize(@ast_cache, unit_id) { @parser.parse(source_code) }
+    end
+
+    # The `def` node for +method_name+, from a per-unit index built off the
+    # memoized parse.
+    #
+    # First definition wins for a name defined more than once, matching
+    # {Ast::MethodExtractor#extract_method}'s first-match lookup.
+    #
+    # @param unit_id [String] cache key
+    # @param source_code [String] the unit's source
+    # @param method_name [String, nil] the method the caller invoked
+    # @return [Ast::Node, nil] nil when no method was named, or the unit does
+    #   not define one by that name
+    def method_node(unit_id, source_code, method_name)
+      return nil unless method_name
+
+      nodes = memoize(@method_node_cache, unit_id) do
+        parsed_source(unit_id, source_code).find_all(:def).each_with_object({}) do |node, index|
+          index[node.method_name] ||= node
+        end
+      end
+
+      nodes[method_name.to_s]
+    end
+
+    # Read +key+ from +cache+, computing and storing it on a miss.
+    #
+    # Least-recently-used, using the fact that a Ruby Hash iterates in
+    # insertion order: a hit re-inserts the key at the young end, and
+    # +Hash#shift+ drops the old end once the cache is over {MEMO_LIMIT}.
+    # A nil value is a real answer (most call targets are not units) and is
+    # cached like any other.
+    #
+    # @param cache [Hash]
+    # @param key [Object]
+    # @return [Object] the cached or freshly computed value
+    def memoize(cache, key)
+      return cache[key] = cache.delete(key) if cache.key?(key)
+
+      value = yield
+      cache[key] = value
+      cache.shift while cache.size > MEMO_LIMIT
+      value
+    end
+
     # Prepend before_action callbacks from controller metadata.
     #
     # Handles two metadata formats:
@@ -296,13 +361,27 @@ module Woods
       end
     end
 
-    # Load an ExtractedUnit's data from its JSON file on disk.
+    # Load an ExtractedUnit's data from its JSON file on disk, memoized per
+    # assembler instance.
+    #
+    # The glob and the JSON parse used to run once per expansion, so a unit
+    # reached from many controllers paid for both once per action of every one
+    # of them. The memo also makes {#extract_route} free: the entry unit it
+    # asks for is the one {#expand} just loaded.
+    #
+    # @param unit_id [String] Unit identifier
+    # @return [Hash, nil] symbolized unit data, or nil when no file matched
+    def load_unit(unit_id)
+      memoize(@unit_cache, unit_id) { read_unit(unit_id) }
+    end
+
+    # The disk half of {#load_unit}.
     #
     # Uses {Extractor#collision_safe_filename} convention (with SHA256 digest suffix).
     # Falls back to legacy {Extractor#safe_filename} for older indexes.
     # Searches across type subdirectories since the extractor writes to
     # `<output_dir>/<type>/<filename>.json`.
-    def load_unit(unit_id)
+    def read_unit(unit_id)
       base = unit_id.gsub('::', '__').gsub(/[^a-zA-Z0-9_-]/, '_')
       digest = Digest::SHA256.hexdigest(unit_id)[0, 8]
       filenames = [

data/lib/woods/flow_precomputer.rb

@@ -89,11 +89,17 @@ module Woods
     #   controller units, rehydrated from the payload this run seeded
     # @param removed_identifiers [Array<String>] controller identifiers
     #   whose unit the run pruned
+    # @param carried_identifiers [Array<String>] controllers the run
+    #   re-extracted whose flows it established cannot have changed. Their
+    #   index entries stay as they are and their annotation is read back out
+    #   of those entries, because a re-extracted unit lost the annotation
+    #   whether or not its flows moved. Skipping the assembly is the whole
+    #   saving; skipping the annotation would diverge from a full run.
     # @return [Hash{String => Hash{String => String}}] per-controller
     #   annotation (action => relative flow path) to write into the units'
     #   metadata; an empty hash means "no flows", which clears any
     #   annotation a previous run had written
-    def recompute_delta(touched_units:, removed_identifiers: [])
+    def recompute_delta(touched_units:, removed_identifiers: [], carried_identifiers: [])
       FileUtils.mkdir_p(@flows_dir)
 
       assembler = FlowAssembler.new(graph: @graph, extracted_dir: @output_dir)
@@ -110,8 +116,13 @@ module Woods
         delta.merge!(entries)
       end
 
+      previous = previous_flow_index
+      Array(carried_identifiers).each do |identifier|
+        annotations[identifier] = carried_annotation(previous, identifier)
+      end
+
       replaced = touched_units.map(&:identifier) + Array(removed_identifiers)
-      carried = previous_flow_index.reject { |entry_point, _path| replaced.include?(controller_of(entry_point)) }
+      carried = previous.reject { |entry_point, _path| replaced.include?(controller_of(entry_point)) }
 
       write_flow_index(carried.merge(delta))
 
@@ -156,6 +167,18 @@ module Woods
       [entries, unit_flow_paths]
     end
 
+    # One controller's annotation, rebuilt from the index entries it keeps.
+    #
+    # @param previous [Hash{String => String}] the previous flow index
+    # @param identifier [String] controller identifier
+    # @return [Hash{String => String}] action to relative flow path
+    def carried_annotation(previous, identifier)
+      previous.each_with_object({}) do |(entry_point, path), annotation|
+        controller, action = entry_point.to_s.split('#', 2)
+        annotation[action] = path if action && controller == identifier
+      end
+    end
+
     # The controller part of a flow index entry point.
     #
     # @param entry_point [String]
@@ -183,9 +206,10 @@ module Woods
 
     # Assemble a flow for one entry point and write the JSON file.
     #
-    # Written via {Woods::AtomicFile} (temp + fsync + rename) like every
-    # other index artifact — a plain +File.write+ interrupted mid-write left
-    # a torn partial for the MCP read side to trip over.
+    # Written via {Woods::AtomicFile} (temp + rename) like every other index
+    # artifact. A plain +File.write+ interrupted mid-write left a torn
+    # partial for the MCP read side to trip over. Not durable by default: see
+    # {#durable_writes?}.
     #
     # @param assembler [FlowAssembler]
     # @param entry_point [String]
@@ -199,7 +223,7 @@ module Woods
 
       filename = Woods::FilenameUtils.flow_filename(controller_id, action)
 
-      Woods::AtomicFile.write(File.join(@flows_dir, filename), canonical_json(flow.to_h))
+      Woods::AtomicFile.write(File.join(@flows_dir, filename), canonical_json(flow.to_h), durable: durable_writes?)
 
       # Relative — this value is persisted (flow_index.json and the unit's
       # metadata[:flow_paths]), so it must not carry this machine's root.
@@ -213,7 +237,20 @@ module Woods
     # @param flow_map [Hash{String => String}]
     def write_flow_index(flow_map)
       index_path = File.join(@flows_dir, 'flow_index.json')
-      Woods::AtomicFile.write(index_path, canonical_json(flow_map))
+      Woods::AtomicFile.write(index_path, canonical_json(flow_map), durable: durable_writes?)
+    end
+
+    # Whether a flow document pays for its own fsync as it is written.
+    #
+    # False by default. A flow document is a payload file: nothing reads it
+    # until +generation.json+ names the payload, and
+    # {Woods::AtomicFile.sync_directory_tree} flushes the whole payload before
+    # that pointer is written. `durable_payload_writes = true` buys the
+    # per-file fsync back for a host that wants it.
+    #
+    # @return [Boolean]
+    def durable_writes?
+      Woods.configuration&.durable_payload_writes ? true : false
     end
 
     # Emit deterministic pretty JSON — keys recursively sorted so two runs

data/lib/woods/graph_analyzer.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'digest'
 require 'set'
 
 module Woods
@@ -55,11 +56,30 @@ module Woods
     # can tell whether it was truncated (B-182).
     DEFAULT_VOLATILE_LIMIT = 20
 
+    # How many distinct cycles {#detect_cycles} enumerates before it stops.
+    # The DFS finds one cycle per back-edge, and a dense graph has tens of
+    # thousands of them; nobody reads past the first few hundred, and the
+    # per-cycle signature work is what made analysis the fixed floor of every
+    # incremental run. `nil` means no cap.
+    DEFAULT_CYCLE_LIMIT = 500
+
+    # The longest cycle {#detect_cycles} will record, counted in distinct
+    # nodes. A back-edge deep in a DFS closes a cycle as long as the path,
+    # which on a large graph is thousands of nodes: unreadable as a report and
+    # expensive to canonicalize. `nil` means no cap.
+    DEFAULT_CYCLE_MAX_LENGTH = 50
+
     # @param dependency_graph [DependencyGraph] The graph to analyze
     # @param volatile_ratio [Numeric] see {#volatile_dependencies}
-    def initialize(dependency_graph, volatile_ratio: DEFAULT_VOLATILE_RATIO)
+    # @param cycle_limit [Integer, nil] see {DEFAULT_CYCLE_LIMIT}
+    # @param cycle_max_length [Integer, nil] see {DEFAULT_CYCLE_MAX_LENGTH}
+    def initialize(dependency_graph, volatile_ratio: DEFAULT_VOLATILE_RATIO,
+                   cycle_limit: DEFAULT_CYCLE_LIMIT, cycle_max_length: DEFAULT_CYCLE_MAX_LENGTH)
       @graph = dependency_graph
       @volatile_ratio = volatile_ratio.to_f
+      @cycle_limit = cycle_limit
+      @cycle_max_length = cycle_max_length
+      @cycle_limit_reached = false
     end
 
     # ══════════════════════════════════════════════════════════════════════
@@ -147,6 +167,19 @@ module Woods
       @cycles ||= detect_cycles
     end
 
+    # Whether {#cycles} is a truncated view of the graph's cycles.
+    #
+    # True when either cap fired: the count cap stopped enumeration, or at
+    # least one cycle was longer than the length cap and was skipped. A reader
+    # of the array alone cannot tell, so this is published as
+    # `stats[:cycle_limit_reached]` alongside it.
+    #
+    # @return [Boolean]
+    def cycle_limit_reached?
+      cycles
+      @cycle_limit_reached
+    end
+
     # Units that bridge different types in the graph.
     #
     # Computes a simplified betweenness centrality metric — for each unit, we
@@ -179,6 +212,7 @@ module Woods
 
       pairs.each do |source, target|
         path = bfs_shortest_path(source, target)
+
         next unless path && path.size > 2
 
         # Credit intermediate nodes (exclude source and target)
@@ -361,6 +395,7 @@ module Woods
           dead_end_count: computed_dead_ends.size,
           hub_count: computed_hubs.size,
           cycle_count: computed_cycles.size,
+          cycle_limit_reached: cycle_limit_reached?,
           cross_database_edge_count: computed_cross_database.size,
           volatile_dependency_count: all_volatile_dependencies.size,
           volatile_dependencies_limit: DEFAULT_VOLATILE_LIMIT,
@@ -802,6 +837,7 @@ module Woods
     # @return [Array<Array<String>>] Detected cycles
     def detect_cycles
       nodes = graph_nodes
+      @cycle_limit_reached = false
       return [] if nodes.empty?
 
       white = 0
@@ -813,53 +849,48 @@ module Woods
       found_cycles = []
       seen_cycle_signatures = Set.new
 
-      nodes.keys.sort.each do |start_node|
-        next unless color[start_node] == white
+      catch(:cycle_limit) do
+        nodes.keys.sort.each do |start_node|
+          next unless color[start_node] == white
 
-        # Iterative DFS using an explicit stack.
-        # Each entry is [node, :enter] or [node, :exit].
-        stack = [[start_node, :enter]]
+          # Iterative DFS using an explicit stack.
+          # Each entry is [node, :enter] or [node, :exit].
+          stack = [[start_node, :enter]]
 
-        # Track the current DFS path for cycle extraction.
-        path = []
+          # Track the current DFS path for cycle extraction.
+          path = []
 
-        while stack.any?
-          node, action = stack.pop
+          while stack.any?
+            node, action = stack.pop
 
-          if action == :exit
-            color[node] = black
-            path.pop
-            next
-          end
+            if action == :exit
+              color[node] = black
+              path.pop
+              next
+            end
 
-          # :enter action
-          next unless color[node] == white
-
-          color[node] = gray
-          path.push(node)
-          stack.push([node, :exit])
-
-          # Not sorted, deliberately: this list is the unit's own declared
-          # dependency order, which is identical in a full and an incremental
-          # run, so sorting it would change nothing any test can observe.
-          neighbors = @graph.dependencies_of(node)
-          neighbors.each do |neighbor|
-            case color[neighbor]
-            when white
-              parent[neighbor] = node
-              stack.push([neighbor, :enter])
-            when gray
-              # Found a cycle — extract it from the path
-              cycle = extract_cycle_from_path(path, neighbor)
-              if cycle
-                sig = normalize_cycle_signature(cycle)
-                unless seen_cycle_signatures.include?(sig)
-                  seen_cycle_signatures.add(sig)
-                  found_cycles << cycle
-                end
+            # :enter action
+            next unless color[node] == white
+
+            color[node] = gray
+            path.push(node)
+            stack.push([node, :exit])
+
+            # Not sorted, deliberately: this list is the unit's own declared
+            # dependency order, which is identical in a full and an incremental
+            # run, so sorting it would change nothing any test can observe.
+            neighbors = @graph.dependencies_of(node)
+            neighbors.each do |neighbor|
+              case color[neighbor]
+              when white
+                parent[neighbor] = node
+                stack.push([neighbor, :enter])
+              when gray
+                # Found a cycle: extract it from the path
+                collect_cycle(path, neighbor, found_cycles, seen_cycle_signatures)
               end
+              # black nodes are fully explored, skip them
             end
-            # black nodes are fully explored, skip them
           end
         end
       end
@@ -869,16 +900,52 @@ module Woods
       found_cycles
     end
 
+    # Record the cycle closed by a back-edge to +cycle_start+.
+    #
+    # Skips a cycle longer than the length cap and one already recorded under
+    # another rotation. Throws +:cycle_limit+ once the count cap is full,
+    # which ends the whole scan in {#detect_cycles}.
+    #
+    # @param path [Array<String>] Current DFS path
+    # @param cycle_start [String] The node that closes the cycle
+    # @param found_cycles [Array<Array<String>>] Accumulator
+    # @param seen [Set<String>] Signatures already recorded
+    # @return [void]
+    def collect_cycle(path, cycle_start, found_cycles, seen)
+      cycle = extract_cycle_from_path(path, cycle_start, max_length: @cycle_max_length)
+      if cycle == :too_long
+        @cycle_limit_reached = true
+        return
+      end
+      return unless cycle
+
+      signature = normalize_cycle_signature(cycle)
+      return if seen.include?(signature)
+
+      seen.add(signature)
+      found_cycles << cycle
+      return unless @cycle_limit && found_cycles.size >= @cycle_limit
+
+      @cycle_limit_reached = true
+      throw :cycle_limit
+    end
+
     # Extracts a cycle from the current DFS path when a back-edge to
     # +cycle_start+ is found.
     #
+    # The length check runs on indexes, before the slice: an over-long cycle
+    # costs nothing beyond the +index+ lookup that found it.
+    #
     # @param path [Array<String>] Current DFS path
     # @param cycle_start [String] The node that closes the cycle
-    # @return [Array<String>, nil] The cycle path ending with cycle_start repeated,
-    #   or nil if cycle_start is not in the path
-    def extract_cycle_from_path(path, cycle_start)
+    # @param max_length [Integer, nil] Longest cycle to build, in distinct nodes
+    # @return [Array<String>, Symbol, nil] The cycle path ending with cycle_start
+    #   repeated; +:too_long+ when it exceeds +max_length+; nil when cycle_start
+    #   is not in the path
+    def extract_cycle_from_path(path, cycle_start, max_length: nil)
       start_index = path.index(cycle_start)
       return nil unless start_index
+      return :too_long if max_length && (path.size - start_index) > max_length
 
       path[start_index..] + [cycle_start]
     end
@@ -886,17 +953,20 @@ module Woods
     # Normalize a cycle so that duplicate rotations are treated as the same cycle.
     # For example, [A, B, C, A] and [B, C, A, B] are the same cycle.
     #
+    # Keyed by digest rather than by the joined path: the set holds one
+    # 64-byte key per cycle instead of a string as long as the cycle, so
+    # membership stays constant-cost however deep the DFS went.
+    #
     # @param cycle [Array<String>] Cycle path with repeated last element
-    # @return [String] Canonical string representation
+    # @return [String] Canonical hex digest of the rotated loop
     def normalize_cycle_signature(cycle)
       # Remove the trailing repeated element to get the raw loop
       loop_nodes = cycle[0..-2]
-      return loop_nodes.join('->') if loop_nodes.empty?
+      return Digest::SHA256.hexdigest('') if loop_nodes.empty?
 
       # Rotate so the lexicographically smallest element is first
       min_index = loop_nodes.each_with_index.min_by { |node, _i| node }.last
-      rotated = loop_nodes.rotate(min_index)
-      rotated.join('->')
+      Digest::SHA256.hexdigest(loop_nodes.rotate(min_index).join('->'))
     end
 
     # ──────────────────────────────────────────────────────────────────────
@@ -927,32 +997,65 @@ module Woods
       pairs.to_a
     end
 
+    # Forward adjacency, resolved once per node per analyzer instance.
+    #
+    # {#bridges} runs `sample_size` whole-graph traversals, and
+    # `DependencyGraph#dependencies_of` sorts and flattens the node's edge
+    # buckets on every call, so an uncached BFS re-derived the same adjacency
+    # list up to 200 times per node. Populated on demand rather than up front:
+    # a traversal that never reaches a node should not pay for it.
+    #
+    # @return [Hash{String => Array<String>}]
+    def adjacency
+      @adjacency ||= Hash.new { |cache, identifier| cache[identifier] = @graph.dependencies_of(identifier) }
+    end
+
     # BFS shortest path between two nodes, following forward edges.
     #
+    # Carries parent pointers rather than a path per queue entry. The old form
+    # allocated a copy of the path so far for every node it enqueued, which on
+    # a large graph is a full array per node per traversal; the path is now
+    # built once, for the one node that matched.
+    #
     # @param source [String] Starting node identifier
     # @param target [String] Target node identifier
     # @return [Array<String>, nil] Shortest path or nil if unreachable
     def bfs_shortest_path(source, target)
       return [source] if source == target
 
-      visited = Set.new([source])
-      queue = [[source, [source]]]
+      parents = { source => nil }
+      queue = [source]
+      head = 0
 
-      while queue.any?
-        current, path = queue.shift
+      while head < queue.size
+        current = queue[head]
+        head += 1
 
-        @graph.dependencies_of(current).each do |neighbor|
-          next if visited.include?(neighbor)
+        adjacency[current].each do |neighbor|
+          next if parents.key?(neighbor)
 
-          new_path = path + [neighbor]
-          return new_path if neighbor == target
+          parents[neighbor] = current
+          return path_to(parents, neighbor) if neighbor == target
 
-          visited.add(neighbor)
-          queue.push([neighbor, new_path])
+          queue.push(neighbor)
         end
       end
 
       nil
     end
+
+    # Walk parent pointers back to the source and reverse.
+    #
+    # @param parents [Hash{String => String, nil}] node => the node it was reached from
+    # @param node [String] the end of the path
+    # @return [Array<String>] source-first path ending at +node+
+    def path_to(parents, node)
+      path = []
+      while node
+        path << node
+        node = parents[node]
+      end
+      path.reverse
+    end
   end
 end

data/lib/woods/payload_store.rb

@@ -99,6 +99,10 @@ module Woods
       return unless from.directory?
 
       to = Pathname.new(target.to_s)
+      # The one directory `find` never offers: it yields the root as `.`, and
+      # a caller seeding a subtree (Extractor#seed_payload_from_flat_root)
+      # passes a target that does not exist yet.
+      FileUtils.mkdir_p(to.to_s)
       from.find do |entry|
         relative = entry.relative_path_from(from)
         next if relative.to_s == '.'
@@ -211,11 +215,20 @@ module Woods
       end
     end
 
+    # `Pathname#find` is a pre-order walk: a directory is always visited
+    # before anything inside it. So the directory branch has already created
+    # every parent a file could need, and the `mkdir_p` the file branch used
+    # to run was one stat-heavy syscall chain per unit re-proving what the
+    # previous entry established. On a payload of 8000+ files that was the
+    # bulk of the incremental seed.
+    #
+    # @param entry [Pathname] source entry, as `find` yielded it
+    # @param destination [Pathname] where it belongs in the target
+    # @return [void]
     def replicate(entry, destination)
       if entry.directory?
         FileUtils.mkdir_p(destination.to_s)
       else
-        FileUtils.mkdir_p(destination.dirname.to_s)
         link_or_copy(entry, destination)
       end
     end

data/lib/woods/ruby_analyzer/trace_enricher.rb

@@ -21,7 +21,10 @@ module Woods
       #
       # @yield Block to trace
       # @return [Array<Hash>] Collected trace events
+      # @raise [ArgumentError] if no block is given
       def self.record(&block)
+        raise ArgumentError, 'block required' unless block
+
         traces = []
 
         trace = TracePoint.new(:call, :return) do |tp|

data/lib/woods/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Woods
-  VERSION = '2.0.0.beta1'
+  VERSION = '2.0.0.beta2'
 end