woods 1.6.1 → 2.0.0.beta2

data/lib/woods/flow_precomputer.rb

@@ -4,6 +4,7 @@ require 'json'
 require 'fileutils'
 require_relative 'filename_utils'
 require_relative 'flow_assembler'
+require_relative 'atomic_file'
 
 module Woods
   # Orchestrates pre-computation of request flow maps for all controller actions.
@@ -11,10 +12,20 @@ module Woods
   # After the dependency graph is built, FlowPrecomputer iterates controller units,
   # runs FlowAssembler for each action, and writes flow documents to disk.
   #
+  # All persisted path values — `flow_index.json` entries and each controller
+  # unit's `metadata[:flow_paths]` — are RELATIVE to the output dir
+  # (e.g. "flows/OrdersController_create.json"), matching the #166 portability
+  # contract: extraction inside a container must not bake `/app/...` prefixes
+  # into artifacts a host reads through a volume mount, and identical trees at
+  # different roots must produce identical bytes (which is also what keeps
+  # {#canonical_json}'s churn-avoidance honest). Readers join the relative
+  # value against their own index dir (see the MCP server's
+  # `load_precomputed_flow`).
+  #
   # @example
   #   precomputer = FlowPrecomputer.new(units: all_units, graph: dep_graph, output_dir: out)
   #   flow_map = precomputer.precompute
-  #   flow_map["OrdersController#create"] #=> "/tmp/woods/flows/OrdersController_create.json"
+  #   flow_map["OrdersController#create"] #=> "flows/OrdersController_create.json"
   #
   class FlowPrecomputer
     # Default maximum recursion depth for flow assembly
@@ -34,7 +45,15 @@ module Woods
 
     # Pre-compute flow documents for all controller actions.
     #
-    # @return [Hash{String => String}] Map of entry_point to flow file path
+    # Assembly is fail closed (M3 review round 3): a per-action failure
+    # raises instead of skipping the entry, so a partial index — missing
+    # entries alongside a freshly written graph — is never written and the
+    # caller aborts before publishing. The full extraction path shares this
+    # contract with the incremental delta.
+    #
+    # @return [Hash{String => String}] Map of entry_point to output_dir-relative
+    #   flow file path (e.g. "flows/OrdersController_create.json")
+    # @raise [Woods::ExtractionError] when any action fails to assemble
     def precompute
       FileUtils.mkdir_p(@flows_dir)
 
@@ -42,18 +61,8 @@ module Woods
       flow_map = {}
 
       controller_units.each do |unit|
-        actions = unit.metadata[:actions] || unit.metadata['actions'] || []
-        unit_flow_paths = {}
-
-        actions.each do |action|
-          entry_point = "#{unit.identifier}##{action}"
-          flow_path = assemble_and_write(assembler, entry_point, unit.identifier, action)
-          next unless flow_path
-
-          flow_map[entry_point] = flow_path
-          unit_flow_paths[action] = flow_path
-        end
-
+        entries, unit_flow_paths = assemble_controller_unit(assembler, unit)
+        flow_map.merge!(entries)
         unit.metadata[:flow_paths] = unit_flow_paths if unit_flow_paths.any?
       end
 
@@ -62,6 +71,64 @@ module Woods
       flow_map
     end
 
+    # Recompute the flow documents for the controllers an incremental run
+    # touched (M3), carrying every untouched controller's entries forward
+    # from the previous generation's flow_index.json — payload seeding
+    # hardlinks it into this run's payload directory — and rewriting the
+    # index with the merged result.
+    #
+    # Touched controllers replace their previous entries wholesale (a full
+    # run would emit exactly their current actions), so an action removed
+    # from a re-extracted controller leaves the index even though the file
+    # still exists. Controllers named in +removed_identifiers+ lost their
+    # unit this run (deleted or renamed) and leave the index entirely;
+    # {Woods::Extractor#sweep_orphaned_flow_files} then removes the
+    # documents nothing references anymore.
+    #
+    # @param touched_units [Array<ExtractedUnit>] the run's re-extracted
+    #   controller units, rehydrated from the payload this run seeded
+    # @param removed_identifiers [Array<String>] controller identifiers
+    #   whose unit the run pruned
+    # @param carried_identifiers [Array<String>] controllers the run
+    #   re-extracted whose flows it established cannot have changed. Their
+    #   index entries stay as they are and their annotation is read back out
+    #   of those entries, because a re-extracted unit lost the annotation
+    #   whether or not its flows moved. Skipping the assembly is the whole
+    #   saving; skipping the annotation would diverge from a full run.
+    # @return [Hash{String => Hash{String => String}}] per-controller
+    #   annotation (action => relative flow path) to write into the units'
+    #   metadata; an empty hash means "no flows", which clears any
+    #   annotation a previous run had written
+    def recompute_delta(touched_units:, removed_identifiers: [], carried_identifiers: [])
+      FileUtils.mkdir_p(@flows_dir)
+
+      assembler = FlowAssembler.new(graph: @graph, extracted_dir: @output_dir)
+      delta = {}
+      annotations = {}
+
+      touched_units.each do |unit|
+        # A per-action skip would publish an index missing that entry, so
+        # assembly raises. The incremental path must not advance publication
+        # over a broken assembly; the caller aborts before the generation
+        # bump.
+        entries, unit_flow_paths = assemble_controller_unit(assembler, unit)
+        annotations[unit.identifier] = unit_flow_paths
+        delta.merge!(entries)
+      end
+
+      previous = previous_flow_index
+      Array(carried_identifiers).each do |identifier|
+        annotations[identifier] = carried_annotation(previous, identifier)
+      end
+
+      replaced = touched_units.map(&:identifier) + Array(removed_identifiers)
+      carried = previous.reject { |entry_point, _path| replaced.include?(controller_of(entry_point)) }
+
+      write_flow_index(carried.merge(delta))
+
+      annotations
+    end
+
     private
 
     # Filter units to only controllers.
@@ -71,33 +138,119 @@ module Woods
       @units.select { |u| u.type.to_s == 'controller' }
     end
 
+    # Assemble and write every flow for one controller unit.
+    #
+    # Both entry points fail closed: a per-action failure aborts the run
+    # rather than publishing a flow index missing that entry (EXTB-14 — the
+    # old `fail_closed:` switch documented a log-and-skip contract for the
+    # full path that no caller ever asked for).
+    #
+    # @param assembler [FlowAssembler]
+    # @param unit [ExtractedUnit]
+    # @return [Array(Hash{String => String}, Hash{String => String})] the
+    #   entry_point => relative-path entries for the flow index, and the
+    #   action => relative-path annotation for the unit's metadata
+    # @raise [Woods::ExtractionError] when an action fails to assemble
+    def assemble_controller_unit(assembler, unit)
+      entries = {}
+      unit_flow_paths = {}
+
+      actions = unit.metadata[:actions] || unit.metadata['actions'] || []
+      actions.each do |action|
+        entry_point = "#{unit.identifier}##{action}"
+        flow_path = assemble_and_write(assembler, entry_point, unit.identifier, action)
+
+        entries[entry_point] = flow_path
+        unit_flow_paths[action] = flow_path
+      end
+
+      [entries, unit_flow_paths]
+    end
+
+    # One controller's annotation, rebuilt from the index entries it keeps.
+    #
+    # @param previous [Hash{String => String}] the previous flow index
+    # @param identifier [String] controller identifier
+    # @return [Hash{String => String}] action to relative flow path
+    def carried_annotation(previous, identifier)
+      previous.each_with_object({}) do |(entry_point, path), annotation|
+        controller, action = entry_point.to_s.split('#', 2)
+        annotation[action] = path if action && controller == identifier
+      end
+    end
+
+    # The controller part of a flow index entry point.
+    #
+    # @param entry_point [String]
+    # @return [String]
+    def controller_of(entry_point)
+      entry_point.to_s.split('#', 2).first
+    end
+
+    # The previous generation's flow index, or an empty hash when none was
+    # published. A missing index means there is nothing to carry forward
+    # (the caller established whether the family is genuinely absent); a
+    # corrupt one raises — publication must not advance over unreadable
+    # prior flow state.
+    #
+    # @return [Hash{String => String}]
+    # @raise [Woods::ExtractionError] when the index does not parse
+    def previous_flow_index
+      path = File.join(@flows_dir, 'flow_index.json')
+      return {} unless File.exist?(path)
+
+      JSON.parse(Woods::AtomicFile.read(path))
+    rescue JSON::ParserError => e
+      raise Woods::ExtractionError, "previous flow_index.json does not parse: #{e.message}"
+    end
+
     # Assemble a flow for one entry point and write the JSON file.
     #
+    # Written via {Woods::AtomicFile} (temp + rename) like every other index
+    # artifact. A plain +File.write+ interrupted mid-write left a torn
+    # partial for the MCP read side to trip over. Not durable by default: see
+    # {#durable_writes?}.
+    #
     # @param assembler [FlowAssembler]
     # @param entry_point [String]
     # @param controller_id [String]
     # @param action [String]
-    # @return [String, nil] The written file path, or nil on failure
+    # @return [String] The output_dir-relative path of the written file
+    #   (e.g. "flows/OrdersController_create.json")
+    # @raise [Woods::ExtractionError] when assembly or the write fails
     def assemble_and_write(assembler, entry_point, controller_id, action)
       flow = assembler.assemble(entry_point, max_depth: @max_depth)
 
       filename = Woods::FilenameUtils.flow_filename(controller_id, action)
-      flow_path = File.join(@flows_dir, filename)
 
-      File.write(flow_path, canonical_json(flow.to_h))
+      Woods::AtomicFile.write(File.join(@flows_dir, filename), canonical_json(flow.to_h), durable: durable_writes?)
 
-      flow_path
+      # Relative — this value is persisted (flow_index.json and the unit's
+      # metadata[:flow_paths]), so it must not carry this machine's root.
+      File.join('flows', filename)
     rescue StandardError => e
-      Rails.logger.error("[Woods] Flow precompute failed for #{entry_point}: #{e.message}")
-      nil
+      raise Woods::ExtractionError, "flow precompute failed for #{entry_point}: #{e.message}"
     end
 
-    # Write the flow index mapping entry points to file paths.
+    # Write the flow index mapping entry points to output_dir-relative file paths.
     #
     # @param flow_map [Hash{String => String}]
     def write_flow_index(flow_map)
       index_path = File.join(@flows_dir, 'flow_index.json')
-      File.write(index_path, canonical_json(flow_map))
+      Woods::AtomicFile.write(index_path, canonical_json(flow_map), durable: durable_writes?)
+    end
+
+    # Whether a flow document pays for its own fsync as it is written.
+    #
+    # False by default. A flow document is a payload file: nothing reads it
+    # until +generation.json+ names the payload, and
+    # {Woods::AtomicFile.sync_directory_tree} flushes the whole payload before
+    # that pointer is written. `durable_payload_writes = true` buys the
+    # per-file fsync back for a host that wants it.
+    #
+    # @return [Boolean]
+    def durable_writes?
+      Woods.configuration&.durable_payload_writes ? true : false
     end
 
     # Emit deterministic pretty JSON — keys recursively sorted so two runs

data/lib/woods/gem_mapper.rb

@@ -0,0 +1,285 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'fileutils'
+require 'json'
+require 'pathname'
+require 'time'
+
+require_relative 'atomic_file'
+require_relative 'coordination/pipeline_lock'
+require_relative 'dependency_graph'
+require_relative 'extracted_unit'
+require_relative 'filename_utils'
+require_relative 'generation'
+require_relative 'graph_analyzer'
+require_relative 'payload_store'
+require_relative 'ruby_analyzer'
+
+module Woods
+  # Publishes a static, source-only index of the Woods implementation.
+  class GemMapper
+    include FilenameUtils
+
+    SOURCE_GLOBS = %w[lib/woods.rb lib/woods/**/*.rb lib/tasks/**/*.rake exe/*].freeze
+    TYPE_DIRECTORIES = {
+      ruby_class: 'ruby_classes', ruby_module: 'ruby_modules',
+      ruby_method: 'ruby_methods', ruby_file: 'ruby_files'
+    }.freeze
+    PROVENANCE = {
+      mode: 'woods_static_ruby_source', runtime_fidelity: 'none', embeddings: 'absent',
+      included: SOURCE_GLOBS, excluded: %w[spec docs .git tmp vendor]
+    }.freeze
+
+    def initialize(root:, output_dir:)
+      @root = Pathname.new(root).expand_path
+      @output_dir = Pathname.new(output_dir).expand_path
+    end
+
+    # Publishes one complete source snapshot, or skips when it is unchanged.
+    def map!
+      with_lock do
+        snapshot = source_snapshot
+        return { status: :skipped, generation: current_generation.number } if current_checksum == snapshot[:checksum]
+
+        units, graph = GemMapSource.new(root: @root).build(snapshot[:sources])
+        payload = create_payload
+        GemMapPublisher.new(payload: payload, units: units, graph: graph, checksum: snapshot[:checksum]).write
+        ensure_snapshot_unchanged!(snapshot[:checksum])
+        marker = publish_generation
+        { status: :published, generation: marker.number, units: units.size }
+      end
+    end
+
+    private
+
+    def with_lock(&block)
+      Coordination::PipelineLock.new(lock_dir: @output_dir.to_s, name: 'woods_self_map').with_lock(&block)
+    end
+
+    def source_files
+      SOURCE_GLOBS.flat_map { |glob| Dir.glob(@root.join(glob).to_s) }
+                  .select { |path| File.file?(path) }.uniq.sort
+    end
+
+    def source_snapshot
+      sources = source_files.to_h { |path| [path, read_source(path)] }
+      { sources: sources, checksum: checksum_for(sources) }
+    end
+
+    def checksum_for(sources)
+      digest = Digest::SHA256.new
+      sources.sort.each { |path, source| digest << relative_path(path) << "\0" << source << "\0" }
+      digest.hexdigest
+    end
+
+    def read_source(path)
+      source = File.read(path, encoding: Encoding::UTF_8)
+      source.valid_encoding? ? source : source.scrub
+    end
+
+    def ensure_snapshot_unchanged!(checksum)
+      return if source_snapshot[:checksum] == checksum
+
+      raise Woods::ExtractionError, 'Woods source changed during self-map; refusing to publish a mixed snapshot'
+    end
+
+    def current_generation
+      Generation.new(output_dir: @output_dir).current
+    end
+
+    def next_generation
+      current_generation.number + 1
+    end
+
+    def current_checksum
+      payload = Generation.new(output_dir: @output_dir).payload_dir(current_generation)
+      manifest = payload.join('manifest.json')
+      return nil unless manifest.file?
+
+      JSON.parse(AtomicFile.read(manifest)).dig('provenance', 'source_checksum')
+    rescue JSON::ParserError, SystemCallError
+      nil
+    end
+
+    def relative_path(path)
+      Pathname.new(path).expand_path.relative_path_from(@root).to_s
+    rescue ArgumentError
+      raise Woods::ExtractionError, "Source path escapes gem root: #{path}"
+    end
+
+    def create_payload
+      PayloadStore.new(@output_dir).create(next_generation)
+    end
+
+    def publish_generation
+      marker = Generation.new(output_dir: @output_dir).bump!(
+        reason: 'woods_static_source_map', payload: PayloadStore.name_for(next_generation)
+      )
+      PayloadStore.new(@output_dir).prune(keep: PayloadStore::DEFAULT_RETENTION, protect: marker.number)
+      marker
+    end
+  end
+
+  # Turns a pre-read Woods source snapshot into typed static analysis units.
+  class GemMapSource
+    def initialize(root:)
+      @root = root
+    end
+
+    def build(sources)
+      units = build_units(sources)
+      graph = DependencyGraph.new
+      units.each { |unit| graph.register(unit) }
+      [units, graph]
+    end
+
+    private
+
+    def build_units(sources)
+      analyzable = sources.select { |path, _| path.end_with?('.rb', '.rake') || executable?(path) }
+      units = RubyAnalyzer.analyze(sources: analyzable)
+      units.each { |unit| unit.file_path = relative_path(unit.file_path) }
+      units.concat(file_units(sources))
+      disambiguate!(units)
+      qualify_dependencies(units)
+      resolve_dependents(units)
+      units.sort_by { |unit| [unit.identifier, unit.type.to_s, unit.file_path] }
+    end
+
+    def executable?(path)
+      path.start_with?(@root.join('exe').to_s + File::SEPARATOR)
+    end
+
+    def file_units(sources)
+      sources.map do |path, source|
+        relative = relative_path(path)
+        unit = ExtractedUnit.new(type: :ruby_file, identifier: "file:#{relative}", file_path: relative)
+        unit.source_code = source
+        unit.metadata = { static_kind: 'ruby_file', executable: executable?(path) }
+        unit
+      end
+    end
+
+    def disambiguate!(units)
+      units.group_by(&:identifier).each do |canonical, colliding|
+        colliding.sort_by! { |unit| [unit.file_path, unit.type.to_s] }
+        colliding.each_with_index do |unit, index|
+          unit.metadata[:canonical_identifier] = canonical
+          next if index.zero?
+
+          unit.identifier = "#{canonical} [#{unit.file_path}]"
+          unit.dependencies << { type: unit.type, target: canonical, via: :definition_of }
+        end
+      end
+    end
+
+    # Resolve only known lexical definitions, retaining external/dynamic names as written.
+    def qualify_dependencies(units)
+      definitions = units.select { |unit| %i[ruby_class ruby_module].include?(unit.type) }.to_h do |unit|
+        [unit.metadata[:canonical_identifier] || unit.identifier, unit]
+      end
+      units.each do |unit|
+        unit.dependencies.each { |dependency| qualify_dependency(unit, dependency, definitions) }
+      end
+    end
+
+    def qualify_dependency(unit, dependency, definitions)
+      target = dependency[:target]
+      return unless target.is_a?(String)
+
+      scopes = Array(unit.metadata[:lexical_scopes])
+      scopes = scopes.drop(1) if dependency[:via] == :inheritance
+      candidates = if target.start_with?('::')
+                     [target.delete_prefix('::')]
+                   else
+                     scopes.map { |scope| "#{scope}::#{target}" } + [target]
+                   end
+      resolved = candidates.find { |candidate| definitions.key?(candidate) }
+      return unless resolved
+
+      dependency[:target] = resolved
+      dependency[:type] = definitions[resolved].type
+    end
+
+    def resolve_dependents(units)
+      by_identifier = units.group_by(&:identifier)
+      units.each do |unit|
+        unit.dependencies.each do |dependency|
+          Array(by_identifier[dependency[:target]]).each do |target|
+            target.dependents << { type: unit.type, identifier: unit.identifier }
+          end
+        end
+      end
+    end
+
+    def relative_path(path)
+      Pathname.new(path).expand_path.relative_path_from(@root).to_s
+    rescue ArgumentError
+      raise Woods::ExtractionError, "Source path escapes gem root: #{path}"
+    end
+  end
+
+  # Writes a complete source-map payload before GemMapper atomically publishes it.
+  class GemMapPublisher
+    include FilenameUtils
+
+    def initialize(payload:, units:, graph:, checksum:)
+      @payload = payload
+      @units = units
+      @graph = graph
+      @checksum = checksum
+    end
+
+    def write
+      GemMapper::TYPE_DIRECTORIES.each do |type, directory|
+        write_type(@payload.join(directory), @units.select { |unit| unit.type == type })
+      end
+      write_graph
+      write_manifest
+      write_summary
+    end
+
+    private
+
+    def write_type(directory, units)
+      FileUtils.mkdir_p(directory)
+      units.each do |unit|
+        AtomicFile.write(directory.join(collision_safe_filename(unit.identifier)), JSON.pretty_generate(unit.to_h))
+      end
+      index = units.map do |unit|
+        { identifier: unit.identifier, file_path: unit.file_path, namespace: unit.namespace,
+          estimated_tokens: unit.estimated_tokens, chunk_count: unit.chunks.size }
+      end
+      AtomicFile.write(directory.join('_index.json'), JSON.pretty_generate(index))
+    end
+
+    def write_graph
+      data = @graph.to_h
+      data[:pagerank] = @graph.pagerank
+      AtomicFile.write(@payload.join('dependency_graph.json'), JSON.pretty_generate(data))
+      AtomicFile.write(@payload.join('graph_analysis.json'), JSON.pretty_generate(GraphAnalyzer.new(@graph).analyze))
+    end
+
+    def write_manifest
+      counts = GemMapper::TYPE_DIRECTORIES.keys.to_h do |type|
+        [GemMapper::TYPE_DIRECTORIES[type], @units.count { |unit| unit.type == type }]
+      end
+      manifest = {
+        extracted_at: Time.now.utc.iso8601, rails_version: nil, ruby_version: RUBY_VERSION,
+        counts: counts, total_units: @units.size, total_chunks: 0, git_sha: nil, git_branch: nil,
+        provenance: GemMapper::PROVENANCE.merge(source_checksum: @checksum),
+        graph_nodes: @graph.to_h.dig(:stats, :node_count), graph_edges: @graph.to_h.dig(:stats, :edge_count)
+      }
+      AtomicFile.write(@payload.join('manifest.json'), JSON.pretty_generate(manifest))
+    end
+
+    def write_summary
+      stats = @graph.to_h.fetch(:stats)
+      AtomicFile.write(@payload.join('SUMMARY.md'), [
+        '# Woods static Ruby source map', '', "Units: #{@units.size}", "Graph nodes: #{stats[:node_count]}",
+        "Graph edges: #{stats[:edge_count]}", 'Static source analysis only; not a Rails runtime extraction.'
+      ].join("\n"))
+    end
+  end
+end