woods 2.0.1 → 2.1.0

data/lib/woods/source_references/extraction.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+require_relative 'pass'
+require_relative 'cache'
+
+module Woods
+  module SourceReferences
+    # Pipeline hooks keep candidate cache, typed forward edges and reverse graph
+    # publication in one generation. A reference-only rewrite is not extraction
+    # and must not refresh runtime metadata, timestamps or Git facts.
+    module Extraction # rubocop:disable Metrics/ModuleLength -- shared full/incremental atomic publication hooks
+      private
+
+      def enrich_source_references_full
+        units = @results.values.flatten
+        records = units.map { |unit| reference_record(unit) }
+        result = reference_pass(records, full: true)
+        units.each do |unit|
+          edges = result.dependencies[[unit.type.to_s, unit.identifier]]
+          next unless edges
+
+          # Full extraction only adds records, so retain each extractor's exact
+          # in-memory representations for its existing specialized relationships.
+          unit.dependencies += internal_reference_edges(edges.drop(unit.dependencies.size))
+        end
+        write_source_reference_cache(result)
+      end
+
+      def internal_reference_edges(edges)
+        edges.map do |edge|
+          edge.transform_keys(&:to_sym).merge(type: edge.fetch('type').to_sym, via: edge.fetch('via').to_sym)
+        end
+      end
+
+      def reference_record(unit)
+        { 'type' => unit.type.to_s, 'identifier' => unit.identifier, 'file_path' => unit.file_path,
+          'metadata' => unit.metadata, 'dependencies' => JSON.parse(JSON.generate(unit.dependencies)) }
+      end
+
+      # Called before any incremental consumer mutates the seeded generation.
+      def prepare_source_reference_baseline
+        @source_reference_baseline = Cache.read(payload_dir.join(Cache::FILE_NAME))
+        return if @source_reference_baseline
+
+        empty_source_reference_baseline!
+      rescue Cache::Invalid
+        empty_source_reference_baseline!
+      end
+
+      def empty_source_reference_baseline!
+        if source_reference_identities.any? { |type, identifier| source_reference_node?(type, identifier) }
+          raise RebuildRequired,
+                'Source-reference baseline is missing or incompatible. Run bin/rails woods:extract once for a full ' \
+                'extraction; the previous published generation remains active.'
+        end
+        @source_reference_baseline = { 'version' => Cache::VERSION, 'files' => {}, 'owners' => [] }
+      end
+
+      def source_reference_node?(type, identifier)
+        return false unless Registry::TYPES.include?(type)
+        return false unless RuntimeLookup::CONSTANT.match?(identifier)
+
+        path = @dependency_graph.node(identifier, type: type)[:file_path]
+        return false if path.nil?
+
+        relative = File.expand_path(path, Rails.root).delete_prefix("#{File.expand_path(Rails.root)}/")
+        relative.match?(%r{\A(?:app|lib)/.*\.rb\z}) &&
+          relative.split('/').none? { |part| %w[vendor node_modules assets].include?(part) }
+      end
+
+      def source_reference_identities
+        @dependency_graph.to_h.fetch(:type_index).flat_map do |type, identifiers|
+          identifiers.map { |identifier| [type.to_sym, identifier] }
+        end
+      end
+
+      def enrich_source_references_incremental(affected_types)
+        records = source_reference_payload_records
+        result = reference_pass(records.values.map { |entry| entry.fetch(:data) }, full: false)
+        touched = Set.new
+        result.dependencies.each do |identity, edges|
+          record = records.fetch(identity)
+          next if record.fetch(:data).fetch('dependencies', []) == edges
+
+          rewrite_source_reference_edges(record, edges, affected_types)
+          touched.add(identity.last)
+        end
+        write_source_reference_cache(result)
+        touched
+      end
+
+      def source_reference_payload_records
+        owned = Array(@source_reference_baseline&.fetch('owners')).to_set do |owner|
+          [owner['type'], owner['identifier']]
+        end
+        source_reference_identities.to_h do |type, identifier|
+          read_caller = Pass::CALLER_TYPES.include?(type.to_s) &&
+                        (source_reference_node?(type, identifier) || owned.include?([type.to_s, identifier]))
+          record = if read_caller
+                     source_reference_caller_record(type, identifier)
+                   else
+                     node = @dependency_graph.node(identifier, type: type)
+                     { data: { 'type' => type.to_s, 'identifier' => identifier,
+                               'file_path' => node[:file_path], 'dependencies' => [] } }
+                   end
+          [[type.to_s, identifier], record]
+        end
+      end
+
+      def source_reference_caller_record(type, identifier)
+        key = self.class::TYPE_TO_EXTRACTOR_KEY.fetch(type)
+        path = payload_dir.join(key.to_s, collision_safe_filename(identifier))
+        data = JSON.parse(AtomicFile.read(path))
+        unless data.is_a?(Hash) && data['type'] == type.to_s && data['identifier'] == identifier
+          raise Woods::ExtractionError, "Source-reference unit identity mismatch for #{type}:#{identifier}"
+        end
+
+        { data: data, path: path, extractor_key: key }
+      end
+
+      def reference_pass(units, full:)
+        Pass.new(root: Rails.root, session: @source_inputs, units: units,
+                 extractor_keys: self.class::TYPE_TO_EXTRACTOR_KEY, baseline: @source_reference_baseline,
+                 refreshed: @source_reference_refreshed || [], full: full).call
+      end
+
+      def rewrite_source_reference_edges(record, edges, affected_types)
+        data = record.fetch(:data)
+        type = data.fetch('type').to_sym
+        identifier = data.fetch('identifier')
+        node = @dependency_graph.node(identifier, type: type)
+        unit = reference_unit(data, node, edges)
+        mark_dependents_dirty(identifier)
+        @dependency_graph.register(unit)
+        @dependency_graph.annotate(identifier, type: type, **DependencyGraph.persisted_node_attributes(node))
+        mark_dependents_dirty(identifier)
+        data['dependencies'] = edges
+        AtomicFile.write(record.fetch(:path), json_serialize(data), durable: payload_writes_durable?)
+        affected_types.add(record.fetch(:extractor_key))
+      end
+
+      def reference_unit(data, node, edges)
+        unit = ExtractedUnit.new(type: data.fetch('type').to_sym, identifier: data.fetch('identifier'),
+                                 file_path: node[:file_path])
+        unit.namespace = node[:namespace]
+        unit.metadata = data.fetch('metadata', {})
+        unit.dependencies = edges
+        unit
+      end
+
+      def write_source_reference_cache(result)
+        Cache.write(payload_dir.join(Cache::FILE_NAME), result.cache)
+        @source_reference_paths = result.paths.to_set
+      end
+
+      # The final source scan is already required by provenance publication.
+      # Reuse it to reject graph evidence whose captured inputs became unstable.
+      def verify_source_reference_publication!(manifest)
+        return if @source_reference_paths.nil? || @source_reference_paths.empty?
+
+        errors = manifest.data.fetch('errors')
+        unstable = errors.select do |error|
+          %w[source_changed_during_extraction source_changed_during_read source_file_unreadable
+             source_tree_unavailable external_source_path nonregular_source unverified_symlink_directory
+             undecodable_source_path
+             scan_time_budget scan_file_budget scan_byte_budget].include?(error['reason'])
+        end
+        return if unstable.empty?
+
+        raise Woods::ExtractionError,
+              'Source-reference source changed or could not be verified before publication ' \
+              "(#{source_verification_details(unstable)})"
+      end
+
+      def source_verification_details(errors)
+        details = errors.first(3).map do |error|
+          path = error['path']
+          path ? "#{error['reason']}: #{SourcePathEncoding.diagnostic(path)}" : error['reason']
+        end
+        details << "#{errors.size - 3} more source verification errors" if errors.size > 3
+        details.join('; ')
+      end
+    end
+  end
+end

data/lib/woods/source_references/inputs.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+require_relative 'registry'
+require_relative '../source_inputs/stable_reader'
+
+module Woods
+  module SourceReferences
+    # Collects original, captured sources once per path and reuses only candidates
+    # whose keyed identity still matches. Reads never advance runtime provenance.
+    class Inputs
+      MAX_SOURCE_BYTES = 256 * 1024 * 1024
+
+      # @param root [String] application root
+      # @param session [SourceInputs::Session] per-run capture and consumer ledger
+      # @param collector [Collector] source-only parser
+      def initialize(root:, session:, collector:)
+        @root = File.expand_path(root.to_s)
+        @session = session
+        @collector = collector
+      end
+
+      # @param units [Array<Hash>] complete typed unit set
+      # @param baseline [Hash, nil] validated prior cache
+      # @param fresh [Proc] true only for genuinely re-extracted units
+      # @param extractor_keys [Hash{Symbol => Symbol}] consumer mapping
+      # @return [Hash] relative paths to identity and candidate evidence
+      def call(units:, baseline:, fresh:, extractor_keys:)
+        @bytes = 0
+        @baseline = baseline
+        @trusted_baseline = @session.respond_to?(:trusted_reference_baseline?) &&
+                            @session.trusted_reference_baseline?(baseline)
+        @baseline_owners = Array(baseline&.fetch('owners')).to_h do |owner|
+          [owner.values_at('type', 'identifier'), owner]
+        end
+        grouped = group_sources(units)
+        grouped.keys.sort.each_with_object({}) do |path, files|
+          identity = captured_identity(path)
+          next unless identity
+
+          grouped.fetch(path).each { |unit| verify_consumption(unit, path, fresh, extractor_keys) }
+          files[path] = analysis(path, identity)
+        end
+      end
+
+      # @param path [String, nil] unit source path
+      # @return [String, nil] application-relative path, or nil outside the root
+      def relative(path)
+        return if path.nil? || path.to_s.empty?
+
+        absolute = File.expand_path(path.to_s, @root)
+        absolute.delete_prefix("#{@root}/") if absolute.start_with?("#{@root}/")
+      end
+
+      private
+
+      def group_sources(units)
+        grouped = Hash.new { |hash, path| hash[path] = [] }
+        units.select { |unit| eligible?(unit) }.each do |unit|
+          SourceContributors.paths(unit).each do |source_path|
+            path = relative(source_path)
+            raise Woods::ExtractionError, 'Invalid source-reference contributor path' unless eligible_path?(path)
+
+            grouped[path] << unit
+          end
+        end
+        grouped
+      end
+
+      def eligible?(unit)
+        path = relative(unit['file_path'])
+        Registry::TYPES.include?(unit['type']&.to_sym) && eligible_path?(path) &&
+          RuntimeLookup::CONSTANT.match?(unit['identifier'].to_s)
+      end
+
+      def eligible_path?(path)
+        path&.end_with?('.rb') && path.match?(%r{\A(?:app|lib)/}) &&
+          path.split('/').none? { |part| %w[vendor node_modules assets].include?(part) }
+      end
+
+      def captured_identity(path)
+        identity = @session.source_identity(path)
+        absolute = File.join(@root, path)
+        if identity.nil? && (File.exist?(absolute) || File.symlink?(absolute))
+          raise Woods::ExtractionError, "Source-reference input was not captured: #{path}"
+        end
+
+        identity
+      end
+
+      def analysis(path, identity)
+        prior = @baseline&.fetch('files')&.[](path)
+        return prior if prior && prior['identity'] == identity
+
+        source = stable_source(path)
+        @bytes += source.fetch('source').bytesize
+        raise Woods::ExtractionError, 'Source-reference parse byte budget exceeded' if @bytes > MAX_SOURCE_BYTES
+
+        result = @collector.call(source.fetch('source'))
+        raise Woods::ExtractionError, "Could not parse source references in #{path}" if result['parse_error']
+
+        { 'identity' => source.fetch('identity'), 'analysis' => result }
+      end
+
+      def stable_source(path)
+        @session.read_source(path)
+      rescue SourceInputs::StableReader::Error => e
+        raise Woods::ExtractionError, "Could not verify source references in #{path}: #{e.reason}. " \
+                                      'Retry the complete extraction batch in a fresh process against stable source; ' \
+                                      'the previous published generation remains active.'
+      end
+
+      def retained_owner?(unit, path)
+        return false unless @trusted_baseline
+
+        prior = @baseline_owners[unit.values_at('type', 'identifier')]
+        return false unless prior
+
+        paths = SourceContributors.paths(unit).map { |source| relative(source) }
+        paths.include?(path) && (prior['file_paths'] || [prior['file_path']]) == paths
+      end
+
+      def verify_consumption(unit, path, fresh, extractor_keys)
+        return if fresh.call(unit)
+
+        consumer = extractor_keys[unit['type'].to_sym]
+        unchanged = @baseline&.dig('files', path, 'identity') == @session.source_identity(path)
+        return if unchanged && consumer && (@session.consumed_source?(consumer, path) || retained_owner?(unit, path))
+
+        raise RebuildRequired, "Source-reference baseline needs a full extraction: unverified source #{path}. " \
+                               'Run bin/rails woods:extract; the previous published generation remains active.'
+      end
+    end
+  end
+end

data/lib/woods/source_references/parser_adapter.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+module Woods
+  module SourceReferences
+    # Optional parser-gem backend with the same ownership rules as Prism.
+    class ParserAdapter
+      KINDS = { class: :declaration, module: :declaration, const: :constant, sclass: :singleton,
+                def: :method, defs: :method }.freeze
+      DYNAMIC_BLOCK_METHODS = %i[class_eval module_eval instance_eval class_exec module_exec instance_exec].freeze
+
+      def parse(source)
+        require 'parser/current' unless defined?(::Parser::CurrentRuby)
+        parser = ::Parser::CurrentRuby.default_parser
+        diagnostics = []
+        parser.diagnostics.consumer = ->(diagnostic) { diagnostics << diagnostic }
+        parser.diagnostics.all_errors_are_fatal = true
+        parser.diagnostics.ignore_warnings = true
+        # Ruby source defaults to UTF-8 even when read as untouched binary bytes.
+        # Source::Buffer still honors an explicit encoding magic comment.
+        input = source.encoding == Encoding::BINARY ? source.dup.force_encoding(Encoding::UTF_8) : source
+        buffer = ::Parser::Source::Buffer.new('(source references)', source: input)
+        [parser.parse(buffer), nil]
+      rescue ::Parser::SyntaxError => e
+        diagnostic = diagnostics.last
+        [nil, { 'message' => e.message, 'line' => diagnostic&.location&.line || 1 }]
+      end
+
+      def kind(node)
+        return KINDS[node.type] if KINDS.key?(node.type)
+
+        case node.type
+        when :casgn then value_constructor(node) ? :value_class : :other
+        when :block, :numblock then dynamic_scope?(node) ? :dynamic_scope : :other
+        else :other
+        end
+      end
+
+      def constant_name(node)
+        return unless node
+        return '' if node.type == :cbase
+        return unless node.type == :const
+
+        parent, name = node.children
+        prefix = parent ? constant_name(parent) : nil
+        return if parent && !prefix
+
+        [prefix, name.to_s].compact.join('::')
+      end
+
+      def children(node)
+        nodes = case node.type
+                when :casgn then [node.children[2]]
+                when :masgn then [node.children[1]]
+                when :defs then node.children.drop(2)
+                else node.children
+                end
+        nodes.grep(::Parser::AST::Node)
+      end
+
+      def value_constructor(node)
+        call = node.children[2]
+        return unless call
+
+        call = call.children.first if %i[block numblock].include?(call.type)
+        return unless call.type == :send
+
+        receiver, method = call.children
+        name = constant_name(receiver)
+        name if { new: %w[Struct ::Struct], define: %w[Data ::Data] }.fetch(method, []).include?(name)
+      end
+
+      def assignment_name(node)
+        parent, name = node.children
+        prefix = parent ? constant_name(parent) : nil
+        return if parent && !prefix
+
+        [prefix, name.to_s].compact.join('::')
+      end
+
+      def declaration_name(node)
+        constant_name(node.children.first)
+      end
+
+      def declaration_kind(node)
+        node.type.to_s
+      end
+
+      def superclass(node)
+        node.children[1] if node.type == :class
+      end
+
+      def body(node)
+        node.children.last
+      end
+
+      def self_receiver?(node)
+        node.children.first.type == :self
+      end
+
+      def local_method?(node)
+        node.type == :def || node.children.first.type == :self
+      end
+
+      def line(node)
+        node.location.expression.line
+      end
+
+      def end_line(node)
+        node.location.expression.last_line
+      end
+
+      private
+
+      def dynamic_scope?(node)
+        call = node.children.first
+        return false unless %i[send csend].include?(call.type)
+
+        receiver, method = call.children
+        return true if DYNAMIC_BLOCK_METHODS.include?(method)
+
+        foreign_method?(method, receiver) || anonymous_class?(method, receiver)
+      end
+
+      def foreign_method?(method, receiver)
+        %i[define_method define_singleton_method].include?(method) && receiver && receiver.type != :self
+      end
+
+      def anonymous_class?(method, receiver)
+        name = constant_name(receiver)&.delete_prefix('::')
+        { new: %w[Class Module Struct], define: ['Data'] }.fetch(method, []).include?(name)
+      end
+    end
+  end
+end

data/lib/woods/source_references/pass.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+require 'set'
+require 'woods'
+require_relative 'collector'
+require_relative 'registry'
+require_relative 'inputs'
+require_relative 'cache'
+
+module Woods
+  module SourceReferences
+    # Retained references cannot be reconstructed safely from current bytes when
+    # their extraction provenance or edge ownership is unavailable.
+    class RebuildRequired < Woods::ExtractionError; end
+
+    # Resolves candidates against the complete current typed registry. Parsing is
+    # cached by captured HMAC, including candidates whose targets do not yet exist.
+    # Runtime lookup is repeated every run: source text alone cannot fingerprint
+    # changed ancestors, aliases or lexical shadows in a reloaded application.
+    class Pass
+      CALLER_TYPES = %w[model controller service poro lib concern].freeze
+      Result = Struct.new(:dependencies, :cache, :paths, keyword_init: true)
+
+      # @param root [String, Pathname] application root
+      # @param session [SourceInputs::Session] original source capture
+      # @param units [Array<Hash>] complete string-keyed typed unit set
+      # @param extractor_keys [Hash{Symbol => Symbol}] unit type to consumer key
+      # @param baseline [Hash, nil] validated preceding generation cache
+      # @param refreshed [Array<Array<String>>] units genuinely re-extracted
+      # @param full [Boolean] all runtime units were re-extracted
+      # @param collector [Collector] source-only parser
+      # rubocop:disable Metrics/ParameterLists -- explicit per-run collaborators and provenance state
+      def initialize(root:, session:, units:, extractor_keys:, baseline: nil, refreshed: [], full: false,
+                     collector: Collector.new)
+        @root = File.expand_path(root.to_s)
+        @session = session
+        @units = units
+        @keys = extractor_keys
+        @baseline = baseline
+        @refreshed = refreshed.to_set
+        @full = full
+        @collector = collector
+      end
+
+      # rubocop:enable Metrics/ParameterLists
+
+      # @return [Result] complete dependency replacements and next cache
+      def call
+        raise_rebuild('reference cache is missing') if !@full && !@baseline
+
+        @inputs = Inputs.new(root: @root, session: @session, collector: @collector)
+        @files = @inputs.call(units: @units, baseline: @baseline, fresh: method(:fresh?), extractor_keys: @keys)
+        sources = @files.transform_values { |entry| entry['analysis'] }
+        @registry = Registry.new(units: @units, sources: sources, root: @root)
+        @previous = Array(@baseline&.fetch('owners')).to_h { |owner| [[owner['type'], owner['identifier']], owner] }
+        build_result
+      end
+
+      private
+
+      def build_result
+        dependencies = {}
+        owners = @units.filter_map do |unit|
+          key = [unit['type'], unit['identifier']]
+          path = @inputs.relative(unit['file_path'])
+          additions = []
+          if CALLER_TYPES.include?(unit['type'])
+            base = original_dependencies(unit, key)
+            additions = all_resolved_dependencies(unit) - base
+            dependencies[key] = base + additions
+          end
+          # Target-only units also crossed Inputs' consumption gate. Retain
+          # their exact ownership so bounded freshness can reuse that proof.
+          owner_record(unit, path, additions)
+        end
+        owners.sort_by! { |owner| [owner['type'], owner['identifier']] }
+        cache = { 'version' => Cache::VERSION, 'files' => @files, 'owners' => owners }
+        Result.new(dependencies: dependencies, cache: cache, paths: @files.keys)
+      end
+
+      def owner_record(unit, path, additions)
+        return unless Registry::TYPES.include?(unit['type']&.to_sym)
+        return unless @files.key?(path) && RuntimeLookup::CONSTANT.match?(unit['identifier'].to_s)
+
+        record = { 'type' => unit['type'], 'identifier' => unit['identifier'], 'file_path' => path,
+                   'added' => additions }
+        record['file_paths'] = reference_paths(unit) if SourceContributors.multiple?(unit)
+        record
+      end
+
+      def fresh?(unit)
+        @full || @refreshed.include?([unit['type'], unit['identifier']])
+      end
+
+      def original_dependencies(unit, key)
+        dependencies = unit.fetch('dependencies', []).map { |edge| normalize_edge(edge) }
+        return dependencies if @full || @refreshed.include?(key)
+
+        prior = ownership_record(unit, key)
+        return dependencies unless prior
+
+        prior.fetch('added').each do |edge|
+          position = dependencies.index(edge)
+          raise_rebuild("reference ownership differs for #{key.join(':')}") unless position
+
+          dependencies.delete_at(position)
+        end
+        dependencies
+      end
+
+      def ownership_record(unit, key)
+        path = @inputs.relative(unit['file_path'])
+        prior = @previous[key]
+        expected = @files.key?(path) || @baseline.fetch('files').key?(path)
+        if expected && RuntimeLookup::CONSTANT.match?(unit['identifier'].to_s) &&
+           !same_owner_paths?(prior, unit, path)
+          raise_rebuild("reference ownership is missing or incompatible for #{key.join(':')}")
+        end
+        prior
+      end
+
+      def all_resolved_dependencies(unit)
+        reference_paths(unit).flat_map { |source| resolved_dependencies(unit, source) }.uniq
+      end
+
+      def same_owner_paths?(prior, unit, path)
+        prior && prior['file_path'] == path && (prior['file_paths'] || [path]) == reference_paths(unit)
+      end
+
+      def reference_paths(unit)
+        SourceContributors.paths(unit).map { |path| @inputs.relative(path) }
+      end
+
+      def normalize_edge(edge)
+        edge.transform_keys(&:to_s).transform_values { |value| value.is_a?(Symbol) ? value.to_s : value }
+      end
+
+      def resolved_dependencies(unit, path)
+        records = @files.dig(path, 'analysis', 'references') || []
+        owned = records.select { |reference| reference['owner'] == unit['identifier'] }
+        edges = owned.filter_map { |reference| @registry.resolve(reference, file_path: path) }
+        edges.map! { |edge| normalize_edge(edge) }
+        edges.uniq.sort_by { |edge| [edge['type'], edge['target'], edge['via']] }
+      end
+
+      def raise_rebuild(reason)
+        raise RebuildRequired, "Source-reference baseline needs a full extraction (#{reason}). " \
+                               'Run bin/rails woods:extract; the previous published generation remains active.'
+      end
+    end
+  end
+end