woods 2.0.0.beta2 → 2.0.0.beta4

data/lib/woods/source_inputs/manifest.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Woods
+  module SourceInputs
+    # Compact wire format: consumers refer to a shared identity table. Internal
+    # mutation uses expanded digests so table indices never leak across runs.
+    class Manifest
+      FILE_NAME = 'source_inputs.json'
+      VERSION = 1
+      MAX_BYTES = 16 * 1024 * 1024
+      class Invalid < StandardError; end
+
+      attr_reader :data
+
+      def self.parse(bytes)
+        raise Invalid, 'source_manifest_too_large' if bytes.bytesize > MAX_BYTES
+
+        new(JSON.parse(bytes))
+      rescue JSON::ParserError
+        raise Invalid, 'invalid_source_manifest'
+      end
+
+      def initialize(data)
+        @data = data
+        validate!
+      end
+
+      def expanded
+        identities = @data.fetch('identities')
+        @data.fetch('scopes').transform_values do |paths|
+          paths.transform_values { |index| identities.fetch(index) }
+        end
+      end
+
+      # Explicit wire fields keep incomplete coverage separate from boot provenance.
+      # rubocop:disable Metrics/AbcSize, Metrics/ParameterLists
+      def self.build(snapshot:, scopes:, boot_verified:, generation:, errors: [], unverified_scopes: [])
+        identities = scopes.values.flat_map(&:values).uniq.sort
+        indices = identities.each_with_index.to_h
+        new('version' => VERSION, 'generation' => generation, 'root' => snapshot.fetch('root'),
+            'key_id' => snapshot.fetch('key_id'), 'rules' => snapshot.fetch('rules'),
+            'extra_roots' => snapshot.fetch('extra_roots'), 'boot_verified' => boot_verified,
+            'complete' => snapshot.fetch('complete'), 'errors' => (snapshot.fetch('errors') + errors).uniq,
+            'unverified_scopes' => unverified_scopes.uniq.sort,
+            'identities' => identities, 'metrics' => snapshot.fetch('metrics'),
+            'scopes' => scopes.sort.to_h.transform_values do |paths|
+              paths.sort.to_h.transform_values { |digest| indices.fetch(digest) }
+            end)
+      end
+
+      # rubocop:enable Metrics/AbcSize, Metrics/ParameterLists
+
+      private
+
+      def validate!
+        raise Invalid, 'unsupported_source_manifest' unless supported_version?
+        raise Invalid, 'invalid_source_manifest_header' unless valid_header?
+
+        validate_tables!
+        return if @data['extra_roots'].all? { |path| valid_path?(path) }
+
+        raise Invalid, 'invalid_source_roots'
+      end
+
+      def validate_tables!
+        identities = @data['identities']
+        scopes = @data['scopes']
+        unless identities.is_a?(Array) && identities.all? { |digest| digest?(digest) } && scopes.is_a?(Hash)
+          raise Invalid, 'invalid_source_identity_table'
+        end
+
+        scopes.each { |scope, paths| validate_scope!(scope, paths, identities) }
+        raise Invalid, 'invalid_source_manifest_coverage' unless valid_coverage?
+      end
+
+      def validate_scope!(scope, paths, identities)
+        return if scope.is_a?(String) && paths.is_a?(Hash) &&
+                  paths.all? { |path, index| valid_entry?(path, index, identities) }
+
+        raise Invalid, 'invalid_source_scope'
+      end
+
+      def supported_version?
+        @data.is_a?(Hash) && @data['version'] == VERSION &&
+          @data['generation'].is_a?(Integer) && @data['generation'].positive?
+      end
+
+      def valid_header?
+        valid_root? && %w[key_id rules].all? { |key| digest?(@data[key]) } &&
+          %w[boot_verified complete].all? { |key| [true, false].include?(@data[key]) }
+      end
+
+      def valid_root?
+        root = @data['root']
+        root.is_a?(String) && root.start_with?('/') && !root.include?("\0")
+      end
+
+      def valid_coverage?
+        @data['errors'].is_a?(Array) && @data['errors'].all? { |error| valid_error?(error) } &&
+          %w[extra_roots unverified_scopes].all? { |key| @data[key].is_a?(Array) && @data[key].all?(String) }
+      end
+
+      def valid_error?(error)
+        error.is_a?(Hash) && error['reason'].is_a?(String) &&
+          (error['path'].nil? || error['path'].is_a?(String))
+      end
+
+      def valid_entry?(path, index, identities)
+        valid_path?(path) && index.is_a?(Integer) && index >= 0 && index < identities.size
+      end
+
+      def valid_path?(path)
+        path.is_a?(String) && !path.empty? && !path.start_with?('/') && !path.include?("\0") &&
+          path.split('/', -1).none? { |part| ['', '.', '..'].include?(part) }
+      end
+
+      def digest?(value)
+        value.is_a?(String) && value.match?(/\A[0-9a-f]{64}\z/)
+      end
+    end
+  end
+end

data/lib/woods/source_inputs/private_key.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+require 'digest'
+require 'fileutils'
+
+module Woods
+  module SourceInputs
+    # Never copied into a published generation. Public identities are keyed even
+    # for ordinary source, so an unrecognized secret-bearing file stays protected.
+    class PrivateKey
+      FILE_NAME = '.source-inputs.key'
+      class Unavailable < StandardError; end
+
+      attr_reader :bytes, :identifier
+
+      def initialize(output_dir:, create: false)
+        path = File.join(output_dir.to_s, FILE_NAME)
+        create_key(path) if create
+        flags = File::RDONLY | File::NONBLOCK
+        flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
+        File.open(path, flags) do |file|
+          stat = file.stat
+          raise Unavailable, 'insecure_identity_key' unless private_regular?(stat)
+
+          @bytes = file.read(33)
+        end
+        raise Unavailable, 'invalid_identity_key' unless @bytes&.bytesize == 32
+
+        @identifier = Digest::SHA256.hexdigest(@bytes)
+      rescue SystemCallError, IOError
+        raise Unavailable, 'identity_key_unavailable'
+      end
+
+      private
+
+      def private_regular?(stat)
+        stat.file? && stat.uid == Process.uid && stat.mode.nobits?(0o077)
+      end
+
+      def create_key(path)
+        FileUtils.mkdir_p(File.dirname(path))
+        File.open(path, File::WRONLY | File::CREAT | File::EXCL, 0o600) do |file|
+          file.binmode
+          file.write(SecureRandom.random_bytes(32))
+          file.flush
+          file.fsync
+        end
+      rescue Errno::EEXIST
+        # Existing permissions/content are validated, never repaired implicitly.
+        nil
+      end
+    end
+  end
+end

data/lib/woods/source_inputs/scanner.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+require 'find'
+require 'openssl'
+require 'woods/source_inputs/private_key'
+require 'woods/source_inputs/scopes'
+require 'woods/watch/watcher'
+
+module Woods
+  module SourceInputs
+    # Unlike the watcher's best-effort walk, every incomplete read is evidence
+    # against claiming complete source coverage. No Rails application is booted.
+    class Scanner # rubocop:disable Metrics/ClassLength -- one bounded traversal owns its counters and read checks
+      DEFAULT_LIMITS = { max_files: 50_000, max_bytes: 128 * 1024 * 1024, max_seconds: 10.0 }.freeze
+      class BudgetExceeded < StandardError; end
+
+      attr_reader :root, :scopes
+
+      def initialize(root:, output_dir:, key:, scopes: Scopes.new, **limits)
+        @root = File.expand_path(root.to_s)
+        @output_dir = File.expand_path(output_dir.to_s)
+        @key = key
+        @scopes = scopes
+        @limits = DEFAULT_LIMITS.merge(limits)
+        return if @limits.values.all? { |value| value.is_a?(Numeric) && value.finite? && value.positive? }
+
+        raise ArgumentError, 'source scan limits must be finite and positive'
+      end
+
+      def call
+        @started = monotonic
+        @files = {}
+        @scope_paths = {}
+        @errors = []
+        @bytes = 0
+        @visited = 0
+        walk
+        result
+      rescue BudgetExceeded => e
+        error(e.message)
+        result
+      rescue SystemCallError, IOError
+        error('source_tree_unavailable')
+        result
+      end
+
+      private
+
+      def walk
+        raise Errno::ENOENT, @root unless File.directory?(@root)
+
+        @resolved_root = File.realpath(@root)
+        Find.find(@root, ignore_error: false) do |path|
+          next if path == @root
+
+          check_budget!
+          relative = path.delete_prefix("#{@root}/")
+          if ignored_directory?(path, relative)
+            Find.prune if File.directory?(path)
+            next
+          end
+          stat = File.lstat(path)
+          next if stat.directory?
+
+          @visited += 1
+          check_budget!
+          visit(path, relative, stat)
+        end
+      end
+
+      def ignored_directory?(path, relative)
+        return true if path == @output_dir || path.start_with?("#{@output_dir}/")
+        return false if declared_path?(relative)
+
+        Watch::TreeScan.hidden?(relative) ||
+          Watch::TreeScan.ignored?(relative, Watch::Watcher::DEFAULT_IGNORED_DIRECTORIES)
+      end
+
+      def declared_path?(relative)
+        @scopes.extra_roots.any? do |root|
+          relative == root || relative.start_with?("#{root}/") || root.start_with?("#{relative}/")
+        end
+      end
+
+      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      # Refuse opaque directories, external paths and unreadable inputs separately.
+      def visit(path, relative, stat)
+        if stat.symlink? && File.directory?(path)
+          error('unverified_symlink_directory', relative)
+          return
+        end
+        consumers = @scopes.for_path(relative)
+        return if consumers.empty?
+
+        real = File.realpath(path)
+        unless real.start_with?("#{@resolved_root}/")
+          error('external_source_path', relative)
+          return
+        end
+        identity = read_identity(path, real, stamp(File.stat(real)))
+        return unless identity
+
+        @files[relative] = identity
+        consumers.each { |scope| (@scope_paths[scope] ||= []) << relative }
+      rescue SystemCallError, IOError
+        error('source_file_unreadable', relative)
+      end
+
+      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
+      def read_identity(path, resolved, expected) # rubocop:disable Metrics/CyclomaticComplexity -- stable descriptor and original resolution checks
+        flags = File::RDONLY | File::NONBLOCK
+        flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
+        File.open(resolved, flags) do |file|
+          before = stamp(file.stat)
+          unless before == expected && File.realpath(path) == resolved
+            error('source_changed_during_read', path.delete_prefix("#{@root}/"))
+            return nil
+          end
+          unless file.stat.file?
+            error('nonregular_source', path.delete_prefix("#{@root}/"))
+            return nil
+          end
+          identity = hash_stream(file)
+          unless before == stamp(file.stat) && before == stamp(File.stat(resolved)) && File.realpath(path) == resolved
+            error('source_changed_during_read', path.delete_prefix("#{@root}/"))
+            return nil
+          end
+          identity
+        end
+      end
+
+      def hash_stream(file)
+        digest = OpenSSL::HMAC.new(@key.bytes, OpenSSL::Digest.new('SHA256'))
+        while (chunk = file.read(16_384))
+          @bytes += chunk.bytesize
+          check_budget!
+          digest.update(chunk)
+        end
+        digest.hexdigest
+      end
+
+      def stamp(stat)
+        [stat.dev, stat.ino, stat.size, stat.mtime.to_r, stat.ctime.to_r]
+      end
+
+      def check_budget!
+        raise BudgetExceeded, 'scan_time_budget' if monotonic - @started > @limits[:max_seconds]
+        raise BudgetExceeded, 'scan_file_budget' if @visited > @limits[:max_files]
+        raise BudgetExceeded, 'scan_byte_budget' if @bytes > @limits[:max_bytes]
+      end
+
+      def error(reason, path = nil)
+        @errors << { 'reason' => reason, 'path' => path }.compact if @errors.size < 20
+      end
+
+      def result
+        { 'root' => @root, 'key_id' => @key.identifier, 'rules' => @scopes.fingerprint,
+          'extra_roots' => @scopes.extra_roots, 'files' => @files.sort.to_h,
+          'scope_paths' => @scope_paths.transform_values(&:sort).sort.to_h,
+          'errors' => @errors, 'complete' => @errors.empty?,
+          'metrics' => { 'visited_files' => @visited, 'hashed_files' => @files.size,
+                         'bytes' => @bytes, 'elapsed_ms' => ((monotonic - @started) * 1000).round(3) } }
+      end
+
+      def monotonic
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/woods/source_inputs/scopes.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'json'
+require 'woods/input_rules'
+require 'woods/watch/watcher'
+require 'woods/watch/tree_scan'
+
+module Woods
+  module SourceInputs
+    # Keep each consumer's provenance separate. A whole-app event scan must not
+    # bless retained service units that happen to share the same source file.
+    class Scopes
+      BOOT_FILES = %w[Rakefile config.ru].freeze
+
+      attr_reader :extra_roots
+
+      def initialize(extra_roots: [])
+        require 'woods/extractor' unless defined?(Woods::Extractor::EXTRACTORS)
+        @extra_roots = Array(extra_roots).map { |path| validate_root(path) }.uniq.sort
+        @file_rules = PathDispatcher.file_rules
+        @whole_rules = PathDispatcher.whole_app_rules
+        @policy = ReloadPolicy.new
+      end
+
+      def for_path(path) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity -- each independent consumer contributes a scope
+        scopes = []
+        action = @policy.classify(path)
+        scopes << 'boot' if boot_path?(path, action)
+        scopes << 'runtime' if runtime_path?(path, action)
+        @file_rules.each { |rule| scopes << "file:#{rule.extractor_key}" if rule.matches?(path) }
+        @whole_rules.each { |rule| scopes << "whole:#{rule.extractor_key}" if rule.matches?(path) }
+        scopes << 'declared' if @extra_roots.any? { |dir| path.start_with?("#{dir}/") }
+        scopes.uniq.sort
+      end
+
+      def boot_path?(path, action)
+        return true if action == :restart || BOOT_FILES.include?(path)
+        return false unless action == :ignore
+
+        path.match?(%r{\Aconfig/.+\.rb\z}) || path.match?(%r{\A[^/]+\.gemspec\z})
+      end
+
+      def runtime_path?(path, action)
+        action == :reload && path.end_with?('.rb') && path.match?(%r{\A(?:app|lib)/})
+      end
+
+      def fingerprint
+        data = [@file_rules.map(&:to_h), @whole_rules.map(&:to_h),
+                PathDispatcher.runtime_rules.map(&:to_h),
+                ReloadPolicy.constants(false).sort.to_h { |name| [name, ReloadPolicy.const_get(name)] },
+                Watch::Watcher::DEFAULT_IGNORED_DIRECTORIES, Watch::TreeScan::NOT_IGNORED_DOTFILES,
+                Watch::TreeScan::NOT_IGNORED_DOTFILE_PREFIXES, 'bounded_no_symlink_dirs_v1',
+                'config_ruby_and_root_gemspecs_v1', BOOT_FILES, @extra_roots]
+        Digest::SHA256.hexdigest(JSON.generate(data))
+      end
+
+      private
+
+      def validate_root(value)
+        path = value.to_s.delete_suffix('/')
+        if path.empty? || path.start_with?('/') || path.include?("\0") ||
+           path.split('/', -1).any? { |part| ['', '.', '..'].include?(part) }
+          raise ArgumentError, 'source roots must be relative directories under the application root'
+        end
+
+        path
+      end
+    end
+  end
+end

data/lib/woods/source_inputs/session.rb

@@ -0,0 +1,214 @@
+# frozen_string_literal: true
+
+require 'set'
+require 'woods/source_inputs/scanner'
+require 'woods/source_inputs/manifest'
+require 'woods/source_inputs/handoff'
+require 'woods/source_inputs/consumer_errors'
+
+module Woods
+  module SourceInputs
+    # Records successful consumers, never merely paths found by a final scan.
+    # An incremental run starts from the preceding generation's scope ledger.
+    class Session # rubocop:disable Metrics/ClassLength -- per-run consumer ledger and publication coverage
+      def initialize(root:, output_dir:, baseline_path:, operation:)
+        @root = File.expand_path(root.to_s)
+        @output = File.expand_path(output_dir.to_s)
+        @operation = operation.to_s
+        @errors = []
+        @unverified = []
+        @baseline = read_baseline(baseline_path)
+        @rules = Scopes.new(extra_roots: declared_roots)
+        @key = PrivateKey.new(output_dir: @output, create: true)
+        handoff = Handoff.read(root: @root, output_dir: @output, operation: @operation, rules: @rules.fingerprint,
+                               key_id: @key.identifier)
+        @snapshot = handoff || scan
+        @verified_boot = !handoff.nil?
+        @identities = initial_identities
+      rescue PrivateKey::Unavailable => e
+        @snapshot = unavailable_snapshot(e.message)
+        @identities = {}
+        @verified_boot = false
+      end
+
+      def consume_file(key, path)
+        consume_path("file:#{key}", path)
+        consume_path("unit:#{key}", path) if @identities.fetch("unit:#{key}", {}).key?(relative_path(path))
+      end
+
+      def consume_unit(key, path)
+        consume_path("unit:#{key}", path)
+      end
+
+      def consume_deleted(path) # rubocop:disable Metrics/CyclomaticComplexity -- deletion preserves boot provenance
+        relative = relative_path(path)
+        return if relative.nil? || @snapshot.fetch('files').key?(relative)
+
+        @rules.for_path(relative).each { |scope| @identities[scope]&.delete(relative) unless scope == 'boot' }
+        @identities.each { |scope, paths| paths.delete(relative) if scope.start_with?('unit:') }
+      end
+
+      def consume_extractor(key, units)
+        ["file:#{key}", "whole:#{key}"].each { |scope| replace_scope(scope) }
+        @identities["unit:#{key}"] = {}
+        Array(units).each { |unit| consume_unit(key, unit.file_path) }
+      end
+
+      def unverified(scope)
+        @unverified << scope.to_s
+      end
+
+      def full_units(results, consumers: {})
+        results.each do |key, units|
+          if ConsumerErrors.failed?(consumers[key])
+            unverified("extractor:#{key}")
+          else
+            consume_extractor(key, units)
+          end
+        end
+      end
+
+      def finish(generation:, eager_load_complete:)
+        verify_stability
+        verify_loaded_source_coverage
+        if @operation != 'full'
+          preserve_baseline_coverage
+          # Fresh discovery is not proof that every retained runtime fact was
+          # serialized again. Keep this limitation explicit even when bytes match.
+          @unverified << 'runtime_consumption' if runtime_inputs_changed?
+          replace_scope('runtime') if @verified_boot && eager_load_complete
+        end
+        @unverified << 'incomplete_eager_load' unless eager_load_complete
+        Manifest.build(snapshot: @snapshot, scopes: @identities, boot_verified: verified_coverage?,
+                       generation: generation, errors: @errors, unverified_scopes: @unverified)
+      end
+
+      private
+
+      def declared_roots
+        roots = Handoff.extra_roots
+        roots.empty? && @baseline ? @baseline.data.fetch('extra_roots') : roots
+      end
+
+      def read_baseline(path)
+        return nil if path.nil? || !File.file?(path)
+
+        File.open(path, File::RDONLY | File::NONBLOCK) do |file|
+          raise Manifest::Invalid, 'source_manifest_too_large' if file.stat.size > Manifest::MAX_BYTES
+
+          Manifest.parse(file.read(Manifest::MAX_BYTES + 1))
+        end
+      rescue Manifest::Invalid, SystemCallError, IOError
+        @errors << { 'reason' => 'invalid_source_baseline' } unless @operation == 'full'
+        nil
+      end
+
+      def initial_identities
+        return snapshot_identities if @operation == 'full'
+        return @baseline.expanded if compatible_baseline?
+
+        @errors << { 'reason' => 'missing_or_incompatible_source_baseline' }
+        {}
+      end
+
+      def compatible_baseline?
+        @baseline && @baseline.data['key_id'] == @snapshot['key_id'] &&
+          @baseline.data['rules'] == @snapshot['rules'] && @baseline.data['root'] == @root
+      end
+
+      def snapshot_identities
+        @snapshot.fetch('scope_paths').transform_values do |paths|
+          paths.to_h { |path| [path, @snapshot.fetch('files').fetch(path)] }
+        end
+      end
+
+      def consume_path(scope, path)
+        relative = relative_path(path)
+        return if relative.nil?
+
+        identity = @snapshot.fetch('files')[relative]
+        if identity
+          (@identities[scope] ||= {})[relative] = identity
+        elsif File.exist?(File.join(@root, relative))
+          @errors << { 'reason' => 'uncaptured_source_path', 'path' => relative }
+        else
+          @identities[scope]&.delete(relative)
+        end
+      end
+
+      def relative_path(path)
+        return nil if path.nil? || path.to_s.empty?
+
+        absolute = File.expand_path(path.to_s, @root)
+        # Installed gem/framework source is outside application-source coverage.
+        return nil unless absolute.start_with?("#{@root}/")
+
+        absolute.delete_prefix("#{@root}/")
+      end
+
+      def replace_scope(scope)
+        paths = @snapshot.fetch('scope_paths').fetch(scope, [])
+        @identities[scope] = paths.to_h { |path| [path, @snapshot.fetch('files').fetch(path)] }
+      end
+
+      def runtime_inputs_changed?
+        return true unless compatible_baseline?
+
+        before = @baseline.expanded.fetch('runtime', {})
+        now = snapshot_identities.fetch('runtime', {})
+        before != now
+      end
+
+      def verified_coverage?
+        return @verified_boot if @operation == 'full'
+
+        !!(@verified_boot && compatible_baseline? && @baseline.data['boot_verified'])
+      end
+
+      def preserve_baseline_coverage
+        return unless @baseline
+
+        @errors.concat(@baseline.data.fetch('errors'))
+        @unverified.concat(@baseline.data.fetch('unverified_scopes'))
+        @errors << { 'reason' => 'incomplete_source_baseline' } unless @baseline.data['complete']
+      end
+
+      def verify_stability
+        return unless @key
+
+        after = scan
+        @errors.concat(after.fetch('errors'))
+        before_files = @snapshot.fetch('files')
+        after_files = after.fetch('files')
+        changed = (before_files.keys | after_files.keys).reject { |path| before_files[path] == after_files[path] }
+        changed.first(30).each { |path| @errors << { 'reason' => 'source_changed_during_extraction', 'path' => path } }
+      end
+
+      # A custom loader can consume Ruby outside the standard dispatch roots.
+      # Loaded-feature evidence can disprove coverage, never expand a capture
+      # after boot and retroactively claim those bytes were consumed.
+      def verify_loaded_source_coverage
+        paths = $LOADED_FEATURES.filter_map do |path|
+          # Ruby 3 also lists built-in pseudo features, such as thread.rb.
+          next unless path.start_with?('/') || File.file?(File.expand_path(path, @root))
+
+          relative_path(path)
+        end.uniq
+        missing = paths - @snapshot.fetch('files').keys
+        missing.first(30).each do |path|
+          @errors << { 'reason' => 'loaded_source_outside_coverage', 'path' => path }
+        end
+      end
+
+      def scan
+        Scanner.new(root: @root, output_dir: @output, key: @key, scopes: @rules).call
+      end
+
+      def unavailable_snapshot(reason)
+        { 'root' => @root, 'key_id' => '0' * 64, 'rules' => @rules.fingerprint,
+          'extra_roots' => @rules.extra_roots, 'files' => {}, 'scope_paths' => {}, 'complete' => false,
+          'errors' => [{ 'reason' => reason }], 'metrics' => {} }
+      end
+    end
+  end
+end