woods 2.0.1 → 2.1.0

data/lib/woods/source_inputs/scanner.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'find'
 require 'openssl'
 require 'woods/source_inputs/private_key'
 require 'woods/source_inputs/scopes'
@@ -17,8 +16,8 @@ module Woods
       attr_reader :root, :scopes
 
       def initialize(root:, output_dir:, key:, scopes: Scopes.new, **limits)
-        @root = File.expand_path(root.to_s)
-        @output_dir = File.expand_path(output_dir.to_s)
+        @root = SourcePathEncoding.expand(root)
+        @output_dir = SourcePathEncoding.expand(output_dir)
         @key = key
         @scopes = scopes
         @limits = DEFAULT_LIMITS.merge(limits)
@@ -28,13 +27,16 @@ module Woods
       end
 
       def call
+        @captured_at = Time.now.to_f
         @started = monotonic
         @files = {}
         @scope_paths = {}
         @errors = []
         @bytes = 0
         @visited = 0
+        @directory_links = {}
         walk
+        verify_directory_links
         result
       rescue BudgetExceeded => e
         error(e.message)
@@ -42,6 +44,9 @@ module Woods
       rescue SystemCallError, IOError
         error('source_tree_unavailable')
         result
+      rescue EncodingError, SourcePathEncoding::Invalid
+        error('undecodable_source_path')
+        result
       end
 
       private
@@ -49,25 +54,91 @@ module Woods
       def walk
         raise Errno::ENOENT, @root unless File.directory?(@root)
 
-        @resolved_root = File.realpath(@root)
-        Find.find(@root, ignore_error: false) do |path|
-          next if path == @root
+        @resolved_root = SourcePathEncoding.utf8!(File.realpath(@root))
+        @resolved_output = File.directory?(@output_dir) ? resolved_path(@output_dir) : nil
+        pending = children(@root).map { |path| [path, [@resolved_root]] }
+        until pending.empty?
+          path, ancestors = pending.pop
+          scan_entry(path, pending, ancestors)
+        end
+      end
+
+      def scan_entry(path, pending, ancestors)
+        check_budget!
+        path = decoded_entry(path)
+        return unless path
 
+        relative = path.delete_prefix("#{@root}/")
+        return if ignored_directory?(path, relative)
+
+        stat = File.lstat(path)
+        if directory_entry?(path, stat)
+          @visited += 1 if stat.symlink?
           check_budget!
-          relative = path.delete_prefix("#{@root}/")
-          if ignored_directory?(path, relative)
-            Find.prune if File.directory?(path)
-            next
-          end
-          stat = File.lstat(path)
-          next if stat.directory?
+          descend(path, relative, pending, ancestors, symlink: stat.symlink?)
+          return
+        end
 
-          @visited += 1
+        @visited += 1
+        check_budget!
+        visit(path, relative)
+      rescue SystemCallError, IOError
+        error('source_tree_unavailable', relative)
+      end
+
+      def directory_entry?(path, stat)
+        stat.directory? || (stat.symlink? && File.directory?(path))
+      end
+
+      # Scope each logical descendant independently. Directory names do not
+      # establish whether their children are inputs (package.yml can occur
+      # outside app/lib). External entries may be enumerated, but visit refuses
+      # any consumed file outside the root before opening its contents.
+      def descend(path, relative, pending, ancestors, symlink:)
+        real = SourcePathEncoding.utf8!(File.realpath(path))
+        return if @resolved_output && (real == @resolved_output || real.start_with?("#{@resolved_output}/"))
+
+        if ancestors.include?(real)
+          error('source_directory_cycle', relative)
+          return
+        end
+
+        @directory_links[path] = real if symlink
+        branch = ancestors + [real]
+        pending.concat(children(path).map { |child| [child, branch] })
+      end
+
+      def verify_directory_links
+        @directory_links.each do |path, target|
           check_budget!
-          visit(path, relative, stat)
+          next if File.symlink?(path) && File.directory?(path) && resolved_path(path) == target
+
+          error('source_changed_during_read', path.delete_prefix("#{@root}/"))
+        rescue SystemCallError, IOError
+          error('source_changed_during_read', path.delete_prefix("#{@root}/"))
         end
       end
 
+      # An unreadable entry cannot certify completeness, but must not hide
+      # accessible siblings. Keep traversal order deterministic and bounded.
+      def children(path)
+        Dir.children(path, encoding: Encoding::BINARY).sort.reverse.map { |name| File.join(path.b, name) }
+      rescue SystemCallError, IOError
+        error('source_tree_unavailable', path == @root ? nil : path.delete_prefix("#{@root}/"))
+        []
+      end
+
+      def decoded_entry(path)
+        decoded = SourcePathEncoding.utf8(path)
+        return decoded if decoded
+
+        @visited += 1
+        check_budget!
+        relative = path.b.delete_prefix("#{@root}/".b)
+        error('undecodable_source_path', SourcePathEncoding.diagnostic(relative))
+        nil
+      end
+
       def ignored_directory?(path, relative)
         return true if path == @output_dir || path.start_with?("#{@output_dir}/")
         return false if declared_path?(relative)
@@ -82,17 +153,12 @@ module Woods
         end
       end
 
-      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-      # Refuse opaque directories, external paths and unreadable inputs separately.
-      def visit(path, relative, stat)
-        if stat.symlink? && File.directory?(path)
-          error('unverified_symlink_directory', relative)
-          return
-        end
+      # Refuse external paths and unreadable inputs without hashing their bytes.
+      def visit(path, relative)
         consumers = @scopes.for_path(relative)
         return if consumers.empty?
 
-        real = File.realpath(path)
+        real = SourcePathEncoding.utf8!(File.realpath(path))
         unless real.start_with?("#{@resolved_root}/")
           error('external_source_path', relative)
           return
@@ -104,16 +170,16 @@ module Woods
         consumers.each { |scope| (@scope_paths[scope] ||= []) << relative }
       rescue SystemCallError, IOError
         error('source_file_unreadable', relative)
+      rescue EncodingError, SourcePathEncoding::Invalid
+        error('undecodable_source_path', SourcePathEncoding.diagnostic(relative))
       end
 
-      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
-
       def read_identity(path, resolved, expected) # rubocop:disable Metrics/CyclomaticComplexity -- stable descriptor and original resolution checks
         flags = File::RDONLY | File::NONBLOCK
         flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
         File.open(resolved, flags) do |file|
           before = stamp(file.stat)
-          unless before == expected && File.realpath(path) == resolved
+          unless before == expected && resolved_path(path) == resolved
             error('source_changed_during_read', path.delete_prefix("#{@root}/"))
             return nil
           end
@@ -122,7 +188,7 @@ module Woods
             return nil
           end
           identity = hash_stream(file)
-          unless before == stamp(file.stat) && before == stamp(File.stat(resolved)) && File.realpath(path) == resolved
+          unless before == stamp(file.stat) && before == stamp(File.stat(resolved)) && resolved_path(path) == resolved
             error('source_changed_during_read', path.delete_prefix("#{@root}/"))
             return nil
           end
@@ -130,6 +196,10 @@ module Woods
         end
       end
 
+      def resolved_path(path)
+        SourcePathEncoding.utf8!(File.realpath(path))
+      end
+
       def hash_stream(file)
         digest = OpenSSL::HMAC.new(@key.bytes, OpenSSL::Digest.new('SHA256'))
         while (chunk = file.read(16_384))
@@ -156,6 +226,7 @@ module Woods
 
       def result
         { 'root' => @root, 'key_id' => @key.identifier, 'rules' => @scopes.fingerprint,
+          'captured_at' => @captured_at,
           'extra_roots' => @scopes.extra_roots, 'files' => @files.sort.to_h,
           'scope_paths' => @scope_paths.transform_values(&:sort).sort.to_h,
           'errors' => @errors, 'complete' => @errors.empty?,

data/lib/woods/source_inputs/scopes.rb

@@ -50,7 +50,7 @@ module Woods
                 PathDispatcher.runtime_rules.map(&:to_h),
                 ReloadPolicy.constants(false).sort.to_h { |name| [name, ReloadPolicy.const_get(name)] },
                 Watch::Watcher::DEFAULT_IGNORED_DIRECTORIES, Watch::TreeScan::NOT_IGNORED_DOTFILES,
-                Watch::TreeScan::NOT_IGNORED_DOTFILE_PREFIXES, 'bounded_no_symlink_dirs_v1',
+                Watch::TreeScan::NOT_IGNORED_DOTFILE_PREFIXES, 'bounded_logical_directory_links_v2',
                 'config_ruby_and_root_gemspecs_v1', BOOT_FILES, @extra_roots]
         Digest::SHA256.hexdigest(JSON.generate(data))
       end

data/lib/woods/source_inputs/session.rb

@@ -1,22 +1,28 @@
 # frozen_string_literal: true
 
+require_relative '../source_contributors'
+
 require 'set'
 require 'woods/source_inputs/scanner'
 require 'woods/source_inputs/manifest'
 require 'woods/source_inputs/handoff'
 require 'woods/source_inputs/consumer_errors'
+require 'woods/source_inputs/stable_reader'
 
 module Woods
   module SourceInputs
     # Records successful consumers, never merely paths found by a final scan.
     # An incremental run starts from the preceding generation's scope ledger.
     class Session # rubocop:disable Metrics/ClassLength -- per-run consumer ledger and publication coverage
+      MAX_ERRORS = 30
+
       def initialize(root:, output_dir:, baseline_path:, operation:)
-        @root = File.expand_path(root.to_s)
-        @output = File.expand_path(output_dir.to_s)
+        @root = SourcePathEncoding.expand(root)
+        @output = SourcePathEncoding.expand(output_dir)
         @operation = operation.to_s
         @errors = []
         @unverified = []
+        @capture_unavailable = Handoff.unavailable_evidence
         @baseline = read_baseline(baseline_path)
         @rules = Scopes.new(extra_roots: declared_roots)
         @key = PrivateKey.new(output_dir: @output, create: true)
@@ -31,6 +37,48 @@ module Woods
         @verified_boot = false
       end
 
+      # Read original captured bytes, without certifying runtime consumption.
+      #
+      # @param path [String] root-relative or absolute application source path
+      # @param limits [Hash] StableReader byte/time budget overrides
+      # @return [Hash] original source bytes, relative path and captured HMAC
+      # @raise [StableReader::Error] when source cannot be read as captured
+      def read_source(path, **limits)
+        raise StableReader::Error, 'source_identity_unavailable' unless @key
+
+        identity = source_identity(path)
+        raise StableReader::Error, 'uncaptured_source_path' unless identity
+
+        StableReader.new(root: @root, key: @key).read(path, identity: identity, **limits)
+      end
+
+      # @param path [String] root-relative or absolute application source path
+      # @return [String, nil] captured private HMAC, never a public raw content hash
+      def source_identity(path)
+        @snapshot.fetch('files')[relative_path(path)]&.dup&.freeze
+      end
+
+      # A fresh read cannot establish which bytes produced retained runtime facts.
+      # This checks only explicit unit consumption against this session's capture.
+      #
+      # @param key [String, Symbol] extractor ledger key
+      # @param path [String] root-relative or absolute application source path
+      # @return [Boolean] whether that unit consumer acknowledged captured bytes
+      def consumed_source?(key, path)
+        identity = source_identity(path)
+        !identity.nil? && @identities.fetch("unit:#{key}", {})[relative_path(path)] == identity
+      end
+
+      # An oversized baseline can omit the comparison ledger while retaining a
+      # cryptographic binding to consumed reference evidence in the same payload.
+      # The caller must still prove exact typed ownership and unchanged HMACs.
+      def trusted_reference_baseline?(cache)
+        return false unless cache && compatible_baseline? && @baseline.unavailable?
+
+        expected = @baseline.data['reference_cache_sha256']
+        expected && expected == Digest::SHA256.hexdigest(JSON.generate(cache))
+      end
+
       def consume_file(key, path)
         consume_path("file:#{key}", path)
         consume_path("unit:#{key}", path) if @identities.fetch("unit:#{key}", {}).key?(relative_path(path))
@@ -51,7 +99,7 @@ module Woods
       def consume_extractor(key, units)
         ["file:#{key}", "whole:#{key}"].each { |scope| replace_scope(scope) }
         @identities["unit:#{key}"] = {}
-        Array(units).each { |unit| consume_unit(key, unit.file_path) }
+        Array(units).each { |unit| SourceContributors.paths(unit).each { |path| consume_unit(key, path) } }
       end
 
       def unverified(scope)
@@ -62,6 +110,16 @@ module Woods
         results.each do |key, units|
           if ConsumerErrors.failed?(consumers[key])
             unverified("extractor:#{key}")
+            # ModelExtractor's per-model rescue returns nil. Every returned
+            # model crossed that boundary successfully, even if a sibling did
+            # not. Other extractors may return partial results after a rescue.
+            if key == :models
+              Array(units).each do |unit|
+                SourceContributors.paths(unit).each do |path|
+                  consume_unit(key, path)
+                end
+              end
+            end
           else
             consume_extractor(key, units)
           end
@@ -80,11 +138,28 @@ module Woods
         end
         @unverified << 'incomplete_eager_load' unless eager_load_complete
         Manifest.build(snapshot: @snapshot, scopes: @identities, boot_verified: verified_coverage?,
-                       generation: generation, errors: @errors, unverified_scopes: @unverified)
+                       generation: generation, errors: @errors, unverified_scopes: @unverified,
+                       comparison_complete: comparison_complete?,
+                       unavailable: unavailable_evidence)
       end
 
       private
 
+      def record_error(error)
+        # Bound path samples independently so noisy unit diagnostics cannot
+        # hide a later instability reason or its watcher catch-up paths.
+        count = @errors.count { |existing| existing['reason'] == error['reason'] }
+        @errors << error if count < MAX_ERRORS && !@errors.include?(error)
+      end
+
+      def unavailable_evidence
+        @capture_unavailable || (@baseline.data['unavailable'] if @operation != 'full' && @baseline&.unavailable?)
+      end
+
+      def comparison_complete?
+        @operation == 'full' || !!(compatible_baseline? && @baseline.comparison_complete?)
+      end
+
       def declared_roots
         roots = Handoff.extra_roots
         roots.empty? && @baseline ? @baseline.data.fetch('extra_roots') : roots
@@ -99,7 +174,7 @@ module Woods
           Manifest.parse(file.read(Manifest::MAX_BYTES + 1))
         end
       rescue Manifest::Invalid, SystemCallError, IOError
-        @errors << { 'reason' => 'invalid_source_baseline' } unless @operation == 'full'
+        record_error('reason' => 'invalid_source_baseline') unless @operation == 'full'
         nil
       end
 
@@ -107,7 +182,7 @@ module Woods
         return snapshot_identities if @operation == 'full'
         return @baseline.expanded if compatible_baseline?
 
-        @errors << { 'reason' => 'missing_or_incompatible_source_baseline' }
+        record_error('reason' => 'missing_or_incompatible_source_baseline')
         {}
       end
 
@@ -130,7 +205,7 @@ module Woods
         if identity
           (@identities[scope] ||= {})[relative] = identity
         elsif File.exist?(File.join(@root, relative))
-          @errors << { 'reason' => 'uncaptured_source_path', 'path' => relative }
+          record_error('reason' => 'uncaptured_source_path', 'path' => relative)
         else
           @identities[scope]&.delete(relative)
         end
@@ -139,11 +214,60 @@ module Woods
       def relative_path(path)
         return nil if path.nil? || path.to_s.empty?
 
-        absolute = File.expand_path(path.to_s, @root)
+        decoded = SourcePathEncoding.utf8(path)
+        unless decoded
+          record_error('reason' => 'undecodable_source_path', 'path' => SourcePathEncoding.diagnostic(path))
+          return nil
+        end
+
+        application_source_path(File.expand_path(decoded, @root))
+      end
+
+      def application_source_path(absolute)
         # Installed gem/framework source is outside application-source coverage.
         return nil unless absolute.start_with?("#{@root}/")
 
-        absolute.delete_prefix("#{@root}/")
+        relative = absolute.delete_prefix("#{@root}/")
+        return relative if @rules.extra_roots.any? { |root| relative.start_with?("#{root}/") }
+        return nil if installed_gem_roots.any? { |root| absolute.start_with?("#{root}/") }
+
+        relative
+      end
+
+      # A vendor-shaped path alone is not ownership evidence. Local path gems
+      # have their gemspec in the checkout, not in an installation's metadata.
+      def installed_gem_roots
+        @installed_gem_roots ||= Gem.loaded_specs.values.filter_map { |spec| installed_gem_root(spec) }
+      end
+
+      def installed_gem_root(spec)
+        return unless spec.loaded_from && File.file?(spec.loaded_from)
+
+        base = File.expand_path(spec.base_dir)
+        path = File.expand_path(spec.full_gem_path)
+        return path if installed_git_gem?(spec, path)
+        return unless File.expand_path(spec.loaded_from).start_with?("#{base}/specifications/") &&
+                      path.start_with?("#{base}/gems/") && File.directory?(path)
+
+        path
+      end
+
+      # Bundler git installations keep their gemspec beside the source rather
+      # than in RubyGems' specifications directory. A local override remains
+      # application source even when it retains a Git source object.
+      def installed_git_gem?(spec, path)
+        return false unless defined?(Bundler::Source::Git) && spec.respond_to?(:source)
+
+        source = spec.source
+        return false unless source.is_a?(Bundler::Source::Git) && !source.local?
+
+        installed = File.expand_path(source.path.to_s)
+        within_installed_source?(path, installed) &&
+          File.expand_path(spec.loaded_from).start_with?("#{path}/") && File.directory?(path)
+      end
+
+      def within_installed_source?(path, installed)
+        path == installed || path.start_with?("#{installed}/")
       end
 
       def replace_scope(scope)
@@ -168,20 +292,28 @@ module Woods
       def preserve_baseline_coverage
         return unless @baseline
 
-        @errors.concat(@baseline.data.fetch('errors'))
+        @baseline.data.fetch('errors').each { |error| record_error(error) }
         @unverified.concat(@baseline.data.fetch('unverified_scopes'))
-        @errors << { 'reason' => 'incomplete_source_baseline' } unless @baseline.data['complete']
+        record_error('reason' => 'incomplete_source_baseline') unless @baseline.data['complete']
       end
 
       def verify_stability
         return unless @key
 
         after = scan
-        @errors.concat(after.fetch('errors'))
+        after.fetch('errors').each { |error| record_error(error) }
+        changed_paths(after).first(MAX_ERRORS).each do |path|
+          record_error('reason' => 'source_changed_during_extraction', 'path' => path)
+        end
+      end
+
+      def changed_paths(after)
         before_files = @snapshot.fetch('files')
         after_files = after.fetch('files')
-        changed = (before_files.keys | after_files.keys).reject { |path| before_files[path] == after_files[path] }
-        changed.first(30).each { |path| @errors << { 'reason' => 'source_changed_during_extraction', 'path' => path } }
+        changed = (before_files.keys & after_files.keys).reject { |path| before_files[path] == after_files[path] }
+        changed.concat(after_files.keys - before_files.keys) if @snapshot['complete']
+        changed.concat(before_files.keys - after_files.keys) if after['complete']
+        changed
       end
 
       # A custom loader can consume Ruby outside the standard dispatch roots.
@@ -196,7 +328,7 @@ module Woods
         end.uniq
         missing = paths - @snapshot.fetch('files').keys
         missing.first(30).each do |path|
-          @errors << { 'reason' => 'loaded_source_outside_coverage', 'path' => path }
+          record_error('reason' => 'loaded_source_outside_coverage', 'path' => path)
         end
       end
 

data/lib/woods/source_inputs/stable_reader.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require 'openssl'
+
+module Woods
+  module SourceInputs
+    # Reads bytes only when their descriptor and captured keyed identity agree.
+    # Paths may be captured internal file symlinks, but their resolution must
+    # remain stable and inside the real application root throughout the read.
+    class StableReader
+      DEFAULT_LIMITS = { max_bytes: 8 * 1024 * 1024, max_seconds: 2.0 }.freeze
+
+      class Error < StandardError
+        attr_reader :reason
+
+        def initialize(reason)
+          @reason = reason
+          super
+        end
+      end
+
+      # @param root [String] application root
+      # @param key [PrivateKey] private identity key used by the source snapshot
+      def initialize(root:, key:)
+        @root = File.expand_path(root.to_s)
+        @key = key
+      end
+
+      # Read original bytes without advancing any source-consumption ledger.
+      # The time budget is cooperative between bounded regular-file reads.
+      #
+      # @param path [String] root-relative or absolute application path
+      # @param identity [String] HMAC from the session snapshot
+      # @param max_bytes [Numeric] maximum bytes retained in memory
+      # @param max_seconds [Numeric] maximum elapsed read time
+      # @return [Hash] frozen path, binary source bytes and captured HMAC
+      # @raise [Error] if containment, stability, identity or bounds cannot be proven
+      def read(path, identity:, max_bytes: DEFAULT_LIMITS[:max_bytes], max_seconds: DEFAULT_LIMITS[:max_seconds])
+        @limits = { max_bytes: max_bytes, max_seconds: max_seconds }
+        validate_limits!
+        @started = monotonic
+        absolute = File.expand_path(path.to_s, @root)
+        resolved = resolve(absolute)
+        expected = stamp(File.stat(resolved))
+        source = read_stable(absolute, resolved, expected)
+        actual = OpenSSL::HMAC.hexdigest('SHA256', @key.bytes, source)
+        check_time!
+        raise Error, 'source_snapshot_mismatch' unless actual == identity
+
+        { 'path' => absolute.delete_prefix("#{@root}/").freeze, 'source' => source.freeze,
+          'identity' => identity.dup.freeze }
+      rescue SystemCallError, IOError
+        raise Error, 'source_file_unreadable'
+      end
+
+      private
+
+      def validate_limits!
+        return if @limits.values.all? { |value| value.is_a?(Numeric) && value.finite? && value.positive? }
+
+        raise ArgumentError, 'source read limits must be finite and positive'
+      end
+
+      def resolve(path)
+        raise Error, 'source_outside_root' unless path.start_with?("#{@root}/")
+
+        @resolved_root = File.realpath(@root)
+        resolved = File.realpath(path)
+        raise Error, 'source_outside_root' unless resolved.start_with?("#{@resolved_root}/")
+
+        resolved
+      end
+
+      def read_stable(path, resolved, expected)
+        flags = File::RDONLY | File::NONBLOCK
+        flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
+        File.open(resolved, flags) do |file|
+          file.binmode
+          before = stamp(file.stat)
+          raise Error, 'nonregular_source' unless file.stat.file?
+          raise Error, 'source_changed_during_read' unless before == expected && stable_path?(path, resolved)
+          raise Error, 'source_read_byte_budget' if file.stat.size > @limits[:max_bytes]
+
+          source = read_bytes(file)
+          verify_after!(file, before, path, resolved)
+          source
+        end
+      end
+
+      def verify_after!(file, before, path, resolved)
+        return if before == stamp(file.stat) && before == stamp(File.stat(resolved)) && stable_path?(path, resolved)
+
+        raise Error, 'source_changed_during_read'
+      end
+
+      def read_bytes(file)
+        source = String.new(encoding: Encoding::BINARY)
+        loop do
+          check_time!
+          chunk = file.read(16_384)
+          break unless chunk
+          raise Error, 'source_read_byte_budget' if source.bytesize + chunk.bytesize > @limits[:max_bytes]
+
+          source << chunk
+        end
+        check_time!
+        source
+      end
+
+      def stable_path?(path, resolved)
+        File.realpath(@root) == @resolved_root && File.realpath(path) == resolved
+      end
+
+      def stamp(stat)
+        [stat.dev, stat.ino, stat.size, stat.mtime.to_r, stat.ctime.to_r]
+      end
+
+      def check_time!
+        raise Error, 'source_read_time_budget' if monotonic - @started > @limits[:max_seconds]
+      end
+
+      def monotonic
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end