rubylens 0.1.0 → 0.2.1

data/lib/rubylens/index/git_package_source.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+require "bundler"
+require "pathname"
+require "set"
+
+module RubyLens
+  module Index
+    # Resolves a git-sourced lockfile entry into the exact files RubyLens may index.
+    #
+    # Git checkouts carry more authority than RubyGems packages: the gemspec is
+    # code from the checkout that names its own require paths, and the tree may
+    # hold symlinks pointing anywhere on disk. Every path here is therefore
+    # resolved and re-checked against the canonical package root before it
+    # becomes indexable, and any breach ends the package rather than narrowing
+    # it, so a partially trusted tree can never contribute files.
+    class GitPackageSource
+      UnsafeRequirePath = Class.new(StandardError)
+      UnsafePackageFile = Class.new(StandardError)
+
+      # Logical roots come from the gemspec and may be symlinked; the canonical
+      # root is the resolved directory every indexable file must sit inside.
+      PackagePaths = Data.define(
+        :logical_root, #: Pathname
+        :canonical_root, #: Pathname
+        :logical_canonical_root, #: Pathname
+      )
+      Resolution = Data.define(
+        :root, #: Pathname
+        :files, #: Array[String]
+      )
+
+      # Nix stores gems in immutable, content-addressed store objects, so a
+      # checkout symlinked into one is trustworthy even though it resolves
+      # outside the bundle directory.
+      class NixStoreProvider
+        STORE_ROOT = Pathname("/nix/store").freeze
+        STORE_OBJECT_PATTERN = /\A[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+\z/
+
+        #: (Pathname | String) -> bool
+        def trusted?(path)
+          path = Pathname(path).expand_path
+          return false unless Paths.inside?(path, STORE_ROOT)
+
+          first_component = path.relative_path_from(STORE_ROOT).each_filename.first
+          return false unless first_component
+
+          STORE_OBJECT_PATTERN.match?(first_component)
+        end
+      end
+
+      #: (lockfile: Pathname) -> void
+      def initialize(lockfile:)
+        @lockfile = lockfile
+        @nix_store_provider = NixStoreProvider.new
+        @specification_indexes = {}.compare_by_identity
+      end
+
+      # Returns a Resolution, or a Symbol naming why the package was skipped.
+      # Callers own how a skip is reported, so every reason surfaces through the
+      # same warning vocabulary as the other package sources.
+      #
+      #: (Bundler::LazySpecification) -> (Resolution | Symbol)
+      def resolve(locked)
+        source = locked.source
+        return :local_only_required if source.allow_git_ops?
+
+        checkout = checkout_path(source)
+        return :checkout_unavailable unless checkout.directory?
+
+        index = (@specification_indexes[source] ||= specification_index(source, checkout))
+        specification = index.search(locked).first
+        return :specification_unavailable unless specification
+
+        paths = package_paths(specification, checkout)
+        return :unsafe_specification_root unless paths
+
+        Resolution.new(
+          root: paths.canonical_root,
+          files: indexable_files(paths, specification.require_paths).freeze,
+        )
+      rescue UnsafeRequirePath
+        :unsafe_require_paths
+      rescue UnsafePackageFile
+        :unsafe_package_files
+      rescue Bundler::BundlerError, Gem::Exception
+        :specification_unreadable
+      rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP
+        :checkout_unavailable
+      end
+
+      private
+
+      #: (Bundler::Source::Git) -> Pathname
+      def checkout_path(source)
+        bundle_root = @lockfile.dirname
+        app_config = ENV["BUNDLE_APP_CONFIG"]
+        app_config_path = if app_config
+          Pathname(app_config).expand_path(bundle_root)
+        else
+          bundle_root.join(".bundle")
+        end
+        bundle_path = Pathname(Bundler::Settings.new(app_config_path).path.path).expand_path(bundle_root)
+        bundle_path.join("bundler/gems", source.extension_dir_name)
+      end
+
+      #: (Bundler::Source::Git, Pathname) -> Bundler::Index
+      def specification_index(source, checkout)
+        source.__send__(:set_install_path!, checkout)
+        source.specs
+      end
+
+      #: (Gem::Specification, Pathname) -> PackagePaths?
+      def package_paths(specification, checkout)
+        logical_root = Pathname(specification.full_gem_path).expand_path
+        logical_gemspec = Pathname(specification.loaded_from).expand_path
+        return unless Paths.inside?(logical_root, checkout)
+        return unless logical_gemspec.dirname == logical_root
+        return unless logical_gemspec.file? && logical_root.directory?
+
+        canonical_checkout = checkout.realpath
+        resolved_gemspec_root = logical_gemspec.realpath.dirname
+        logical_canonical_root = logical_root.realpath
+        return unless resolved_gemspec_root.directory?
+
+        normal_checkout = !checkout.symlink? &&
+          Paths.inside?(resolved_gemspec_root, canonical_checkout) &&
+          Paths.inside?(logical_canonical_root, canonical_checkout)
+        return unless normal_checkout || @nix_store_provider.trusted?(resolved_gemspec_root)
+
+        PackagePaths.new(
+          logical_root: logical_root,
+          canonical_root: normal_checkout ? logical_canonical_root : resolved_gemspec_root,
+          logical_canonical_root: logical_canonical_root,
+        )
+      rescue TypeError, ArgumentError
+        nil
+      end
+
+      #: (PackagePaths, Array[String]) -> Array[String]
+      def indexable_files(paths, require_paths)
+        logical_require_paths = require_paths.map do |relative_path|
+          raise UnsafeRequirePath unless relative_path.is_a?(String)
+
+          relative = Pathname(relative_path)
+          raise UnsafeRequirePath if relative.absolute? || relative.each_filename.include?("..")
+
+          candidate = paths.logical_root.join(relative).cleanpath
+          raise UnsafeRequirePath unless Paths.inside?(candidate, paths.logical_root)
+
+          candidate
+        rescue ArgumentError, EncodingError
+          raise UnsafeRequirePath
+        end
+
+        files = []
+        visited_directories = Set.new
+        logical_require_paths.each do |path|
+          next unless path.exist? || path.symlink?
+
+          traverse(path, paths, files, visited_directories, Set.new)
+        end
+        files.uniq.sort
+      end
+
+      # Descends one require path. `active_directories` carries the current
+      # branch so a symlink cycle raises instead of recursing forever, while
+      # `visited_directories` spans the whole walk so a diamond of symlinks is
+      # merely skipped.
+      #
+      #: (Pathname, PackagePaths, Array[String], Set[Pathname], Set[Pathname]) -> void
+      def traverse(path, paths, files, visited_directories, active_directories)
+        resolved = path.realpath
+        if resolved.file?
+          raise UnsafePackageFile unless Paths.inside?(resolved, paths.canonical_root)
+
+          files << resolved.to_s if INDEXABLE_EXTENSIONS.include?(path.extname)
+          return
+        end
+        return unless resolved.directory?
+
+        unless Paths.inside?(resolved, paths.logical_canonical_root) || Paths.inside?(resolved, paths.canonical_root)
+          raise UnsafePackageFile
+        end
+        raise UnsafePackageFile if active_directories.include?(resolved)
+        return if visited_directories.include?(resolved)
+
+        active_directories.add(resolved)
+        begin
+          path.children.sort_by(&:to_s).each do |child|
+            traverse(child, paths, files, visited_directories, active_directories)
+          end
+          visited_directories.add(resolved)
+        ensure
+          active_directories.delete(resolved)
+        end
+      rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP
+        raise UnsafePackageFile
+      end
+    end
+  end
+end

data/lib/rubylens/index/location_index.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require "set"
+require_relative "source_path"
+
+module RubyLens
+  module Index
+    # Answers the questions the collectors ask about a Rubydex location: which
+    # file it names, whether that file is in the workspace, whether it is test
+    # or core code, and which package owns it.
+    #
+    # Rubydex hands out a fresh wrapper and a fresh URI string on every call and
+    # a codebase has far more definitions than files, so each answer is memoized
+    # by URI. That memoization is why the collectors can afford to ask per
+    # definition instead of hoisting the questions themselves.
+    class LocationIndex
+      # 1 for a file under a test directory, 0 for any other workspace file.
+      TEST = 1
+      CORE = 0
+      TEST_SEGMENTS = %w[test tests spec specs feature features].freeze
+
+      #: Set[String]
+      attr_reader :package_document_paths
+
+      # The manifest is duck-typed: production passes a Manifest, tests pass
+      # stubs answering the same four questions.
+      #
+      #: (untyped manifest) -> void
+      def initialize(manifest)
+        @manifest = manifest
+        @path_by_uri = {}
+        @workspace_by_uri = {}
+        @scope_by_uri = {}
+        @package_index_by_uri = {}
+        @package_document_paths = Set.new
+      end
+
+      # One sweep over the indexed documents resolves each URI once, recording
+      # which of them belong to audited packages and returning the
+      # [document, path] pairs so callers need not re-enumerate or re-parse.
+      #
+      #: (Rubydex::Graph) -> Array[[Rubydex::Document, String]]
+      def resolve_documents(graph)
+        audited = @manifest.packages.flat_map(&:files).to_set
+        documents_with_paths = []
+        graph.documents.each do |document|
+          path = path_for(document.uri)
+          next unless path
+
+          documents_with_paths << [document, path]
+          @package_document_paths << path if audited.include?(path)
+        end
+        documents_with_paths
+      end
+
+      #: (String) -> String?
+      def path_for(uri)
+        @path_by_uri.fetch(uri) do
+          @path_by_uri[uri] = SourcePath.from_file_uri(uri)
+        end
+      end
+
+      #: (Rubydex::Location) -> bool
+      def workspace?(location)
+        uri = location.uri
+        @workspace_by_uri.fetch(uri) do
+          path = path_for(uri)
+          @workspace_by_uri[uri] = path ? @manifest.workspace_path?(path) : false
+        end
+      rescue StandardError
+        false
+      end
+
+      # TEST, CORE, or nil when the URI has no workspace-relative path.
+      #
+      #: (String) -> Integer?
+      def scope_for(uri)
+        @scope_by_uri.fetch(uri) do
+          relative = @manifest.relative_workspace_path(path_for(uri))
+          @scope_by_uri[uri] =
+            if relative
+              relative.split(File::SEPARATOR).any? { |segment| TEST_SEGMENTS.include?(segment) } ? TEST : CORE
+            end
+        end
+      end
+
+      # Package attribution is deliberately limited to documents Rubydex actually
+      # indexed, so a path that merely sits inside a package root cannot claim it.
+      #
+      #: (Rubydex::Location) -> Integer?
+      def package_index_for(location)
+        uri = location.uri
+        @package_index_by_uri.fetch(uri) do
+          path = path_for(uri)
+          @package_index_by_uri[uri] =
+            if path && @package_document_paths.include?(path)
+              @manifest.package_index_for(path)
+            end
+        end
+      end
+    end
+  end
+end

data/lib/rubylens/index/manifest.rb

@@ -2,33 +2,17 @@
 
 require "bundler"
 require "bundler/lockfile_parser"
-require "find"
 require "pathname"
 require "set"
 require_relative "../dependency_warning"
+require_relative "git_package_source"
 
 module RubyLens
   module Index
     class Manifest
       Package = Data.define(:name, :version, :role, :location, :root, :files)
       DependencySystem = Data.define(:id, :package_indexes, :label_package_index)
-      GitPackagePaths = Data.define(:logical_root, :canonical_root, :logical_canonical_root)
 
-      class NixStoreProvider
-        STORE_ROOT = Pathname("/nix/store").freeze
-        STORE_OBJECT_PATTERN = /\A[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+\z/
-
-        def trusted?(path)
-          path = Pathname(path).expand_path
-          return false unless Paths.inside?(path, STORE_ROOT)
-
-          first_component = path.relative_path_from(STORE_ROOT).each_filename.first
-          first_component && STORE_OBJECT_PATTERN.match?(first_component)
-        end
-      end
-
-      UnsafeGitRequirePath = Class.new(StandardError)
-      UnsafeGitPackageFile = Class.new(StandardError)
       GIT_SKIP_REASONS = DependencyWarning::REASONS
 
       attr_reader :root, :files, :workspace_files, :packages, :dependency_systems, :warnings, :dependency_warnings
@@ -40,7 +24,7 @@ module RubyLens
       def initialize(root:, lockfile: nil)
         @root = Pathname(root).expand_path.realpath
         @lockfile = Pathname(lockfile || @root.join("Gemfile.lock")).expand_path
-        @nix_store_provider = NixStoreProvider.new
+        @git_package_source = GitPackageSource.new(lockfile: @lockfile)
         @warnings = []
         @dependency_warnings = []
         @packages = []
@@ -49,7 +33,7 @@ module RubyLens
         @package_index_by_file = {}
         @package_index_cache = {}
         @relative_workspace_path_cache = {}
-        @git_spec_indexes = {}.compare_by_identity
+        @workspace_path_cache = {}
       end
 
       def build
@@ -70,23 +54,27 @@ module RubyLens
 
       def package_index_for(path)
         path = path.to_s
-        return @package_index_by_file[path] if @package_index_by_file.key?(path)
-        return @package_index_cache[path] if @package_index_cache.key?(path)
-
-        @package_index_cache[path] = uncached_package_index_for(path)
+        @package_index_by_file.fetch(path) do
+          @package_index_cache.fetch(path) do
+            @package_index_cache[path] = uncached_package_index_for(path)
+          end
+        end
       rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP
         nil
       end
 
       def workspace_path?(path)
-        Paths.inside?(path, @root)
+        path = path.to_s
+        @workspace_path_cache.fetch(path) do
+          @workspace_path_cache[path] = Paths.inside?(path, @root)
+        end
       end
 
       def relative_workspace_path(path)
         path = path.to_s
-        return @relative_workspace_path_cache[path] if @relative_workspace_path_cache.key?(path)
-
-        @relative_workspace_path_cache[path] = uncached_relative_workspace_path(path)
+        @relative_workspace_path_cache.fetch(path) do
+          @relative_workspace_path_cache[path] = uncached_relative_workspace_path(path)
+        end
       end
 
       private
@@ -188,134 +176,17 @@ module RubyLens
       end
 
       def git_package(locked, role)
-        source = locked.source
-        return skip_git_dependency(locked, :local_only_required) if source.allow_git_ops?
-
-        checkout = git_checkout_path(source)
-        return skip_git_dependency(locked, :checkout_unavailable) unless checkout.directory?
-
-        specification = (@git_spec_indexes[source] ||= git_specifications(source, checkout)).search(locked).first
-        return skip_git_dependency(locked, :specification_unavailable) unless specification
-
-        paths = git_package_paths(specification, checkout)
-        return skip_git_dependency(locked, :unsafe_specification_root) unless paths
-
-        files = git_indexable_files(paths, specification.require_paths)
+        resolution = @git_package_source.resolve(locked)
+        return skip_git_dependency(locked, resolution) if resolution.is_a?(Symbol)
 
         Package.new(
           name: locked.name,
           version: locked.version.to_s,
           role: role,
           location: "external",
-          root: paths.canonical_root,
-          files: files.freeze,
+          root: resolution.root,
+          files: resolution.files,
         )
-      rescue UnsafeGitRequirePath
-        skip_git_dependency(locked, :unsafe_require_paths)
-      rescue UnsafeGitPackageFile
-        skip_git_dependency(locked, :unsafe_package_files)
-      rescue Bundler::BundlerError, Gem::Exception
-        skip_git_dependency(locked, :specification_unreadable)
-      rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP
-        skip_git_dependency(locked, :checkout_unavailable)
-      end
-
-      def git_checkout_path(source)
-        bundle_root = @lockfile.dirname
-        app_config = ENV["BUNDLE_APP_CONFIG"]
-        app_config_path = if app_config
-          Pathname(app_config).expand_path(bundle_root)
-        else
-          bundle_root.join(".bundle")
-        end
-        bundle_path = Pathname(Bundler::Settings.new(app_config_path).path.path).expand_path(bundle_root)
-        bundle_path.join("bundler/gems", source.extension_dir_name)
-      end
-
-      def git_specifications(source, checkout)
-        source.__send__(:set_install_path!, checkout)
-        source.specs
-      end
-
-      def git_package_paths(specification, checkout)
-        logical_root = Pathname(specification.full_gem_path).expand_path
-        logical_gemspec = Pathname(specification.loaded_from).expand_path
-        return unless Paths.inside?(logical_root, checkout)
-        return unless logical_gemspec.dirname == logical_root
-        return unless logical_gemspec.file? && logical_root.directory?
-
-        canonical_checkout = checkout.realpath
-        resolved_gemspec_root = logical_gemspec.realpath.dirname
-        logical_canonical_root = logical_root.realpath
-        return unless resolved_gemspec_root.directory?
-        normal_checkout = !checkout.symlink? &&
-          Paths.inside?(resolved_gemspec_root, canonical_checkout) &&
-          Paths.inside?(logical_canonical_root, canonical_checkout)
-        return unless normal_checkout || @nix_store_provider.trusted?(resolved_gemspec_root)
-
-        canonical_root = normal_checkout ? logical_canonical_root : resolved_gemspec_root
-
-        GitPackagePaths.new(
-          logical_root: logical_root,
-          canonical_root: canonical_root,
-          logical_canonical_root: logical_canonical_root,
-        )
-      rescue TypeError, ArgumentError
-        nil
-      end
-
-      def git_indexable_files(paths, require_paths)
-        logical_require_paths = require_paths.map do |relative_path|
-          raise UnsafeGitRequirePath unless relative_path.is_a?(String)
-
-          relative = Pathname(relative_path)
-          raise UnsafeGitRequirePath if relative.absolute? || relative.each_filename.include?("..")
-
-          candidate = paths.logical_root.join(relative).cleanpath
-          raise UnsafeGitRequirePath unless Paths.inside?(candidate, paths.logical_root)
-
-          candidate
-        rescue ArgumentError, EncodingError
-          raise UnsafeGitRequirePath
-        end
-
-        files = []
-        visited_directories = Set.new
-        logical_require_paths.each do |path|
-          next unless path.exist? || path.symlink?
-
-          traverse_git_package_path(path, paths, files, visited_directories, Set.new)
-        end
-        files.uniq.sort
-      end
-
-      def traverse_git_package_path(path, paths, files, visited_directories, active_directories)
-        resolved = path.realpath
-        if resolved.file?
-          raise UnsafeGitPackageFile unless Paths.inside?(resolved, paths.canonical_root)
-
-          files << resolved.to_s if indexable?(path)
-          return
-        end
-        return unless resolved.directory?
-
-        unless Paths.inside?(resolved, paths.logical_canonical_root) || Paths.inside?(resolved, paths.canonical_root)
-          raise UnsafeGitPackageFile
-        end
-        raise UnsafeGitPackageFile if active_directories.include?(resolved)
-        return if visited_directories.include?(resolved)
-
-        active_directories.add(resolved)
-        begin
-          path.children.sort_by(&:to_s).each do |child|
-            traverse_git_package_path(child, paths, files, visited_directories, active_directories)
-          end
-          visited_directories.add(resolved)
-        ensure
-          active_directories.delete(resolved)
-        end
-      rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP
-        raise UnsafeGitPackageFile
       end
 
       def skip_git_dependency(locked, reason_code)
@@ -428,20 +299,54 @@ module RubyLens
         end.flat_map { |path| enumerate(path, root) }.uniq.sort
       end
 
+      # Walks an already-canonical directory (the caller resolved and containment-checked
+      # it) without re-resolving every entry: descending only non-symlink directories
+      # keeps each joined path canonical, so regular files join the list with a cheap
+      # prefix containment check and only symlinked entries pay for a realpath. Symlinked
+      # directories are never descended, matching Find.find's lstat-based traversal.
       def enumerate(path, root)
         return [path.to_s] if path.file? && indexable?(path) && Paths.inside?(path.realpath, root)
         return [] unless path.directory?
+        return [] if path.symlink?
 
         files = []
-        Find.find(path) do |candidate|
-          candidate = Pathname(candidate)
-          next if candidate.directory?
-          next unless indexable?(candidate)
-
-          resolved = candidate.realpath
-          files << resolved.to_s if Paths.inside?(resolved, root)
-        rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP
-          next
+        root_prefix = "#{root}#{File::SEPARATOR}"
+        start = path.to_s
+        # Child names must come back in the starting path's encoding. A package
+        # root that is not valid UTF-8 otherwise yields UTF-8 entry names that
+        # cannot be joined to it, and the whole build aborts on a tree the
+        # Find-based walk indexed fine.
+        encoding = start.encoding == Encoding::US_ASCII ? Encoding.find("filesystem") : start.encoding
+        pending = [start]
+        while (directory = pending.pop)
+          entries = begin
+            Dir.children(directory, encoding: encoding)
+          rescue Errno::ENOENT, Errno::EACCES, Errno::ENOTDIR, Errno::ELOOP, Errno::ENAMETOOLONG, Errno::EINVAL
+            next
+          end
+          entries.each do |entry|
+            candidate = File.join(directory, entry)
+            stat = begin
+              File.lstat(candidate)
+            rescue Errno::ENOENT, Errno::EACCES, Errno::ENOTDIR, Errno::ELOOP, Errno::ENAMETOOLONG, Errno::EINVAL
+              next
+            end
+            if stat.symlink?
+              next if File.directory?(candidate)
+              next unless INDEXABLE_EXTENSIONS.include?(File.extname(entry))
+
+              begin
+                resolved = File.realpath(candidate)
+                files << resolved if Paths.inside?(resolved, root)
+              rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP
+                next
+              end
+            elsif stat.directory?
+              pending << candidate
+            elsif INDEXABLE_EXTENSIONS.include?(File.extname(entry))
+              files << candidate if candidate.start_with?(root_prefix)
+            end
+          end
         end
         files
       end

data/lib/rubylens/index/rspec_extractor.rb

@@ -11,28 +11,24 @@ module RubyLens
       SPEC_SEGMENTS = Set.new(%w[spec specs]).freeze
       PROXY_NAME_PREFIX = "RSpec example group #"
 
-      Group = Data.define(:name, :component)
       Result = Data.define(:groups, :method_count)
 
-      def initialize(graph:, manifest:, package_document_paths: Set.new)
+      def initialize(graph:, manifest:, package_document_paths: Set.new, documents_with_paths: nil)
         @graph = graph
         @manifest = manifest
         @package_document_paths = package_document_paths
+        @documents_with_paths = documents_with_paths
       end
 
       def call
         groups = []
         method_count = 0
 
-        spec_documents.each do |document, relative|
+        spec_documents.each do |document, _relative|
           group_count, example_count = reference_counts(document.method_references)
           method_count += example_count
-          component = SourcePath.component_for(relative)
           group_count.times do
-            groups << Group.new(
-              format("%s%06d", PROXY_NAME_PREFIX, groups.length + 1),
-              component,
-            )
+            groups << format("%s%06d", PROXY_NAME_PREFIX, groups.length + 1)
           end
         end
 
@@ -41,10 +37,16 @@ module RubyLens
 
       private
 
+      # The adapter passes its already-resolved [document, path] pairs so the
+      # documents are not re-enumerated and their URIs not re-parsed here;
+      # standalone construction still resolves them from the graph.
       def spec_documents
-        @graph.documents.filter_map do |document|
+        pairs = @documents_with_paths || @graph.documents.filter_map do |document|
           path = SourcePath.from_file_uri(document.uri)
-          next unless path && @manifest.workspace_path?(path)
+          [document, path] if path
+        end
+        pairs.filter_map do |document, path|
+          next unless @manifest.workspace_path?(path)
           next if @package_document_paths.include?(path)
 
           relative = @manifest.relative_workspace_path(path)