rubylens 0.2.0 → 0.2.1

data/lib/rubylens/index/declaration_collector.rb

@@ -0,0 +1,288 @@
+# frozen_string_literal: true
+
+require "rubydex"
+require_relative "../model/dependency_aggregation"
+
+module RubyLens
+  module Index
+    # Streams Rubydex's declarations once and splits them into the three
+    # populations the snapshot needs: workspace namespaces that become scene
+    # points, Ruby construct tallies per category, and dependency declaration
+    # rows aggregated per package.
+    #
+    # It is a single pass because Rubydex materializes a new wrapper for every
+    # accessor call, so revisiting the stream costs as much as producing it.
+    class DeclarationCollector
+      # Shapes shared with the other collectors. A site key is a definition's
+      # location flattened for comparison and grouping; a definition range is a
+      # namespace's span within one file, tagged with the namespace's ordinal.
+      #
+      # @rbs!
+      #   type site_key = [String, Integer, Integer, Integer, Integer]
+      #   type definition_range = [Integer, Integer, Integer, Integer, Integer]
+      #   type dependency_package = { ruby_counts: Array[Integer], declarations: Array[Array[Integer]] }
+
+      # A workspace namespace that earned a scene point, with the canonical
+      # definition sites that decide where its constant references land.
+      Namespace = Data.define(
+        :declaration, #: Rubydex::Namespace
+        :name, #: String
+        :definition_sites, #: Integer
+        :scope, #: Integer
+      )
+
+      Result = Data.define(
+        :namespaces, #: Array[Namespace]
+        :definition_ranges, #: Hash[String, Array[definition_range]]
+        :category_stats, #: Hash[String, Array[Integer]]
+        :dependency_packages, #: Array[dependency_package]
+        :dependency_signal_maxima, #: Array[Integer]
+        :dependency_ordinal_by_name, #: Hash[String, Integer]
+      )
+
+      CLASS = 0
+      MODULE = 1
+      METHOD = 2
+      CONSTANT = 3
+      OTHER_KIND = 2
+      MIXED_SCOPE = 2
+
+      #: (manifest: untyped, locations: LocationIndex) -> void
+      def initialize(manifest:, locations:)
+        @manifest = manifest
+        @locations = locations
+      end
+
+      #: (Enumerable[Rubydex::Declaration]) -> Result
+      def call(declarations)
+        namespaces = []
+        category_stats = { "core" => Array.new(4, 0), "tests" => Array.new(4, 0) }
+        aggregation = Model::DependencyAggregation.new(package_count: @manifest.packages.length)
+        definition_ranges = Hash.new { |ranges, uri| ranges[uri] = [] }
+        dependency_positions = {}
+
+        declarations.each do |declaration|
+          name = declaration.name
+          next unless eligible?(declaration, name)
+
+          namespace = declaration.is_a?(Rubydex::Namespace)
+          construct = construct_index(declaration)
+          # Destructured immediately rather than wrapped: this runs once per
+          # declaration in the largest loop RubyLens has, and the names below
+          # are the only place these five values are ever read.
+          workspace_count, tests_only, site_keys, canonical_scope, package_site_keys =
+            summarize(declaration, namespace)
+
+          if site_keys
+            site_keys = site_keys.uniq if site_keys.length > 1
+            ordinal = namespaces.length
+            site_keys.each do |uri, start_line, start_column, end_line, end_column|
+              definition_ranges[uri] << [start_line, start_column, end_line, end_column, ordinal]
+            end
+            namespaces << Namespace.new(
+              declaration: declaration,
+              name: name,
+              definition_sites: site_keys.length,
+              scope: canonical_scope,
+            )
+          end
+          if construct && workspace_count.positive?
+            category_stats.fetch(tests_only ? "tests" : "core")[construct] += 1
+          end
+          if package_site_keys
+            collect_dependency(
+              aggregation, dependency_positions, declaration, name, namespace, construct, package_site_keys
+            )
+          end
+        end
+
+        Result.new(
+          namespaces: namespaces,
+          definition_ranges: definition_ranges,
+          category_stats: category_stats,
+          dependency_packages: aggregation.packages,
+          dependency_signal_maxima: aggregation.signal_maxima,
+          dependency_ordinal_by_name: global_dependency_ordinals(aggregation.packages, dependency_positions),
+        )
+      end
+
+      # Rubydex reports synthetic declarations RubyLens never draws: anonymous
+      # namespaces, Todo placeholders standing in for unresolved constants, and
+      # singleton classes attached to either.
+      #
+      #: (Rubydex::Declaration, ?String?) -> bool
+      def eligible?(declaration, name = declaration.name)
+        return false if name.nil? || name.empty? || name.include?("<anonymous>")
+        return false if declaration.is_a?(Rubydex::Todo)
+        return true unless declaration.is_a?(Rubydex::SingletonClass)
+
+        attached = declaration.attached_class
+        !attached.is_a?(Rubydex::SingletonClass) && !attached.is_a?(Rubydex::Todo)
+      end
+
+      #: (Rubydex::Declaration) -> Integer?
+      def construct_index(declaration)
+        case declaration
+        when Rubydex::SingletonClass then nil
+        when Rubydex::Class then CLASS
+        when Rubydex::Module then MODULE
+        when Rubydex::Method then METHOD
+        when Rubydex::Constant, Rubydex::ConstantAlias then CONSTANT
+        end
+      end
+
+      private
+
+      # One pass over a declaration's definitions, so each definition's location
+      # and its per-URI workspace, scope, and package answers are read once.
+      # Returns [workspace_count, tests_only, canonical_site_keys,
+      # canonical_scope, package_site_keys]; the site-key collections stay nil
+      # until a definition contributes one, since most declarations contribute
+      # neither a canonical namespace site nor a package site.
+      #
+      #: (Rubydex::Declaration, bool) -> [Integer, bool, Array[site_key]?, Integer, Hash[Integer, Array[site_key]]?]
+      def summarize(declaration, namespace)
+        workspace_count = 0
+        tests_seen = false
+        core_seen = false
+        canonical_tests_seen = false
+        canonical_core_seen = false
+        canonical_site_keys = nil
+        package_site_keys = nil
+
+        declaration.definitions.each do |definition|
+          location = definition.location
+          site_key = nil
+          if @locations.workspace?(location)
+            workspace_count += 1
+            scope = @locations.scope_for(location.uri)
+            if scope == LocationIndex::TEST
+              tests_seen = true
+            elsif scope
+              core_seen = true
+            end
+            if namespace && canonical_definition?(declaration, definition)
+              (canonical_site_keys ||= []) << (site_key = location.comparable_values)
+              if scope == LocationIndex::TEST
+                canonical_tests_seen = true
+              elsif scope
+                canonical_core_seen = true
+              end
+            end
+          end
+          package_index = @locations.package_index_for(location)
+          if package_index
+            ((package_site_keys ||= {})[package_index] ||= []) << (site_key || location.comparable_values)
+          end
+        end
+
+        [
+          workspace_count,
+          tests_seen && !core_seen,
+          canonical_site_keys,
+          canonical_scope(canonical_tests_seen, canonical_core_seen),
+          package_site_keys,
+        ]
+      end
+
+      #: (bool, bool) -> Integer
+      def canonical_scope(tests_seen, core_seen)
+        return LocationIndex::CORE unless tests_seen
+
+        core_seen ? MIXED_SCOPE : LocationIndex::TEST
+      end
+
+      # A declaration reopened across packages is attributed to the one holding
+      # the most of its definitions, ties going to the earliest package.
+      #
+      #: (Model::DependencyAggregation, Hash[String, [Integer, Integer]], Rubydex::Declaration, String, bool, Integer?, Hash[Integer, Array[site_key]]) -> void
+      def collect_dependency(aggregation, positions, declaration, name, namespace, construct, package_site_keys)
+        package_index = nil
+        site_keys = nil
+        package_site_keys.each do |index, keys|
+          if site_keys.nil? || keys.length > site_keys.length ||
+              (keys.length == site_keys.length && index < package_index)
+            package_index = index
+            site_keys = keys
+          end
+        end
+
+        sites = site_keys.length > 1 ? site_keys.uniq.length : site_keys.length
+        references = length_of(declaration, :references)
+        row = [
+          namespace_kind(declaration),
+          namespace ? [declaration.ancestors.count - 1, 0].max : 0,
+          sites,
+          [sites - 1, 0].max,
+          namespace ? [declaration.descendants.count - 1, 0].max : 0,
+          references,
+          namespace ? length_of(declaration, :members) : 0,
+        ].freeze
+        local_index = aggregation.add(package_index: package_index, row: row, construct_index: construct)
+        if references.positive? && constant_reference_target?(declaration, namespace)
+          positions[name] = [package_index, local_index]
+        end
+      end
+
+      # Dependency stars are addressed by a single ordinal spanning every
+      # package, so per-package positions are rebased onto that flat sequence.
+      #
+      #: (Array[dependency_package], Hash[String, [Integer, Integer]]) -> Hash[String, Integer]
+      def global_dependency_ordinals(packages, positions)
+        offsets = []
+        offset = 0
+        packages.each do |package|
+          offsets << offset
+          offset += package.fetch(:declarations).length
+        end
+        positions.transform_values! do |package_index, local_index|
+          offsets.fetch(package_index) + local_index
+        end
+        positions.freeze
+      end
+
+      #: (Rubydex::Declaration) -> Integer
+      def namespace_kind(declaration)
+        case declaration
+        when Rubydex::Class then CLASS
+        when Rubydex::Module then MODULE
+        else OTHER_KIND
+        end
+      end
+
+      #: (Rubydex::Declaration, bool) -> bool
+      def constant_reference_target?(declaration, namespace)
+        namespace ||
+          declaration.is_a?(Rubydex::Constant) ||
+          declaration.is_a?(Rubydex::ConstantAlias)
+      end
+
+      #: (Rubydex::Declaration, Rubydex::Definition) -> bool
+      def canonical_definition?(declaration, definition)
+        (declaration.is_a?(Rubydex::Class) && definition.is_a?(Rubydex::ClassDefinition)) ||
+          (declaration.is_a?(Rubydex::Module) && definition.is_a?(Rubydex::ModuleDefinition))
+      end
+
+      # Rubydex collections expose size on some shapes and only count on others,
+      # and this pre-1.0 interface can raise from either. A dependency star that
+      # cannot report a signal is still a real declaration, so an unreadable
+      # count degrades to zero rather than losing the row.
+      #
+      #: (untyped, Symbol) -> Integer
+      def length_of(object, name)
+        records = object.public_send(name)
+        size = records.size
+        size.nil? ? count_of(records) : size
+      rescue StandardError
+        count_of(records)
+      end
+
+      #: (untyped) -> Integer
+      def count_of(records)
+        records ? records.count : 0
+      rescue StandardError
+        0
+      end
+    end
+  end
+end

data/lib/rubylens/index/git_package_source.rb

@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+require "bundler"
+require "pathname"
+require "set"
+
+module RubyLens
+  module Index
+    # Resolves a git-sourced lockfile entry into the exact files RubyLens may index.
+    #
+    # Git checkouts carry more authority than RubyGems packages: the gemspec is
+    # code from the checkout that names its own require paths, and the tree may
+    # hold symlinks pointing anywhere on disk. Every path here is therefore
+    # resolved and re-checked against the canonical package root before it
+    # becomes indexable, and any breach ends the package rather than narrowing
+    # it, so a partially trusted tree can never contribute files.
+    class GitPackageSource
+      UnsafeRequirePath = Class.new(StandardError)
+      UnsafePackageFile = Class.new(StandardError)
+
+      # Logical roots come from the gemspec and may be symlinked; the canonical
+      # root is the resolved directory every indexable file must sit inside.
+      PackagePaths = Data.define(
+        :logical_root, #: Pathname
+        :canonical_root, #: Pathname
+        :logical_canonical_root, #: Pathname
+      )
+      Resolution = Data.define(
+        :root, #: Pathname
+        :files, #: Array[String]
+      )
+
+      # Nix stores gems in immutable, content-addressed store objects, so a
+      # checkout symlinked into one is trustworthy even though it resolves
+      # outside the bundle directory.
+      class NixStoreProvider
+        STORE_ROOT = Pathname("/nix/store").freeze
+        STORE_OBJECT_PATTERN = /\A[0123456789abcdfghijklmnpqrsvwxyz]{32}-.+\z/
+
+        #: (Pathname | String) -> bool
+        def trusted?(path)
+          path = Pathname(path).expand_path
+          return false unless Paths.inside?(path, STORE_ROOT)
+
+          first_component = path.relative_path_from(STORE_ROOT).each_filename.first
+          return false unless first_component
+
+          STORE_OBJECT_PATTERN.match?(first_component)
+        end
+      end
+
+      #: (lockfile: Pathname) -> void
+      def initialize(lockfile:)
+        @lockfile = lockfile
+        @nix_store_provider = NixStoreProvider.new
+        @specification_indexes = {}.compare_by_identity
+      end
+
+      # Returns a Resolution, or a Symbol naming why the package was skipped.
+      # Callers own how a skip is reported, so every reason surfaces through the
+      # same warning vocabulary as the other package sources.
+      #
+      #: (Bundler::LazySpecification) -> (Resolution | Symbol)
+      def resolve(locked)
+        source = locked.source
+        return :local_only_required if source.allow_git_ops?
+
+        checkout = checkout_path(source)
+        return :checkout_unavailable unless checkout.directory?
+
+        index = (@specification_indexes[source] ||= specification_index(source, checkout))
+        specification = index.search(locked).first
+        return :specification_unavailable unless specification
+
+        paths = package_paths(specification, checkout)
+        return :unsafe_specification_root unless paths
+
+        Resolution.new(
+          root: paths.canonical_root,
+          files: indexable_files(paths, specification.require_paths).freeze,
+        )
+      rescue UnsafeRequirePath
+        :unsafe_require_paths
+      rescue UnsafePackageFile
+        :unsafe_package_files
+      rescue Bundler::BundlerError, Gem::Exception
+        :specification_unreadable
+      rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP
+        :checkout_unavailable
+      end
+
+      private
+
+      #: (Bundler::Source::Git) -> Pathname
+      def checkout_path(source)
+        bundle_root = @lockfile.dirname
+        app_config = ENV["BUNDLE_APP_CONFIG"]
+        app_config_path = if app_config
+          Pathname(app_config).expand_path(bundle_root)
+        else
+          bundle_root.join(".bundle")
+        end
+        bundle_path = Pathname(Bundler::Settings.new(app_config_path).path.path).expand_path(bundle_root)
+        bundle_path.join("bundler/gems", source.extension_dir_name)
+      end
+
+      #: (Bundler::Source::Git, Pathname) -> Bundler::Index
+      def specification_index(source, checkout)
+        source.__send__(:set_install_path!, checkout)
+        source.specs
+      end
+
+      #: (Gem::Specification, Pathname) -> PackagePaths?
+      def package_paths(specification, checkout)
+        logical_root = Pathname(specification.full_gem_path).expand_path
+        logical_gemspec = Pathname(specification.loaded_from).expand_path
+        return unless Paths.inside?(logical_root, checkout)
+        return unless logical_gemspec.dirname == logical_root
+        return unless logical_gemspec.file? && logical_root.directory?
+
+        canonical_checkout = checkout.realpath
+        resolved_gemspec_root = logical_gemspec.realpath.dirname
+        logical_canonical_root = logical_root.realpath
+        return unless resolved_gemspec_root.directory?
+
+        normal_checkout = !checkout.symlink? &&
+          Paths.inside?(resolved_gemspec_root, canonical_checkout) &&
+          Paths.inside?(logical_canonical_root, canonical_checkout)
+        return unless normal_checkout || @nix_store_provider.trusted?(resolved_gemspec_root)
+
+        PackagePaths.new(
+          logical_root: logical_root,
+          canonical_root: normal_checkout ? logical_canonical_root : resolved_gemspec_root,
+          logical_canonical_root: logical_canonical_root,
+        )
+      rescue TypeError, ArgumentError
+        nil
+      end
+
+      #: (PackagePaths, Array[String]) -> Array[String]
+      def indexable_files(paths, require_paths)
+        logical_require_paths = require_paths.map do |relative_path|
+          raise UnsafeRequirePath unless relative_path.is_a?(String)
+
+          relative = Pathname(relative_path)
+          raise UnsafeRequirePath if relative.absolute? || relative.each_filename.include?("..")
+
+          candidate = paths.logical_root.join(relative).cleanpath
+          raise UnsafeRequirePath unless Paths.inside?(candidate, paths.logical_root)
+
+          candidate
+        rescue ArgumentError, EncodingError
+          raise UnsafeRequirePath
+        end
+
+        files = []
+        visited_directories = Set.new
+        logical_require_paths.each do |path|
+          next unless path.exist? || path.symlink?
+
+          traverse(path, paths, files, visited_directories, Set.new)
+        end
+        files.uniq.sort
+      end
+
+      # Descends one require path. `active_directories` carries the current
+      # branch so a symlink cycle raises instead of recursing forever, while
+      # `visited_directories` spans the whole walk so a diamond of symlinks is
+      # merely skipped.
+      #
+      #: (Pathname, PackagePaths, Array[String], Set[Pathname], Set[Pathname]) -> void
+      def traverse(path, paths, files, visited_directories, active_directories)
+        resolved = path.realpath
+        if resolved.file?
+          raise UnsafePackageFile unless Paths.inside?(resolved, paths.canonical_root)
+
+          files << resolved.to_s if INDEXABLE_EXTENSIONS.include?(path.extname)
+          return
+        end
+        return unless resolved.directory?
+
+        unless Paths.inside?(resolved, paths.logical_canonical_root) || Paths.inside?(resolved, paths.canonical_root)
+          raise UnsafePackageFile
+        end
+        raise UnsafePackageFile if active_directories.include?(resolved)
+        return if visited_directories.include?(resolved)
+
+        active_directories.add(resolved)
+        begin
+          path.children.sort_by(&:to_s).each do |child|
+            traverse(child, paths, files, visited_directories, active_directories)
+          end
+          visited_directories.add(resolved)
+        ensure
+          active_directories.delete(resolved)
+        end
+      rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP
+        raise UnsafePackageFile
+      end
+    end
+  end
+end

data/lib/rubylens/index/location_index.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require "set"
+require_relative "source_path"
+
+module RubyLens
+  module Index
+    # Answers the questions the collectors ask about a Rubydex location: which
+    # file it names, whether that file is in the workspace, whether it is test
+    # or core code, and which package owns it.
+    #
+    # Rubydex hands out a fresh wrapper and a fresh URI string on every call and
+    # a codebase has far more definitions than files, so each answer is memoized
+    # by URI. That memoization is why the collectors can afford to ask per
+    # definition instead of hoisting the questions themselves.
+    class LocationIndex
+      # 1 for a file under a test directory, 0 for any other workspace file.
+      TEST = 1
+      CORE = 0
+      TEST_SEGMENTS = %w[test tests spec specs feature features].freeze
+
+      #: Set[String]
+      attr_reader :package_document_paths
+
+      # The manifest is duck-typed: production passes a Manifest, tests pass
+      # stubs answering the same four questions.
+      #
+      #: (untyped manifest) -> void
+      def initialize(manifest)
+        @manifest = manifest
+        @path_by_uri = {}
+        @workspace_by_uri = {}
+        @scope_by_uri = {}
+        @package_index_by_uri = {}
+        @package_document_paths = Set.new
+      end
+
+      # One sweep over the indexed documents resolves each URI once, recording
+      # which of them belong to audited packages and returning the
+      # [document, path] pairs so callers need not re-enumerate or re-parse.
+      #
+      #: (Rubydex::Graph) -> Array[[Rubydex::Document, String]]
+      def resolve_documents(graph)
+        audited = @manifest.packages.flat_map(&:files).to_set
+        documents_with_paths = []
+        graph.documents.each do |document|
+          path = path_for(document.uri)
+          next unless path
+
+          documents_with_paths << [document, path]
+          @package_document_paths << path if audited.include?(path)
+        end
+        documents_with_paths
+      end
+
+      #: (String) -> String?
+      def path_for(uri)
+        @path_by_uri.fetch(uri) do
+          @path_by_uri[uri] = SourcePath.from_file_uri(uri)
+        end
+      end
+
+      #: (Rubydex::Location) -> bool
+      def workspace?(location)
+        uri = location.uri
+        @workspace_by_uri.fetch(uri) do
+          path = path_for(uri)
+          @workspace_by_uri[uri] = path ? @manifest.workspace_path?(path) : false
+        end
+      rescue StandardError
+        false
+      end
+
+      # TEST, CORE, or nil when the URI has no workspace-relative path.
+      #
+      #: (String) -> Integer?
+      def scope_for(uri)
+        @scope_by_uri.fetch(uri) do
+          relative = @manifest.relative_workspace_path(path_for(uri))
+          @scope_by_uri[uri] =
+            if relative
+              relative.split(File::SEPARATOR).any? { |segment| TEST_SEGMENTS.include?(segment) } ? TEST : CORE
+            end
+        end
+      end
+
+      # Package attribution is deliberately limited to documents Rubydex actually
+      # indexed, so a path that merely sits inside a package root cannot claim it.
+      #
+      #: (Rubydex::Location) -> Integer?
+      def package_index_for(location)
+        uri = location.uri
+        @package_index_by_uri.fetch(uri) do
+          path = path_for(uri)
+          @package_index_by_uri[uri] =
+            if path && @package_document_paths.include?(path)
+              @manifest.package_index_for(path)
+            end
+        end
+      end
+    end
+  end
+end