rubylens 0.1.0.pre.1

data/lib/rubylens/index/manifest.rb

@@ -0,0 +1,231 @@
+# frozen_string_literal: true
+
+require "bundler"
+require "bundler/lockfile_parser"
+require "find"
+require "pathname"
+require "set"
+
+module RubyLens
+  module Index
+    class Manifest
+      Package = Data.define(:name, :version, :role, :location, :root, :files)
+
+      attr_reader :root, :files, :workspace_files, :packages, :warnings
+
+      def self.build(root:, lockfile: nil)
+        new(root: root, lockfile: lockfile).tap(&:build)
+      end
+
+      def initialize(root:, lockfile: nil)
+        @root = Pathname(root).expand_path.realpath
+        @lockfile = Pathname(lockfile || @root.join("Gemfile.lock")).expand_path
+        @warnings = []
+        @packages = []
+        @package_roots = []
+        @package_index_by_file = {}
+        @package_index_cache = {}
+        @relative_workspace_path_cache = {}
+      end
+
+      def build
+        @workspace_files = GitRepository.new(@root).selected_files.freeze
+        build_packages
+        @package_roots = @packages.each_with_index.map { |package, index| [package.root, index] }
+          .sort_by { |package_root, _index| -package_root.to_s.length }
+        build_package_index
+        @files = (@workspace_files + @packages.flat_map(&:files)).uniq.freeze
+        self
+      end
+
+      def package_index_for(path)
+        path = path.to_s
+        return @package_index_by_file[path] if @package_index_by_file.key?(path)
+        return @package_index_cache[path] if @package_index_cache.key?(path)
+
+        @package_index_cache[path] = uncached_package_index_for(path)
+      rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP
+        nil
+      end
+
+      def workspace_path?(path)
+        Paths.inside?(path, @root)
+      end
+
+      def relative_workspace_path(path)
+        path = path.to_s
+        return @relative_workspace_path_cache[path] if @relative_workspace_path_cache.key?(path)
+
+        @relative_workspace_path_cache[path] = Pathname(path).realpath.relative_path_from(@root).to_s
+      rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP, ArgumentError
+        nil
+      end
+
+      private
+
+      def build_package_index
+        package_index_by_file = {}
+        @package_roots.each do |_package_root, index|
+          @packages.fetch(index).files.each do |path|
+            path = path.to_s
+            package_index_by_file[path] = index unless package_index_by_file.key?(path)
+          end
+        end
+        @workspace_files.each do |path|
+          path = path.to_s
+          package_index_by_file[path] = nil unless package_index_by_file.key?(path)
+        end
+        @package_index_by_file = package_index_by_file.freeze
+      end
+
+      def uncached_package_index_for(path)
+        resolved = Pathname(path).expand_path
+        resolved = resolved.realpath if resolved.exist?
+        resolved_path = resolved.to_s
+        return @package_index_by_file[resolved_path] if @package_index_by_file.key?(resolved_path)
+
+        entry = @package_roots.find { |package_root, _index| Paths.inside?(resolved, package_root) }
+        entry&.last
+      end
+
+      def build_packages
+        unless @lockfile.file?
+          @warnings << "No Gemfile.lock found; dependency systems were omitted."
+          return
+        end
+
+        parser = Bundler::LockfileParser.new(@lockfile.read)
+        direct_names = parser.dependencies.keys.to_set
+        excluded_names = tool_only_dependency_names(parser, direct_names)
+        parser.specs.uniq { |specification| [specification.name, specification.version.to_s] }.each do |locked|
+          next if excluded_names.include?(locked.name)
+
+          package = package_for(locked, direct_names)
+          @packages << package if package
+        end
+      rescue Bundler::LockfileError, Errno::EACCES => error
+        @warnings << "Gemfile.lock could not be read: #{error.class}."
+      end
+
+      def package_for(locked, direct_names)
+        source = locked.source.class.name.split("::").last.downcase
+        role = direct_names.include?(locked.name) ? "direct" : "transitive"
+        if source == "path"
+          return local_package(locked)
+        end
+        unless source == "rubygems"
+          @warnings << "Skipped #{locked.name} #{locked.version}: #{source} dependencies are not indexed yet."
+          return nil
+        end
+
+        specification = installed_specification(locked)
+        unless specification
+          @warnings << "Skipped #{locked.name} #{locked.version}: exact installed gem not found."
+          return nil
+        end
+
+        root = Pathname(specification.full_gem_path).realpath
+        files = indexable_files(root, specification.require_paths)
+        Package.new(
+          name: locked.name,
+          version: locked.version.to_s,
+          role: role,
+          location: "external",
+          root: root,
+          files: files.freeze,
+        )
+      rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP => error
+        @warnings << "Skipped #{locked.name} #{locked.version}: #{error.class}."
+        nil
+      end
+
+      def local_package(locked)
+        source_path = locked.source.respond_to?(:path) ? locked.source.path : nil
+        unless source_path
+          @warnings << "Skipped #{locked.name} #{locked.version}: local source path is unavailable."
+          return nil
+        end
+
+        root = Pathname(source_path.to_s)
+        root = @lockfile.dirname.join(root) unless root.absolute?
+        root = root.realpath
+        unless Paths.inside?(root, @root)
+          @warnings << "Skipped #{locked.name} #{locked.version}: local dependency is outside the project."
+        end
+        nil
+      end
+
+      def tool_only_dependency_names(parser, direct_names)
+        return Set.new unless direct_names.include?("rubylens")
+
+        dependencies = parser.specs.each_with_object(Hash.new { |hash, name| hash[name] = Set.new }) do |specification, graph|
+          specification.dependencies.each { |dependency| graph[specification.name] << dependency.name }
+        end
+        tool_reach = dependency_reach("rubylens", dependencies)
+        project_reach = (direct_names - ["rubylens"]).each_with_object(Set.new) do |root, names|
+          names.merge(dependency_reach(root, dependencies))
+        end
+        tool_reach - project_reach
+      end
+
+      def dependency_reach(root, dependencies)
+        visited = Set.new
+        pending = [root]
+        until pending.empty?
+          name = pending.pop
+          next unless visited.add?(name)
+
+          pending.concat(dependencies.fetch(name, []).to_a)
+        end
+        visited
+      end
+
+      def installed_specification(locked)
+        candidates = Gem::Specification.find_all_by_name(locked.name, "= #{locked.version}")
+        platform = locked.platform.to_s
+        candidates.find { |candidate| candidate.platform.to_s == platform } ||
+          candidates.find { |candidate| candidate.platform.to_s == Gem::Platform.local.to_s } ||
+          candidates.find { |candidate| candidate.platform.to_s == "ruby" } ||
+          candidates.first
+      end
+
+      def indexable_files(root, require_paths)
+        require_paths.filter_map do |relative_path|
+          next if File.absolute_path?(relative_path)
+
+          candidate = root.join(relative_path)
+          next unless candidate.exist?
+
+          resolved = candidate.realpath
+          next unless Paths.inside?(resolved, root)
+
+          resolved
+        rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP
+          nil
+        end.flat_map { |path| enumerate(path, root) }.uniq.sort
+      end
+
+      def enumerate(path, root)
+        return [path.to_s] if path.file? && indexable?(path) && Paths.inside?(path.realpath, root)
+        return [] unless path.directory?
+
+        files = []
+        Find.find(path) do |candidate|
+          candidate = Pathname(candidate)
+          next if candidate.directory?
+          next unless indexable?(candidate)
+
+          resolved = candidate.realpath
+          files << resolved.to_s if Paths.inside?(resolved, root)
+        rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP
+          next
+        end
+        files
+      end
+
+      def indexable?(path)
+        GitRepository::INDEXABLE_EXTENSIONS.include?(path.extname)
+      end
+    end
+  end
+end

data/lib/rubylens/index/rubydex_adapter.rb

@@ -0,0 +1,322 @@
+# frozen_string_literal: true
+
+require "rubydex"
+require "uri"
+require_relative "../model/dependency_aggregation"
+
+module RubyLens
+  module Index
+    class RubydexAdapter
+      TEST_SEGMENTS = %w[test tests spec specs feature features].freeze
+
+      def initialize(graph_factory: nil)
+        @graph_factory = graph_factory || ->(root) { Rubydex::Graph.new(workspace_path: root.to_s) }
+      end
+
+      def index(manifest)
+        @location_path_cache = {}
+        @workspace_location_cache = {}
+        graph = @graph_factory.call(manifest.root)
+        index_errors = Array(graph.index_all(manifest.files))
+        graph.resolve
+        integrity_failures = Array(graph.check_integrity)
+        collected = collect_declarations(graph.declarations, manifest)
+        workspace = workspace_namespaces(collected.fetch(:workspace_records), manifest)
+        inbound_references = inbound_workspace_references(graph, manifest, workspace.fetch(:ordinal_by_name))
+
+        {
+          "schema" => "rubylens.snapshot.v5",
+          "project_name" => project_name(manifest),
+          "components" => workspace.fetch(:component_counts),
+          "namespace_names" => workspace.fetch(:records).map { |declaration, _definitions| declaration.name },
+          "namespaces" => build_workspace_rows(workspace, inbound_references, manifest),
+          "category_stats" => collected.fetch(:category_stats),
+          "dependency_signal_maxima" => collected.fetch(:dependency_aggregation).signal_maxima,
+          "packages" => build_package_rows(collected.fetch(:dependency_aggregation), manifest),
+          "warning_counts" => {
+            "manifest" => manifest.warnings.length,
+            "index" => index_errors.length,
+            "integrity" => integrity_failures.length,
+          },
+        }
+      ensure
+        @location_path_cache = nil
+        @workspace_location_cache = nil
+      end
+
+      private
+
+      def collect_declarations(declarations, manifest)
+        records = []
+        category_stats = { "core" => Array.new(4, 0), "tests" => Array.new(4, 0) }
+        aggregation = Model::DependencyAggregation.new(package_count: manifest.packages.length)
+
+        declarations.each do |declaration|
+          if namespace?(declaration)
+            definitions = declaration.definitions.select do |definition|
+              canonical_namespace_definition?(declaration, definition) && workspace_location?(definition.location, manifest)
+            end
+            records << [declaration, definitions] unless definitions.empty?
+          end
+          collect_category_stat(category_stats, declaration, manifest)
+          collect_dependency_declaration(aggregation, declaration, manifest)
+        end
+
+        { workspace_records: records, category_stats:, dependency_aggregation: aggregation }
+      end
+
+      def workspace_namespaces(records, manifest)
+        ordinal_by_name = records.each_with_index.to_h { |(declaration, _definitions), index| [declaration.name, index] }
+        components = records.map { |_declaration, definitions| component_for(definitions, manifest) }
+        component_ids = components.uniq.sort.each_with_index.to_h
+
+        {
+          records: records,
+          ordinal_by_name: ordinal_by_name,
+          component_ids: component_ids,
+          components: components,
+          component_counts: components.tally.sort_by { |name, _count| component_ids.fetch(name) }.map(&:last),
+        }
+      end
+
+      def build_workspace_rows(workspace, inbound_references, manifest)
+        workspace.fetch(:records).each_with_index.map do |(declaration, definitions), index|
+          sites = definitions.map { |definition| site_key(definition.location) }.uniq.length
+          scope = scope_for(definitions, manifest)
+          descendants = declaration.descendants.count do |descendant|
+            descendant.name != declaration.name && workspace.fetch(:ordinal_by_name).key?(descendant.name)
+          end
+          [
+            workspace.fetch(:component_ids).fetch(workspace.fetch(:components)[index]),
+            declaration.is_a?(Rubydex::Class) ? 0 : 1,
+            scope,
+            [declaration.ancestors.count - 1, 0].max,
+            sites,
+            [sites - 1, 0].max,
+            descendants,
+            inbound_references.fetch(index, 0),
+            workspace_member_count(declaration, manifest),
+            *namespace_ruby_counts(declaration, manifest),
+            namespace_instance_variable_count(declaration, manifest, scope),
+          ]
+        end
+      end
+
+      def build_package_rows(aggregation, manifest)
+        aggregates = aggregation.packages
+        manifest.packages.each_with_index.map do |package, index|
+          aggregate = aggregates.fetch(index)
+          {
+            "name" => package.name,
+            "role" => package.role == "direct" ? 0 : 1,
+            "location" => package.location == "workspace" ? 0 : 1,
+            "declaration_count" => aggregate.fetch(:declaration_count),
+            "ruby_counts" => aggregate.fetch(:ruby_counts),
+            "declarations" => aggregate.fetch(:declarations),
+          }
+        end
+      end
+
+      def collect_dependency_declaration(aggregation, declaration, manifest)
+        package_index, definitions = dominant_package_definitions(declaration, manifest)
+        return unless package_index
+
+        sites = definitions.map { |definition| site_key(definition.location) }.uniq.length
+        row = [
+          namespace_kind(declaration),
+          namespace?(declaration) ? [declaration.ancestors.count - 1, 0].max : 0,
+          sites,
+          [sites - 1, 0].max,
+          namespace?(declaration) ? [declaration.descendants.count - 1, 0].max : 0,
+          safe_length(declaration, :references),
+          namespace?(declaration) ? safe_length(declaration, :members) : 0,
+        ]
+        aggregation.add(package_index:, row:, construct_index: ruby_construct_index(declaration))
+      end
+
+      def inbound_workspace_references(graph, manifest, ordinal_by_name)
+        graph.constant_references.each_with_object(Hash.new(0)) do |reference, counts|
+          next unless reference.respond_to?(:declaration)
+
+          next unless workspace_location?(reference.location, manifest)
+
+          target = reference.declaration
+          ordinal = ordinal_by_name[target.name]
+          counts[ordinal] += 1 if ordinal
+        rescue StandardError
+          next
+        end
+      end
+
+      def dominant_package_definitions(declaration, manifest)
+        grouped = declaration.definitions.group_by do |definition|
+          package_index_for_location(definition.location, manifest)
+        end
+        grouped.reject { |index, _records| index.nil? }
+          .max_by { |index, records| [records.length, -index] }
+      end
+
+      def package_index_for_location(location, manifest)
+        uri_string = location.uri
+        return nil unless URI.parse(uri_string).scheme == "file"
+
+        manifest.package_index_for(location_path(location, uri_string))
+      end
+
+      def workspace_member_count(declaration, manifest)
+        members = declaration.members.to_a
+        members.concat(declaration.singleton_class.members.to_a) if declaration.singleton_class
+        members.uniq(&:name).count do |member|
+          member.definitions.any? { |definition| workspace_location?(definition.location, manifest) }
+        end
+      rescue StandardError
+        0
+      end
+
+      def namespace_ruby_counts(declaration, manifest)
+        counts = Array.new(4, 0)
+        own_construct = ruby_construct_index(declaration)
+        counts[own_construct] += 1 if own_construct == 0 || own_construct == 1
+
+        members = declaration.members.to_a
+        members.concat(declaration.singleton_class.members.to_a) if declaration.singleton_class
+        members.uniq(&:name).each do |member|
+          construct_index = ruby_construct_index(member)
+          next unless construct_index && construct_index >= 2
+          next unless member.definitions.any? { |definition| workspace_location?(definition.location, manifest) }
+
+          counts[construct_index] += 1
+        end
+        counts
+      rescue StandardError
+        counts || Array.new(4, 0)
+      end
+
+      def namespace_instance_variable_count(declaration, manifest, scope)
+        return 0 unless declaration.is_a?(Rubydex::Class) && scope != 1
+
+        declaration.members.to_a.uniq(&:name).count do |member|
+          member.is_a?(Rubydex::InstanceVariable) &&
+            member.definitions.any? { |definition| workspace_location?(definition.location, manifest) }
+        end
+      rescue StandardError
+        0
+      end
+
+      def collect_category_stat(stats, declaration, manifest)
+        construct_index = ruby_construct_index(declaration)
+        return unless construct_index
+
+        definitions = declaration.definitions.select do |definition|
+          workspace_location?(definition.location, manifest)
+        end
+        return if definitions.empty?
+
+        category = scope_for(definitions, manifest) == 1 ? "tests" : "core"
+        stats.fetch(category)[construct_index] += 1
+      rescue StandardError
+        nil
+      end
+
+      def workspace_location?(location, manifest)
+        uri = location.uri
+        @workspace_location_cache ||= {}
+        return @workspace_location_cache[uri] if @workspace_location_cache.key?(uri)
+
+        @workspace_location_cache[uri] = manifest.workspace_path?(location_path(location, uri))
+      rescue StandardError
+        false
+      end
+
+      def component_for(definitions, manifest)
+        candidates = definitions.filter_map do |definition|
+          relative = manifest.relative_workspace_path(location_path(definition.location))
+          next unless relative
+
+          segments = relative.split(File::SEPARATOR)
+          first = segments.first || "root"
+          if %w[lib app test tests spec specs].include?(first)
+            "#{first}/#{segments[1] || "root"}"
+          else
+            first
+          end
+        end
+        candidates.tally.max_by { |name, count| [count, name] }&.first || "root"
+      end
+
+      def scope_for(definitions, manifest)
+        scopes = definitions.filter_map do |definition|
+          relative = manifest.relative_workspace_path(location_path(definition.location))
+          next unless relative
+
+          relative.split(File::SEPARATOR).any? { |segment| TEST_SEGMENTS.include?(segment) } ? 1 : 0
+        end.uniq
+        scopes.length > 1 ? 2 : scopes.first || 0
+      end
+
+      def site_key(location)
+        [location_path(location), location.start_line, location.start_column, location.end_line, location.end_column]
+      end
+
+      def namespace?(declaration)
+        declaration.is_a?(Rubydex::Namespace)
+      end
+
+      def canonical_namespace_definition?(declaration, definition)
+        (declaration.is_a?(Rubydex::Class) && definition.is_a?(Rubydex::ClassDefinition)) ||
+          (declaration.is_a?(Rubydex::Module) && definition.is_a?(Rubydex::ModuleDefinition))
+      end
+
+      def namespace_kind(declaration)
+        case declaration
+        when Rubydex::Class then 0
+        when Rubydex::Module then 1
+        else 2
+        end
+      end
+
+      def ruby_construct_index(declaration)
+        case declaration
+        when Rubydex::SingletonClass then nil
+        when Rubydex::Class then 0
+        when Rubydex::Module then 1
+        when Rubydex::Method then 2
+        when Rubydex::Constant, Rubydex::ConstantAlias then 3
+        end
+      end
+
+      def location_path(location, uri_string = nil)
+        uri_string ||= location.uri
+        @location_path_cache ||= {}
+        return @location_path_cache[uri_string] if @location_path_cache.key?(uri_string)
+
+        uri = URI.parse(uri_string)
+        raise Error, "Rubydex returned a non-file location" unless uri.scheme == "file"
+
+        @location_path_cache[uri_string] = URI::RFC2396_PARSER.unescape(uri.path)
+      end
+
+      def safe_length(object, method)
+        records = object.public_send(method)
+        size = records.size
+        size.nil? ? safe_count(records) : size
+      rescue StandardError
+        safe_count(records)
+      end
+
+      def safe_count(records)
+        records ? records.count : 0
+      rescue StandardError
+        0
+      end
+
+      def project_name(manifest)
+        basename = manifest.root.basename.to_s
+        return "IRB" if basename.casecmp("irb").zero?
+        return "RDoc" if basename.casecmp("rdoc").zero?
+
+        basename.split(/[-_]+/).map(&:capitalize).join(" ")
+      end
+    end
+  end
+end

data/lib/rubylens/model/dependency_aggregation.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module RubyLens
+  module Model
+    class DependencyAggregation
+      DEFAULT_ROW_LIMIT = 18_000
+      SIGNAL_COLUMNS = (1..6).freeze
+
+      def initialize(package_count:, row_limit: DEFAULT_ROW_LIMIT, seed: 0x51A7_E11A)
+        raise ArgumentError, "package_count must be nonnegative" if package_count.negative?
+        raise ArgumentError, "row_limit must be nonnegative" if row_limit.negative?
+
+        @row_limit = row_limit
+        @random = Random.new(seed)
+        @ordinal = 0
+        @seen_nonrepresentative = 0
+        @counts = Array.new(package_count, 0)
+        @ruby_counts = Array.new(package_count) { Array.new(4, 0) }
+        @signal_maxima = Array.new(6, 0)
+        @representatives = Array.new(package_count)
+        @represented_packages = []
+        @representative_count = 0
+        @nonempty_package_count = 0
+        @reservoir = []
+      end
+
+      def add(package_index:, row:, construct_index:)
+        @counts[package_index] += 1
+        @ruby_counts[package_index][construct_index] += 1 if construct_index
+        SIGNAL_COLUMNS.each_with_index do |column, index|
+          @signal_maxima[index] = [@signal_maxima[index], row[column]].max
+        end
+
+        entry = [package_index, @ordinal, row.dup.freeze]
+        @ordinal += 1
+        if @counts[package_index] == 1
+          add_first_package_entry(package_index, entry)
+        elsif @representatives[package_index] && @random.rand(@counts[package_index]).zero?
+          displaced = @representatives[package_index]
+          @representatives[package_index] = entry
+          add_to_reservoir(displaced)
+        else
+          add_to_reservoir(entry)
+        end
+      end
+
+      def packages
+        sampled_rows = Array.new(@counts.length) { [] }
+        (@representatives.compact + @reservoir).sort_by { |_package_index, ordinal, _row| ordinal }
+          .each { |package_index, _ordinal, row| sampled_rows[package_index] << row }
+        @counts.each_index.map do |index|
+          {
+            declaration_count: @counts[index],
+            ruby_counts: @ruby_counts[index].dup.freeze,
+            declarations: sampled_rows[index].freeze,
+          }.freeze
+        end.freeze
+      end
+
+      def signal_maxima
+        @signal_maxima.dup.freeze
+      end
+
+      private
+
+      def add_first_package_entry(package_index, entry)
+        @nonempty_package_count += 1
+        if @representative_count < @row_limit
+          @representatives[package_index] = entry
+          @represented_packages << package_index
+          @representative_count += 1
+          trim_reservoir
+        elsif @row_limit.positive? && @random.rand(@nonempty_package_count) < @row_limit
+          slot = @random.rand(@represented_packages.length)
+          displaced_package = @represented_packages[slot]
+          add_to_reservoir(@representatives[displaced_package])
+          @representatives[displaced_package] = nil
+          @representatives[package_index] = entry
+          @represented_packages[slot] = package_index
+        else
+          add_to_reservoir(entry)
+        end
+      end
+
+      def add_to_reservoir(entry)
+        capacity = reservoir_capacity
+        replacement = @random.rand(@seen_nonrepresentative + 1)
+        if @reservoir.length < capacity
+          @reservoir << entry
+        elsif replacement < capacity
+          @reservoir[replacement] = entry
+        end
+        @seen_nonrepresentative += 1
+      end
+
+      def trim_reservoir
+        @reservoir.delete_at(@random.rand(@reservoir.length)) while @reservoir.length > reservoir_capacity
+      end
+
+      def reservoir_capacity
+        @row_limit - @representative_count
+      end
+    end
+  end
+end

data/lib/rubylens/paths.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module RubyLens
+  module Paths
+    module_function
+
+    def inside?(path, directory)
+      path = File.expand_path(path)
+      directory = File.expand_path(directory)
+      path == directory || path.start_with?("#{directory}#{File::SEPARATOR}")
+    end
+  end
+end