scint 0.1.0

data/lib/scint/index/client.rb

@@ -0,0 +1,301 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+require "zlib"
+require_relative "cache"
+require_relative "parser"
+
+module Scint
+  module Index
+    # Compact index client for rubygems.org (or any compact index source).
+    # Thread-safe. Uses ETag/Range for efficient updates.
+    class Client
+      ACCEPT_ENCODING = "gzip"
+      USER_AGENT = "scint/0.1.0"
+      DEFAULT_TIMEOUT = 15
+
+      attr_reader :source_uri
+
+      def initialize(source_uri, cache_dir: nil, credentials: nil)
+        @source_uri = source_uri.to_s.chomp("/")
+        @uri = URI.parse(@source_uri)
+        @cache = Cache.new(cache_dir || default_cache_dir)
+        @parser = Parser.new
+        @credentials = credentials
+        @mutex = Thread::Mutex.new
+        @fetched = {}  # track which endpoints we've already fetched this session
+        @connections = Thread::Queue.new
+      end
+
+      # Fetch the list of all gem names from this source.
+      def fetch_names
+        data = fetch_endpoint("names")
+        @parser.parse_names(data)
+      end
+
+      # Fetch the versions list. Returns { name => [[name, version, platform], ...] }
+      # Also populates info checksums for cache validation.
+      def fetch_versions
+        data = fetch_endpoint("versions")
+        @parser.parse_versions(data)
+      end
+
+      # Fetch info for a single gem. Returns parsed info entries.
+      # Uses binary cache when checksum matches.
+      def fetch_info(gem_name)
+        checksums = @parser.info_checksums
+        checksum = checksums[gem_name]
+
+        # Try binary cache first
+        if checksum && !checksum.empty?
+          cached = @cache.read_binary_info(gem_name, checksum)
+          return cached if cached
+        end
+
+        # Check if local info file matches remote checksum
+        if checksum && !checksum.empty? && @cache.info_fresh?(gem_name, checksum)
+          data = @cache.info(gem_name)
+        else
+          data = fetch_info_endpoint(gem_name)
+        end
+
+        return [] unless data
+
+        result = @parser.parse_info(gem_name, data)
+
+        # Write binary cache
+        if checksum && !checksum.empty? && !result.empty?
+          @cache.write_binary_info(gem_name, checksum, result)
+        end
+
+        result
+      end
+
+      # Prefetch info for multiple gems concurrently.
+      # Uses a thread pool for parallel HTTP requests.
+      def prefetch(gem_names, worker_count: nil)
+        names = Array(gem_names).uniq
+        return if names.empty?
+
+        # Ensure versions are fetched first (populates checksums)
+        fetch_versions unless @parser.info_checksums.any?
+
+        # Filter to names that actually need fetching
+        checksums = @parser.info_checksums
+        to_fetch = names.select do |name|
+          checksum = checksums[name]
+          if checksum && !checksum.empty?
+            cached = @cache.read_binary_info(name, checksum)
+            !cached
+          else
+            true
+          end
+        end.reject do |name|
+          checksum = checksums[name]
+          checksum && !checksum.empty? && @cache.info_fresh?(name, checksum)
+        end
+
+        return if to_fetch.empty?
+
+        pool_size = worker_count || [to_fetch.size, 8].min
+        results = {}
+        queue = Thread::Queue.new
+        to_fetch.each { |n| queue.push(n) }
+        pool_size.times { queue.push(:done) }
+
+        threads = pool_size.times.map do
+          Thread.new do
+            while (name = queue.pop) != :done
+              begin
+                data = fetch_info_endpoint(name)
+                if data
+                  parsed = @parser.parse_info(name, data)
+                  checksum = checksums[name]
+                  if checksum && !checksum.empty? && !parsed.empty?
+                    @cache.write_binary_info(name, checksum, parsed)
+                  end
+                  @mutex.synchronize { results[name] = parsed }
+                end
+              rescue StandardError => e
+                $stderr.puts "prefetch warning: #{name}: #{e.message}" if ENV["SCINT_DEBUG"]
+              end
+            end
+          end
+        end
+
+        threads.each(&:join)
+        results
+      end
+
+      # Shut down any pooled connections.
+      def close
+        while !@connections.empty?
+          begin
+            conn = @connections.pop(true)
+            conn.finish if conn.started?
+          rescue StandardError
+            # ignore
+          end
+        end
+      end
+
+      private
+
+      def default_cache_dir
+        xdg = ENV["XDG_CACHE_HOME"] || File.join(Dir.home, ".cache")
+        File.join(xdg, "scint", "index", Cache.slug_for(@uri))
+      end
+
+      # Fetch a top-level endpoint (names or versions).
+      # Uses ETag for conditional requests and Range for incremental versions updates.
+      def fetch_endpoint(endpoint)
+        @mutex.synchronize do
+          return @fetched[endpoint] if @fetched.key?(endpoint)
+        end
+
+        if endpoint == "versions"
+          data = fetch_versions_with_range
+        else
+          etag = @cache.names_etag
+          response = http_get("#{@source_uri}/#{endpoint}", etag: etag)
+
+          case response
+          when Net::HTTPNotModified
+            data = @cache.names
+          when Net::HTTPSuccess
+            data = decode_body(response)
+            @cache.write_names(data, etag: extract_etag(response))
+          else
+            raise NetworkError, "Failed to fetch #{endpoint}: HTTP #{response.code}"
+          end
+        end
+
+        @mutex.synchronize { @fetched[endpoint] = data }
+        data
+      end
+
+      def fetch_versions_with_range
+        etag = @cache.versions_etag
+        local_size = @cache.versions_size
+
+        if local_size > 0
+          # Try range request (subtract 1 byte for overlap)
+          response = http_get("#{@source_uri}/versions", etag: etag, range_start: local_size - 1)
+
+          case response
+          when Net::HTTPNotModified
+            return @cache.versions
+          when Net::HTTPPartialContent
+            body = decode_body(response)
+            # Skip the overlapping byte
+            tail = body.byteslice(1..)
+            if tail && !tail.empty?
+              @cache.write_versions(tail, etag: extract_etag(response), append: true)
+            end
+            return @cache.versions
+          when Net::HTTPSuccess
+            # Server ignored range, gave us full response
+            data = decode_body(response)
+            @cache.write_versions(data, etag: extract_etag(response))
+            return data
+          when Net::HTTPRequestedRangeNotSatisfiable
+            # Fall through to full fetch
+          else
+            raise NetworkError, "Failed to fetch versions (range): HTTP #{response.code}"
+          end
+        end
+
+        # Full fetch
+        response = http_get("#{@source_uri}/versions", etag: etag)
+        case response
+        when Net::HTTPNotModified
+          @cache.versions
+        when Net::HTTPSuccess
+          data = decode_body(response)
+          @cache.write_versions(data, etag: extract_etag(response))
+          data
+        else
+          raise NetworkError, "Failed to fetch versions: HTTP #{response.code}"
+        end
+      end
+
+      # Fetch info endpoint for a single gem.
+      def fetch_info_endpoint(gem_name)
+        etag = @cache.info_etag(gem_name)
+        response = http_get("#{@source_uri}/info/#{gem_name}", etag: etag)
+
+        case response
+        when Net::HTTPNotModified
+          @cache.info(gem_name)
+        when Net::HTTPSuccess
+          data = decode_body(response)
+          @cache.write_info(gem_name, data, etag: extract_etag(response))
+          data
+        when Net::HTTPNotFound
+          nil
+        else
+          raise NetworkError, "Failed to fetch info/#{gem_name}: HTTP #{response.code}"
+        end
+      end
+
+      def http_get(url, etag: nil, range_start: nil)
+        uri = URI.parse(url)
+        conn = checkout_connection(uri)
+
+        begin
+          request = Net::HTTP::Get.new(uri.request_uri)
+          request["User-Agent"] = USER_AGENT
+          request["Accept-Encoding"] = ACCEPT_ENCODING
+          request["If-None-Match"] = %("#{etag}") if etag
+          request["Range"] = "bytes=#{range_start}-" if range_start
+          @credentials&.apply!(request, uri)
+
+          conn.request(request)
+        ensure
+          checkin_connection(conn)
+        end
+      end
+
+      def checkout_connection(uri)
+        begin
+          conn = @connections.pop(true)
+          return conn if conn.started?
+        rescue ThreadError
+          # Queue empty
+        end
+
+        conn = Net::HTTP.new(uri.host, uri.port)
+        conn.use_ssl = (uri.scheme == "https")
+        conn.open_timeout = DEFAULT_TIMEOUT
+        conn.read_timeout = DEFAULT_TIMEOUT
+        conn.start
+        conn
+      end
+
+      def checkin_connection(conn)
+        @connections.push(conn) if conn.started?
+      rescue StandardError
+        # ignore
+      end
+
+      def decode_body(response)
+        body = response.body
+        return body unless body
+
+        if response["Content-Encoding"] == "gzip"
+          Zlib::GzipReader.new(StringIO.new(body)).read
+        else
+          body
+        end
+      end
+
+      def extract_etag(response)
+        return nil unless response["ETag"]
+        etag = response["ETag"].delete_prefix("W/")
+        etag = etag.delete_prefix('"').delete_suffix('"')
+        etag.empty? ? nil : etag
+      end
+    end
+  end
+end

data/lib/scint/index/parser.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+module Scint
+  module Index
+    class Parser
+      # Parse the compact index "names" endpoint response.
+      # Returns array of gem name strings.
+      def parse_names(data)
+        return [] if data.nil? || data.empty?
+        lines = strip_header(data)
+        lines
+      end
+
+      # Parse the compact index "versions" endpoint response.
+      # Returns a hash:
+      #   { name => [[name, version, platform], ...], ... }
+      # Also stores checksums accessible via #info_checksums.
+      def parse_versions(data)
+        @versions_by_name = Hash.new { |h, k| h[k] = [] }
+        @info_checksums = {}
+
+        return @versions_by_name if data.nil? || data.empty?
+
+        strip_header(data).each do |line|
+          line.freeze
+
+          name_end = line.index(" ")
+          next unless name_end
+
+          versions_end = line.index(" ", name_end + 1)
+          name = line[0, name_end].freeze
+
+          if versions_end
+            versions_string = line[name_end + 1, versions_end - name_end - 1]
+            @info_checksums[name] = line[versions_end + 1, line.size - versions_end - 1]
+          else
+            versions_string = line[name_end + 1, line.size - name_end - 1]
+            @info_checksums[name] = ""
+          end
+
+          versions_string.split(",") do |version|
+            delete = version.delete_prefix!("-")
+            parts = version.split("-", 2)
+            entry = parts.unshift(name)
+            if delete
+              @versions_by_name[name].delete(entry)
+            else
+              @versions_by_name[name] << entry
+            end
+          end
+        end
+
+        @versions_by_name
+      end
+
+      # Returns checksums collected during parse_versions.
+      # Keys are gem names, values are checksum strings.
+      def info_checksums
+        @info_checksums || {}
+      end
+
+      # Parse a compact index "info/{gem_name}" endpoint response.
+      # Returns array of entries:
+      #   [name, version, platform, deps_hash, requirements_hash]
+      #
+      # deps_hash: { "dep_name" => "version_constraints", ... }
+      # requirements_hash: { "ruby" => ">= 2.7", "rubygems" => ">= 3.0" }
+      def parse_info(name, data)
+        return [] if data.nil? || data.empty?
+
+        strip_header(data).map do |line|
+          parse_info_line(name, line)
+        end
+      end
+
+      private
+
+      # Strip the "---\n" header that compact index responses may include.
+      def strip_header(data)
+        lines = data.split("\n")
+        header = lines.index("---")
+        header ? lines[(header + 1)..] : lines
+      end
+
+      # Parse a single info line using the compact index format:
+      #   VERSION[-PLATFORM] DEP1:REQ1&REQ2,DEP2:REQ3|RUBY_REQ:VAL1&VAL2,RUBYGEMS_REQ:VAL3
+      #
+      # - Space separates version from the rest
+      # - "|" separates dependency list from requirement list
+      # - "," separates individual deps or reqs
+      # - ":" separates name from version constraints within each dep/req
+      # - "&" separates multiple version constraints for one dep/req
+      def parse_info_line(name, line)
+        version_and_platform, rest = line.split(" ", 2)
+
+        # Split version and platform
+        version, platform = version_and_platform.split("-", 2)
+        platform ||= "ruby"
+
+        # Split rest into deps and requirements by "|"
+        deps = {}
+        reqs = {}
+
+        if rest && !rest.empty?
+          deps_str, reqs_str = rest.split("|", 2)
+
+          # Parse dependencies
+          if deps_str && !deps_str.empty?
+            deps_str.split(",").each do |dep_entry|
+              parts = dep_entry.split(":")
+              dep_name = parts[0]
+              next if dep_name.nil? || dep_name.empty?
+              dep_name = -dep_name  # freeze and deduplicate
+              if parts.size > 1
+                deps[dep_name] = parts[1].split("&").join(", ")
+              else
+                deps[dep_name] = ">= 0"
+              end
+            end
+          end
+
+          # Parse requirements (ruby, rubygems version constraints)
+          if reqs_str && !reqs_str.empty?
+            reqs_str.split(",").each do |req_entry|
+              parts = req_entry.split(":")
+              req_name = parts[0]
+              next if req_name.nil? || req_name.empty?
+              req_name = -req_name.strip
+              if parts.size > 1
+                reqs[req_name] = parts[1].split("&").join(", ")
+              else
+                reqs[req_name] = ">= 0"
+              end
+            end
+          end
+        end
+
+        [name, version, platform, deps, reqs]
+      end
+    end
+  end
+end

data/lib/scint/installer/extension_builder.rb

@@ -0,0 +1,264 @@
+# frozen_string_literal: true
+
+require_relative "../fs"
+require_relative "../platform"
+require_relative "../errors"
+require "open3"
+
+module Scint
+  module Installer
+    module ExtensionBuilder
+      module_function
+
+      # Build native extensions for a prepared gem.
+      # prepared_gem: PreparedGem struct
+      # bundle_path:  .bundle/ root
+      # abi_key:      e.g. "ruby-3.3.0-arm64-darwin24" (defaults to Platform.abi_key)
+      def build(prepared_gem, bundle_path, cache_layout, abi_key: Platform.abi_key, compile_slots: 1, output_tail: nil)
+        spec = prepared_gem.spec
+        ruby_dir = ruby_install_dir(bundle_path)
+        build_ruby_dir = cache_layout.install_ruby_dir
+
+        # Check global extension cache first
+        cached_ext = cache_layout.ext_path(spec, abi_key)
+        if Dir.exist?(cached_ext) && File.exist?(File.join(cached_ext, "gem.build_complete"))
+          link_extensions(cached_ext, ruby_dir, spec, abi_key)
+          return true
+        end
+
+        # Build in a temp dir, then cache
+        src_dir = prepared_gem.extracted_path
+        ext_dirs = find_extension_dirs(src_dir)
+        raise ExtensionBuildError, "No extension directories found for #{spec.name}" if ext_dirs.empty?
+
+        FS.with_tempdir("scint-ext") do |tmpdir|
+          build_dir = File.join(tmpdir, "build")
+          install_dir = File.join(tmpdir, "install")
+          FS.mkdir_p(build_dir)
+          FS.mkdir_p(install_dir)
+
+          ext_dirs.each do |ext_dir|
+            compile_extension(ext_dir, build_dir, install_dir, src_dir, spec, build_ruby_dir, compile_slots, output_tail)
+          end
+
+          # Write marker
+          File.write(File.join(install_dir, "gem.build_complete"), "")
+
+          # Cache globally
+          FS.mkdir_p(File.dirname(cached_ext))
+          FS.atomic_move(install_dir, cached_ext)
+        end
+
+        link_extensions(cached_ext, ruby_dir, spec, abi_key)
+        true
+      end
+
+      # True when a completed global extension build exists for this spec + ABI.
+      def cached_build_available?(spec, cache_layout, abi_key: Platform.abi_key)
+        cached_ext = cache_layout.ext_path(spec, abi_key)
+        Dir.exist?(cached_ext) && File.exist?(File.join(cached_ext, "gem.build_complete"))
+      end
+
+      # Link already-compiled extensions from global cache into bundle_path.
+      # Returns true when cache was present and linked, false otherwise.
+      def link_cached_build(prepared_gem, bundle_path, cache_layout, abi_key: Platform.abi_key)
+        spec = prepared_gem.spec
+        return false unless cached_build_available?(spec, cache_layout, abi_key: abi_key)
+
+        ruby_dir = ruby_install_dir(bundle_path)
+        cached_ext = cache_layout.ext_path(spec, abi_key)
+        link_extensions(cached_ext, ruby_dir, spec, abi_key)
+        true
+      end
+
+      # --- private ---
+
+      def buildable_source_dir?(gem_dir)
+        find_extension_dirs(gem_dir).any?
+      end
+
+      def find_extension_dirs(gem_dir)
+        dirs = []
+
+        # extconf.rb in ext/ subdirectories
+        Dir.glob(File.join(gem_dir, "ext", "**", "extconf.rb")).each do |path|
+          dirs << File.dirname(path)
+        end
+
+        # CMakeLists.txt in ext/
+        Dir.glob(File.join(gem_dir, "ext", "**", "CMakeLists.txt")).each do |path|
+          dir = File.dirname(path)
+          dirs << dir unless dirs.include?(dir)
+        end
+
+        # Rakefile-based (look for Rakefile in ext/)
+        if dirs.empty?
+          Dir.glob(File.join(gem_dir, "ext", "**", "Rakefile")).each do |path|
+            dirs << File.dirname(path)
+          end
+        end
+
+        dirs.uniq
+      end
+
+      def compile_extension(ext_dir, build_dir, install_dir, gem_dir, spec, build_ruby_dir, compile_slots, output_tail = nil)
+        make_jobs = adaptive_make_jobs(compile_slots)
+        env = build_env(gem_dir, build_ruby_dir, make_jobs)
+
+        if File.exist?(File.join(ext_dir, "extconf.rb"))
+          compile_extconf(ext_dir, build_dir, install_dir, env, make_jobs, output_tail)
+        elsif File.exist?(File.join(ext_dir, "CMakeLists.txt"))
+          compile_cmake(ext_dir, build_dir, install_dir, env, make_jobs, output_tail)
+        elsif File.exist?(File.join(ext_dir, "Rakefile"))
+          compile_rake(ext_dir, build_dir, install_dir, build_ruby_dir, env, output_tail)
+        else
+          raise ExtensionBuildError, "No known build system in #{ext_dir}"
+        end
+      end
+
+      def compile_extconf(ext_dir, build_dir, install_dir, env, make_jobs, output_tail = nil)
+        run_cmd(env, RbConfig.ruby, File.join(ext_dir, "extconf.rb"),
+                "--with-opt-dir=#{RbConfig::CONFIG["prefix"]}",
+                chdir: build_dir, output_tail: output_tail)
+        run_cmd(env, "make", "-j#{make_jobs}", "-C", build_dir, output_tail: output_tail)
+        run_cmd(env, "make", "install", "DESTDIR=", "sitearchdir=#{install_dir}", "sitelibdir=#{install_dir}",
+                chdir: build_dir, output_tail: output_tail)
+      end
+
+      def compile_cmake(ext_dir, build_dir, install_dir, env, make_jobs, output_tail = nil)
+        run_cmd(env, "cmake", ext_dir, "-B", build_dir,
+                "-DCMAKE_INSTALL_PREFIX=#{install_dir}", output_tail: output_tail)
+        run_cmd(env, "cmake", "--build", build_dir, "--parallel", make_jobs.to_s, output_tail: output_tail)
+        run_cmd(env, "cmake", "--install", build_dir, output_tail: output_tail)
+      end
+
+      def compile_rake(ext_dir, build_dir, install_dir, ruby_dir, env, output_tail = nil)
+        rake_exe = find_rake_executable(ruby_dir)
+        begin
+          if rake_exe
+            run_cmd(env, RbConfig.ruby, rake_exe, "compile",
+                    chdir: ext_dir, output_tail: output_tail)
+          else
+            run_cmd(env, RbConfig.ruby, "-S", "rake", "compile",
+                    chdir: ext_dir, output_tail: output_tail)
+          end
+        rescue ExtensionBuildError => e
+          # Some gems ship a Rakefile but do not expose a compile task.
+          # Treat this as "nothing to build" rather than a hard failure.
+          raise unless e.message.include?("Don't know how to build task 'compile'")
+          return
+        end
+        # Copy built artifacts to install_dir
+        Dir.glob(File.join(ext_dir, "**", "*.{so,bundle,dll,dylib}")).each do |so|
+          FileUtils.cp(so, install_dir)
+        end
+      end
+
+      def find_rake_executable(ruby_dir)
+        gems_dir = File.join(ruby_dir, "gems")
+        return nil unless Dir.exist?(gems_dir)
+
+        # Prefer highest installed rake version.
+        rake_dirs = Dir.glob(File.join(gems_dir, "rake-*")).sort.reverse
+        rake_dirs.each do |dir|
+          %w[exe bin].each do |subdir|
+            path = File.join(dir, subdir, "rake")
+            return path if File.file?(path)
+          end
+        end
+        nil
+      end
+
+      def link_extensions(cached_ext, ruby_dir, spec, abi_key)
+        ext_install_dir = File.join(ruby_dir, "extensions",
+                                    Platform.gem_arch, Platform.extension_api_version,
+                                    spec_full_name(spec))
+        return if Dir.exist?(ext_install_dir)
+
+        FS.hardlink_tree(cached_ext, ext_install_dir)
+      end
+
+      def build_env(gem_dir, build_ruby_dir, make_jobs)
+        ruby_bin = File.join(build_ruby_dir, "bin")
+        path = [ruby_bin, ENV["PATH"]].compact.reject(&:empty?).join(File::PATH_SEPARATOR)
+        {
+          "GEM_HOME" => build_ruby_dir,
+          "GEM_PATH" => build_ruby_dir,
+          "BUNDLE_PATH" => build_ruby_dir,
+          "BUNDLE_GEMFILE" => "",
+          "MAKEFLAGS" => "-j#{make_jobs}",
+          "PATH" => path,
+          "CFLAGS" => "-I#{RbConfig::CONFIG["rubyhdrdir"]} -I#{RbConfig::CONFIG["rubyarchhdrdir"]}",
+        }
+      end
+
+      def adaptive_make_jobs(compile_slots)
+        slots = [compile_slots.to_i, 1].max
+        jobs = Platform.cpu_count / slots
+        [jobs, 1].max
+      end
+
+      def run_cmd(env, *cmd, chdir: nil, output_tail: nil)
+        opts = { chdir: chdir }.compact
+
+        if ENV["SCINT_DEBUG"]
+          pid = Process.spawn(env, *cmd, **opts)
+          _, status = Process.wait2(pid)
+          unless status.success?
+            raise ExtensionBuildError,
+                  "Command failed (exit #{status.exitstatus}): #{cmd.join(" ")}"
+          end
+          return
+        end
+
+        # Stream output line-by-line so the UX gets live compile progress
+        # instead of waiting for the entire subprocess to finish.
+        all_output = +""
+        tail_lines = []
+        cmd_label = "$ #{cmd.join(" ")}"
+
+        Open3.popen2e(env, *cmd, **opts) do |stdin, out_err, wait_thr|
+          stdin.close
+
+          out_err.each_line do |line|
+            stripped = line.rstrip
+            all_output << line
+            next if stripped.empty?
+
+            tail_lines << stripped
+            tail_lines.shift if tail_lines.length > 5
+
+            if output_tail
+              output_tail.call([cmd_label, *tail_lines])
+            end
+          end
+
+          status = wait_thr.value
+          unless status.success?
+            details = all_output.strip
+            message = "Command failed (exit #{status.exitstatus}): #{cmd.join(" ")}"
+            message = "#{message}\n#{details}" unless details.empty?
+            raise ExtensionBuildError, message
+          end
+        end
+      end
+
+      def spec_full_name(spec)
+        name = spec.name
+        version = spec.version
+        plat = spec.respond_to?(:platform) ? spec.platform : nil
+        base = "#{name}-#{version}"
+        (plat.nil? || plat.to_s == "ruby" || plat.to_s.empty?) ? base : "#{base}-#{plat}"
+      end
+
+      def ruby_install_dir(bundle_path)
+        File.join(bundle_path, "ruby", RUBY_VERSION.split(".")[0, 2].join(".") + ".0")
+      end
+
+      private_class_method :find_extension_dirs, :compile_extension,
+                           :compile_extconf, :compile_cmake, :compile_rake,
+                           :find_rake_executable, :link_extensions, :build_env, :run_cmd,
+                           :spec_full_name, :ruby_install_dir
+    end
+  end
+end